diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0646198..13f2301 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,8 +15,8 @@ jobs: with: python-version: "3.11" - run: pip install ruff - - run: ruff check src/ tests/ examples/ reproduce/ - - run: ruff format --check src/ tests/ examples/ reproduce/ + - run: ruff check src/ tests/ examples/ reproduce/ baselines/ + - run: ruff format --check src/ tests/ examples/ reproduce/ baselines/ test: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 6535576..414ec29 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ dev: test: pytest -LINT_PATHS = src/ tests/ examples/ reproduce/ +LINT_PATHS = src/ tests/ examples/ reproduce/ baselines/ lint: ruff check $(LINT_PATHS) diff --git a/README.md b/README.md index cb44c5d..9721772 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ is `three_dbench`. |---|---| | Datasets | [`EscheWang/3dcs`](https://huggingface.co/datasets/EscheWang/3dcs), configs `chirality`, `rotation`, `traj_frames`, `traj_energies` (license CC BY-SA 4.0) | | Baseline embeddings and original metric outputs | [`EscheWang/3dcs-embeddings`](https://huggingface.co/datasets/EscheWang/3dcs-embeddings) (see [docs/EMBEDDINGS.md](docs/EMBEDDINGS.md)) | +| Embedding-extraction scripts | [baselines/](baselines/README.md), one per model | | Metric definitions (`paper` and `v2`) | [docs/METRICS.md](docs/METRICS.md) | | Per-table reproduction scripts | [reproduce/](reproduce/README.md) | | rMD17 splits | [splits/rmd17/](splits/rmd17/README.md) | @@ -82,7 +83,7 @@ python -m three_dbench evaluate rotation \ Every evaluation writes a summary (`summary.csv`), per-molecule results and the configuration to `--output-dir` (default `results//`, relative to the working directory or `$THREE_DBENCH_HOME`). `--metric-version paper` (default) uses the definitions behind the published -numbers; `--metric-version v2` uses the corrected definitions described in +numbers; `--metric-version v2` uses the alternative definitions described in [docs/METRICS.md](docs/METRICS.md). See [docs/USAGE.md](docs/USAGE.md) for all options, embedding layouts and the Python API. @@ -92,11 +93,39 @@ A small demo with bundled GemNet fixtures: python examples/demo.py all ``` -## Reproducing the paper +## Baseline embeddings + +The embeddings of the seven baseline models are published in +[`EscheWang/3dcs-embeddings`](https://huggingface.co/datasets/EscheWang/3dcs-embeddings) and are +what the reference values in [reproduce/](reproduce/README.md) are computed from. +[docs/EMBEDDINGS.md](docs/EMBEDDINGS.md) lists the array key, shape and layout of every file. + +[baselines/](baselines/README.md) holds one extraction script per model, which turns the chirality +conformers into the same matrix: + +```bash +python baselines/mace/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality --out chirality_mace.npz \ + --device cuda --batch-size 1 --compress --verify +``` + +All seven take the same `--dataset` values — `hf:[:]`, `hfdisk:` or a plain +`save_to_disk` directory, a bare Hub dataset id, a pickle of RDKit molecules, or `lmdb:` — +print the versions and checksums of everything they used, and write the documented array key. +`--verify` compares the result with the published file of that model and prints the checksums, the +elementwise differences and the per-row cosine similarity; with `--limit`/`--start` it compares the +rows the run covers (`--verify-rows`). No third-party code or weights are redistributed: every model +directory has an `ENVIRONMENT.md` with the upstream repository and commit, the weight file with its +SHA-256 and where to download it, and the exact install commands. +[baselines/README.md](baselines/README.md) tabulates, per model, the output dimension, array key, +hydrogen handling, pooling and the measured agreement with the published file. + +## Reproducing the tables Each table has a directory under [`reproduce/`](reproduce/README.md) with a `run.sh` (download, -evaluation, `results.csv`) and an `expected.csv` (printed value, recomputed value with 6 decimals, -tolerance and notes). `reproduce/compare.py` prints PASS/FAIL per cell. +evaluation, `results.csv`) and an `expected.csv` (the value as printed in the paper, the reference +value computed with this code at 6 decimals, a tolerance and notes). `reproduce/compare.py` compares +a run against the reference values and prints PASS/FAIL per cell. | Table | Script | What it recomputes | Runtime (24 workers) | |---|---|---|---| @@ -107,40 +136,36 @@ tolerance and notes). `reproduce/compare.py` prints PASS/FAIL per cell. Download sizes: rotation dataset ~7.5 GB, rotation GemNet embeddings ~5.2 GB, trajectory embeddings ~7 GB, chirality embeddings ~0.4 GB. -### Status +### Coverage -"Reproducible" means that the published value is recomputed from released data, embeddings and code -within the tolerance in `expected.csv`; the notes in each `expected.csv` list the known exceptions. +Each `expected.csv` holds the reference values this code computes for the cells that the published +artifacts cover: -| Table | E3FP | GemNet | MolAE | MolSpectra | UniMol | FMG | MACE | -|---|---|---|---|---|---|---|---| -| 1 Geometry | pending (rotation embeddings) | full 16-shard run pending | pending (rotation embeddings) | pending (rotation embeddings) | pending (rotation embeddings) | not in table | not in table | -| 2 Chirality (zero-shot) | reproducible | reproducible | reproducible | reproducible | reproducible | reproducible | reproducible | -| 3 / 6 / 7 Energy (zero-shot) | reproducible | reproducible | reproducible | reproducible | reproducible | partially | partially | -| 4 Chirality correlation | pending (evaluation code) | pending (evaluation code) | pending (evaluation code) | pending (evaluation code) | pending (evaluation code) | not in table | not in table | -| 5 Chirality fine-tuning | not in table | pending | pending | pending | pending | pending | pending | -| 8 / 9 rMD17 fine-tuning | pending | pending | pending | pending | pending | pending | pending | +| Table | What the published artifacts cover | +|---|---| +| 1 Geometry | GemNet rotation embeddings (all 16 shards); for E3FP, MolAE, MolSpectra and UniMol, the per-molecule outputs of the original runs (`results/rotation/` in the embeddings repository) | +| 2 Chirality (zero-shot) | chirality embeddings of all 7 models | +| 3 / 6 / 7 Energy (zero-shot) | trajectory embeddings of all 7 models | +| 4 Chirality correlation | the summary of the original run (`results/chirality/chirality_metrics_summary.csv`) and the embeddings of the earlier 15,218-conformer set (`chirality_legacy_15218/`); this release has no script for this table | +| 5 / 8 / 9 Fine-tuning | this repository is the evaluation toolkit; fine-tuning code and checkpoints are not part of it | Further notes: -- **Embedding extraction.** The published embeddings are the files used for the paper. Extraction - scripts are not part of this repository; the per-model status (known, partially known, unknown) is - listed in [docs/EMBEDDINGS.md](docs/EMBEDDINGS.md). +- **Embedding extraction.** The published embeddings are the files used for the paper's evaluations. + [docs/EMBEDDINGS.md](docs/EMBEDDINGS.md) documents the format, array key, shape and provenance of + each file, and [baselines/](baselines/README.md) holds the extraction script and environment of + each model. - **Table 1.** The published rows come from two runs: Spearman, Kendall, CKA, isotonic R² and Torsion-SP from a 10 % molecule sample (its key list is in `reproduce/table1_geometry/`), LIE@k and AS from all molecules. The per-molecule outputs of both runs for all five models are published in - the embeddings repository (`results/rotation/`). GemNet is the only model whose rotation embeddings - were kept, and a full 16-shard recomputation has not been run yet; `reproduce/table1_geometry/` - currently verifies one shard against the backed-up per-molecule outputs. Use - `--replicate-offset-drift` to match the row alignment of the published run; see - [docs/metrics/geometry.md](docs/metrics/geometry.md) for the exact definitions used by each mode. -- **Table 4.** The backed-up summary behind the Spearman, Kendall and CKA rows is published - (`results/chirality/chirality_metrics_summary.csv` in the embeddings repository); outputs for the - OPD rows were not found in our backups. The code for this table is not in the release. -- **Fine-tuning (Tables 5, 8, 9).** Fine-tuning code and checkpoints are not part of this release; - see [Recommended use](#recommended-use). The splits are released: the rMD17 tables use the official - split 01 ([splits/rmd17/](splits/rmd17/README.md)) and the chirality fine-tuning split is in - [splits/chirality_finetune/](splits/chirality_finetune/README.md). + the embeddings repository (`results/rotation/`). LIE@k and AS of that full run correspond to + embedding rows shifted by 3 and 7 positions in parts of shards 1 and 2; + `--replicate-offset-drift` recomputes the metrics with the same indexing, and the regular columns + use the per-shard `offset` of the dataset. See [docs/metrics/geometry.md](docs/metrics/geometry.md). +- **Splits.** The rMD17 fine-tuning inputs for Tables 8 and 9 correspond to the official split 01 + ([splits/rmd17/](splits/rmd17/README.md)); the chirality fine-tuning split is in + [splits/chirality_finetune/](splits/chirality_finetune/README.md). Fine-tuning code and checkpoints + are not part of this repository; see [Recommended use](#recommended-use). ### Recommended use @@ -175,6 +200,7 @@ distributed). - [docs/USAGE.md](docs/USAGE.md): CLI and Python API - [docs/EMBEDDINGS.md](docs/EMBEDDINGS.md): embedding formats and the published baseline embeddings +- [baselines/README.md](baselines/README.md): the embedding-extraction script of each baseline model - [docs/METRICS.md](docs/METRICS.md): metric definitions (`paper` and `v2`) - [reproduce/README.md](reproduce/README.md): reproducing the paper tables - [CONTRIBUTING.md](CONTRIBUTING.md): development setup diff --git a/baselines/README.md b/baselines/README.md new file mode 100644 index 0000000..a99fca9 --- /dev/null +++ b/baselines/README.md @@ -0,0 +1,273 @@ +# Baseline embedding extraction + +One script per baseline model that turns the 3DCS chirality conformers into the embedding +matrix the evaluator reads. Each script is standalone, runs in its own model environment and +writes a file in the same layout as the corresponding published embedding in +[`EscheWang/3dcs-embeddings`](https://huggingface.co/datasets/EscheWang/3dcs-embeddings). + +``` +baselines/ + common.py input loading, npz writing, and the --verify comparison + / + extract_chirality.py + ENVIRONMENT.md upstream repository + commit, weight file + sha256 + download + location, exact install commands, run command, cost + requirements.txt pinned Python environment +``` + +No third-party source code and no model weights are redistributed here. Every script drives +an upstream checkout or package that you install yourself, and `ENVIRONMENT.md` says which +commit and which weight file, with its SHA-256 and where it comes from. + +## What each script produces + +| Model | Dim | Output key | Hydrogens | Pooling | Upstream dependency | +|---|---|---|---|---|---| +| [E3FP](e3fp/) | 1024 bits | `e3fp` (pickle) | kept | — (fingerprint) | `e3fp==1.2.7` | +| [GemNet-Q](gemnet/) | 128 | `gemnet` | `RemoveAllHs`, heavy atoms only | mean over atoms | `TUM-DAML/gemnet_pytorch` @ `a0164f7` + its `pretrained/GemNet-Q` | +| [Uni-Mol](unimol/) | 512 | `arr_0` | removed (no-H checkpoint) | `[CLS]` token | `deepmodeling/Uni-Mol` @ `90f52c4`, `dptech-corp/Uni-Core` @ `ace6fae`, `mol_pre_no_h_220816.pt` | +| [Mol-AE](molae/) | 512 | `arr_0` | removed (no-H checkpoint) | `[CLS]` token | same Uni-Mol / Uni-Core, Mol-AE `checkpoint_7_1000000.pt` | +| [MolSpectra](molspectra/) | 256 | `arr_0` | removed (`Z != 1`) | sum over atoms | `AzureLeon1/MolSpectra` @ `8846530` + a checkpoint you supply | +| [MACE](mace/) | 256 | `arr_0` | kept | mean over atoms | `mace-torch==0.3.15`, MACE-MP-0 `medium` | +| [FMG](fmg/) | 128 | `embeddings` | dropped (C, O, N, F channels only) | mean over the 24³ grid | `Dumitrescu-Alexandru/FMG` @ `13a0a7c` + `model-120qm9_3rd_run.pt` | + +Every file holds one row per conformer, in the row order of the `chirality` config of +[`EscheWang/3dcs`](https://huggingface.co/datasets/EscheWang/3dcs) (52,391 rows). E3FP writes +a pickle of RDKit bit vectors; the others write an `.npz` with a single float32 array. + +## Running one + +```bash +# in the environment of that model, see /ENVIRONMENT.md +python baselines/mace/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --out chirality_mace.npz \ + --device cuda --batch-size 1 --compress --verify +``` + +The command for each of the seven, with the flags that model needs, is under +[One command per model](#one-command-per-model). + +Common to all of them: + +- `--dataset` takes the same input specification in every script, resolved by + `common.py` (`parse_dataset_spec`): + + | `--dataset` | what it reads | needs | + |---|---|---| + | `hf:EscheWang/3dcs:chirality` | that Hub dataset and that config | `datasets` | + | `hf:EscheWang/3dcs`, `EscheWang/3dcs` | the same repository, config `chirality` | `datasets` | + | `hfdisk:data/hf/chirality`, `data/hf/chirality` | a `save_to_disk` directory of that config | `datasets` | + | `conformers.pkl` | a pickle of RDKit molecules: a list, or a dict of lists concatenated in insertion order | — | + | `lmdb:rotation_conformers_0.lmdb` | a rotation shard: one list of `(Mol, energy, torsion)` per key | `lmdb` | + + Rows of the Hugging Face config are read in ascending `offset`, and each row's `offset` is + checked against the running conformer count, so `mol_blocks` concatenated that way is the + row order of the published embedding files. `python -m three_dbench download dataset --task + chirality` writes the `save_to_disk` directory that the fourth form reads. +- `--limit N` stops after N conformers and `--start N` skips the first N, so a slice can be + run first; `--verify` then compares that slice (see `--verify-rows`). +- `--verify [REFERENCE]` compares the file just written with a reference: with no argument, + the published embedding of that model, downloaded from `EscheWang/3dcs-embeddings` + (`pip install huggingface_hub`); otherwise `hub:` or a local path. + It prints both checksums, the elementwise differences, the per-row cosine similarity and + the number of rows outside a few thresholds. It does not change the exit status. +- `--verify-rows` says which rows of the reference the output covers. The default compares a + short output with the first rows of the reference (`prefix`) and a full output row by row + (`full`); `--verify-rows 2000:4000` (or `2000+`) compares against that slice, and + `--verify-rows @rows.npy` against the listed 0-based reference rows. The report then names + the selection and how many rows it compared. +- Each script prints the versions of every numerically relevant library, its own arguments, + the SHA-256 of the weights it loaded and the SHA-256 of the file it wrote. +- Rows are never skipped: a conformer that cannot be featurised is an error, not a silent + shift of the row order. + +## One command per model + +Every command below reads the published dataset and verifies against the published +embedding. The paths in angle brackets are the checkout, weight file and dictionary that +`/ENVIRONMENT.md` gives the download command and SHA-256 for. Add `--limit 2000` to +any of them for a first run over a slice. + +```bash +# E3FP (CPU only) +python baselines/e3fp/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --out sampled_chi.pkl --jobs 24 --verify + +# MACE +python baselines/mace/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --out chirality.npz \ + --device cuda --batch-size 1 --compress --verify + +# Uni-Mol +python baselines/unimol/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --unimol-repo /unimol \ + --weights \ + --out chirality_unimol.npz \ + --batch-size 256 --device cuda:0 --verify + +# Mol-AE +python baselines/molae/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --weights \ + --unimol-dir /unimol/unimol \ + --dict /unimol/example_data/molecule/dict.txt \ + --out molae_chirality.npz \ + --batch-size 256 --device cuda:0 --num-workers 8 --verify + +# GemNet-Q +python baselines/gemnet/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --gemnet-repo \ + --out gemnet_chirality.npz \ + --batch-size 8 --device cuda --checkpoint-every 8000 --verify + +# FMG +python baselines/fmg/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --fmg-repo \ + --checkpoint \ + --out chirality_fmg.npz \ + --batch-size 32 --device cuda:0 --verify + +# MolSpectra (architecture only; the checkpoint is supplied by the caller) +python baselines/molspectra/extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --repo \ + --checkpoint \ + --out molspectra_chirality.npz \ + --batch-size 128 --device cuda \ + --arch torchmdnet --hydrogens remove --pool add --verify +``` + +The output goes straight into the evaluator: + +```bash +python -m three_dbench evaluate chirality \ + --dataset-dir data/hf/chirality \ + --embeddings chirality_mace.npz --embedding-key arr_0 \ + --model-name mace --output-dir out/ \ + --distance euclidean --metric-version paper --unsup-kmax n-1 +``` + +## Agreement with the published embeddings + +**Which input.** The published embedding files were computed from the source RDKit molecules, +whose coordinates carry full float precision. The public `EscheWang/3dcs` dataset stores the +same geometries as V2000 MOL blocks, which hold four decimals, so a run that starts from the +dataset starts up to 5e-5 A away from the coordinates behind the published file. The first +table below is the comparison from the full-precision molecules, the second is what the +published dataset gives; [Input precision](#input-precision) is why they differ. + +### From the full-precision molecules, all 52,391 conformers + +Each script was run over all 52,391 conformers and the result compared with the published +file of that model. "Metrics" is the largest absolute difference over the six chirality +metrics (ES-AUC, NN@1-Acc, SCI, SCI_unsup, Hopkins, DBI) computed by `three_dbench evaluate +chirality --distance euclidean --metric-version paper` from the published file and from the +regenerated file. + +| Model | Bytes identical | Per-row cosine | max abs diff | mean abs diff | Metrics (max abs diff) | +|---|---|---|---|---|---| +| E3FP | yes, all 52,391 fingerprints | — (identical bits) | — | — | 0 | +| GemNet-Q | no | min 0.99923; 52,390 of 52,391 rows ≥ 0.9999996 | — | 3.3e-04 | 1.6e-04 | +| Uni-Mol | no | min 0.9999999995 | 1.18e-04 | 1.01e-06 | 1.42e-05 | +| Mol-AE | no | min 0.9999997616 | 7.77e-04 | 5.72e-07 | 8.3e-05 | +| MolSpectra | no | mean −0.18 | — | — | 5.5e-02 | +| MACE | no | min 0.9999997634 | 4.90e-03 | 2.42e-08 | 8.9e-07 | +| FMG | no | min 0.99999970 | 6.00e-04 | 6.60e-06 | 6.6e-05 | + +Notes on individual rows: + +- **E3FP** is computed on the CPU from integer bit operations and is reproducible exactly: + `baselines/e3fp/extract_chirality.py` run over the full-precision RDKit molecules reproduces + all 52,391 fingerprints of the published file bit for bit (`identical_fraction 1`, + `tanimoto_min 1`; e3fp 1.2.7, rdkit 2026.03.6, 214 s with `--jobs 24`). The pickle itself is + not byte-identical, because the container is rewritten. Evaluating the regenerated file gives + ES-AUC 0.485935, NN@1-Acc 0.177939, SCI −0.012543 and SCI_unsup 0.033825, equal at six + decimals to the reference values in `reproduce/table2_chirality/expected.csv` (Hopkins is not + defined for fingerprints). Starting from the four-decimal MOL blocks of the Hugging Face + dataset instead, 2,846 of 3,000 sampled fingerprints match. +- **GemNet-Q**: the one row below cosine 0.9999996 (row 49740) is a structure of seven + disconnected fragments including a lone `[H]` and several radical carbons. +- **MACE**: the largest absolute difference comes from two rows of norm 1,743 and 15,228, + where the float32 spacing is already O(1e-2); no row has a cosine below 1 − 1e-6. +- **MolSpectra**: the script computes the documented quantity — the per-atom scalar + representation of the MolSpectra equivariant Transformer summed over the atoms of the + molecule — but the MolSpectra authors publish no checkpoint, so it has to be supplied by + the caller. The row above used the public `denoised-pcqm4mv2.ckpt` that MolSpectra + initialises from; with that checkpoint the output is a MolSpectra-architecture embedding of + the same shape, not a copy of the published file. The published + `chirality/molspectra/sampled_mol_feature.npz` is the artifact the paper's MolSpectra + values were computed from. Its entry in the table above is the largest difference over the + five metrics of `reproduce/table2_chirality/expected.csv`; DBI was not recorded for that run. +- **FMG**: the `smiles` side array of the output is metadata written by the RDKit build in + use; 108 of 52,391 entries are written differently by rdkit 2024.9.6 than in the published + file (`[O][Na]` against `O[Na]`), and all 108 re-canonicalise to the same molecule. + +### From the published `mol_blocks`, first 2,000 conformers + +Each script over the first 2,000 conformers read from the published `mol_blocks` +(`--dataset hf:EscheWang/3dcs:chirality --limit 2000 --verify`, or the `hfdisk:` form of a +local copy of the same config), compared with the first 2,000 rows of the published file of +that model — what `--verify` prints for a run an outside reader can reproduce from the +released dataset. One A100-80GB, each model in the environment its `ENVIRONMENT.md` builds. + +| Model | Per-row cosine | max abs diff | mean abs diff | +|---|---|---|---| +| E3FP | 1,925 of 2,000 fingerprints bit-identical; Tanimoto mean 0.9946, min 0.511 | — | — | +| GemNet-Q | mean 0.99999989, min 0.99999966; 0 rows < 1 − 1e−6 | 4.98e-03 | 3.35e-04 | +| Uni-Mol | mean 0.9999965, min 0.9945; 32 rows < 1 − 1e−6 | 3.85e-01 | 2.44e-04 | +| Mol-AE | mean 0.9999948, min 0.9921; 56 rows < 1 − 1e−6 | 4.36e-01 | 3.16e-04 | +| MolSpectra | mean −0.184 (a different checkpoint, see the note above) | 3.76e+02 | 1.95e+01 | +| MACE | mean 0.9999999995, min 0.9999999906; 0 rows < 1 − 1e−6 | 1.57e-03 | 1.33e-06 | +| FMG | mean 0.9999861, min 0.9946; 9 rows < 0.999 | 7.02e-01 | 3.10e-04 | + +Over those 2,000 conformers (163 molecules), the chirality metrics computed from the +regenerated file differ from the ones computed from the same rows of the published file by at +most 1.9e-03 (ES-AUC), 2.4e-03 (NN@1-Acc), 1.8e-02 (DBI), 1.9e-04 (Hopkins) and 2.9e-03 +(SCI_unsup), taking the largest value over MACE, Uni-Mol, Mol-AE, GemNet-Q and FMG. A run from +a pickle of the full-precision molecules gives the first table instead. + +None of the neural files are byte-identical, and none are expected to be: the forward passes +run in float32 on the GPU, where the reduction order depends on the batch size, the library +build and the device. Two runs of the same script on the same machine measure the size of +that effect — for FMG, two runs with identical settings differ by up to 6e-4 per element, the +same size as the difference from the published file; for MACE, batch size 1 against 16 moves +the output by at most 1.4e-7 on a 128-conformer sample; for GemNet, TF32 on against off moves +it by 2.0e-3. + +## Input precision + +The `mol_blocks` of the Hugging Face dataset are V2000 MDL molblocks, which store coordinates +with four decimals (at most 5e-5 A from the values in the source RDKit molecules the published +files were computed from). Models that build a neighbour graph with a hard cutoff, that encode +pair distances, or that align a molecule onto its principal axes, can react to that rounding on +a small fraction of conformers. Measured per model, running from the MOL blocks against running +from the full-precision molecules: + +| Model | rows the rounding moves | reference | +|---|---|---| +| Mol-AE | 2.8 % of rows below cosine 1 − 1e−6, lowest 0.9921 | [molae/ENVIRONMENT.md](molae/ENVIRONMENT.md#input-precision) | +| MolSpectra | 1.2 % of rows below cosine 0.9999 | [molspectra/ENVIRONMENT.md](molspectra/ENVIRONMENT.md#input-precision) | +| FMG | 0.4 % of rows (PCA axis swaps), lowest cosine 0.9931 | [fmg/ENVIRONMENT.md](fmg/ENVIRONMENT.md#input-precision) | +| Uni-Mol | 0.1 % of rows below cosine 0.9999, lowest 0.9945 | [unimol/ENVIRONMENT.md](unimol/ENVIRONMENT.md#input-precision) | +| E3FP | 2,846 of 3,000 sampled fingerprints bit-identical | [e3fp/ENVIRONMENT.md](e3fp/ENVIRONMENT.md#input-precision) | +| GemNet-Q | no row below cosine 1 − 1e−6; max abs diff 5.1e-04 | [gemnet/ENVIRONMENT.md](gemnet/ENVIRONMENT.md#input-precision) | +| MACE | max abs diff 3.8e-05, per-row cosine at least 0.9999998 | [mace/ENVIRONMENT.md](mace/ENVIRONMENT.md#input-precision) | + +ES-AUC, NN@1-Acc, SCI, SCI_unsup and Hopkins then move by less than 5.3e-4 in every case; DBI, a +ratio that is heavy-tailed over molecules, moves more (up to 4.4e-2 for FMG, where a single +molecule of 3,903 accounts for most of the shift). The source molecules are not part of the +release; a run from the dataset is the reproducible path, and the numbers above are its cost. + +## Other tracks + +The chirality set is what these scripts cover. For the rotation track, the GemNet script also +reads the per-shard LMDB inputs and writes one file per shard in the layout the +`--layout by-shard` reader expects; see `gemnet/ENVIRONMENT.md`. The trajectory (rMD17) +embeddings are published in `EscheWang/3dcs-embeddings` and are not regenerated by anything +here. diff --git a/baselines/common.py b/baselines/common.py new file mode 100644 index 0000000..f4c1eba --- /dev/null +++ b/baselines/common.py @@ -0,0 +1,706 @@ +"""Shared helpers for the baseline embedding-extraction scripts in ``baselines/``. + +Every per-model script under ``baselines//`` runs in its own model environment and +depends only on that model's own stack. This module holds what they have in common: + +* :func:`parse_dataset_spec` / :func:`load_conformers` / :func:`iter_conformers` -- the one + ``--dataset`` syntax that every script accepts, read into RDKit molecules in benchmark + row order; +* :func:`write_npz` / :func:`sha256_file` -- write the output array under a given key and + report its checksum; +* :func:`compare_vectors`, :func:`compare_fingerprints`, :func:`verify` -- compare a file + that was just written against a reference file, usually the published embedding of the + same model in ``EscheWang/3dcs-embeddings``, over all rows or over a subset; +* :func:`parse_row_selection` -- how the rows of a partial run line up with the rows of + the reference file; +* :func:`print_versions` -- print the versions of the libraries that were used. + +Nothing here imports torch or any model package, so it can be used from every one of the +model environments. +""" + +from __future__ import annotations + +import hashlib +import pickle +import sys +from collections.abc import Iterable, Iterator, Sequence +from pathlib import Path +from typing import Any + +import numpy as np + +EMBEDDINGS_REPO_ID = "EscheWang/3dcs-embeddings" +DATASET_REPO_ID = "EscheWang/3dcs" + +#: Published chirality embedding of each baseline: ``model -> (path in the repo, array key)``. +#: Mirrors ``three_dbench.embeddings.published.PUBLISHED_EMBEDDINGS`` for the chirality task, +#: so that ``--verify`` works in a model environment where ``three_dbench`` is not installed. +PUBLISHED_CHIRALITY: dict[str, tuple[str, str]] = { + "e3fp": ("chirality/e3fp/sampled_chi.pkl", "e3fp"), + "gemnet": ("chirality/gemnet/sampled_feature.npz", "gemnet"), + "molae": ("chirality/molae/1.npz", "arr_0"), + "molspectra": ("chirality/molspectra/sampled_mol_feature.npz", "arr_0"), + "unimol": ("chirality/unimol/1.npz", "arr_0"), + "fmg": ("chirality/fmg/chirality_bench_conformers_noised_only_aslist_embed.npz", "embeddings"), + "mace": ("chirality/mace/chirality.npz", "arr_0"), +} + +#: Number of conformers in the chirality set. +N_CHIRALITY_CONFORMERS = 52391 + + +# --------------------------------------------------------------------------- # +# files +# --------------------------------------------------------------------------- # +def sha256_file(path: str | Path, chunk_size: int = 1 << 22) -> str: + """SHA-256 of a file, streamed.""" + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_npz(path: str | Path, array: np.ndarray, key: str = "arr_0", compress: bool = False) -> str: + """Write ``array`` to ``path`` under ``key`` and return the SHA-256 of the file.""" + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + writer = np.savez_compressed if compress else np.savez + writer(path, **{key: array}) + return sha256_file(path) + + +def load_embedding(path: str | Path, key: str | None = None) -> Any: + """Load an embedding container written by one of these scripts or published. + + Returns a NumPy array for ``.npz`` / ``.npy``, and whatever the pickle holds for + ``.pkl`` (the E3FP file is a dict of lists of RDKit bit vectors). + """ + path = Path(path) + suffix = path.suffix.lower() + if suffix == ".npz": + with np.load(path, allow_pickle=False) as data: + if key is not None: + if key not in data.files: + raise KeyError(f"{path} has keys {data.files}, not {key!r}") + return np.asarray(data[key]) + numeric = [k for k in data.files if np.issubdtype(np.asarray(data[k]).dtype, np.number)] + if len(numeric) != 1: + raise KeyError(f"{path} holds {data.files}; pass an explicit key") + return np.asarray(data[numeric[0]]) + if suffix == ".npy": + return np.load(path) + if suffix == ".pkl": + with open(path, "rb") as handle: + payload = pickle.load(handle) + if isinstance(payload, dict): + if key is None: + if len(payload) != 1: + raise KeyError(f"{path} holds {sorted(payload)}; pass an explicit key") + return next(iter(payload.values())) + return payload[key] + return payload + raise ValueError(f"unsupported embedding file type: {path.suffix}") + + +# --------------------------------------------------------------------------- # +# which rows of the reference a partial run covers +# --------------------------------------------------------------------------- # +#: ``--verify-rows`` help text, identical in every script. +ROW_SELECTION_HELP = ( + "which rows of the --verify reference the output covers, for a run that produced only " + "part of the file: 'prefix' (the default: output row i is reference row i), 'full' " + "(require the two files to have the same number of rows), ':' for a slice " + "of the reference (e.g. '2000:4000', open ends allowed), '+' for the rows from " + " on, or '@' for a .npy / text file of 0-based reference row indices" +) + + +def parse_row_selection( + spec: str | None, + *, + produced_rows: int, + reference_rows: int, +) -> tuple[np.ndarray | None, str]: + """Resolve ``--verify-rows`` into indices into the reference file. + + Returns ``(indices, mode)``. ``indices`` is ``None`` when every row of the reference is + compared, i.e. the two files line up one to one; ``mode`` names the resolution + (``"full"``, ``"prefix"``, ``"slice"`` or ``"file"``) for the report. + """ + if spec in (None, "", "auto"): + spec = "full" if produced_rows == reference_rows else "prefix" + spec = str(spec) + + if spec == "full": + if produced_rows != reference_rows: + raise ValueError( + f"--verify-rows full: the output has {produced_rows} rows and the reference " + f"{reference_rows}; pass 'prefix', a ':' slice or '@'" + ) + return None, "full" + + if spec == "prefix": + if produced_rows > reference_rows: + raise ValueError( + f"--verify-rows prefix: the output has {produced_rows} rows, more than the " + f"{reference_rows} of the reference" + ) + if produced_rows == reference_rows: + return None, "full" + return np.arange(produced_rows, dtype=np.int64), "prefix" + + if spec.startswith("@"): + path = Path(spec[1:]) + if path.suffix.lower() == ".npy": + indices = np.asarray(np.load(path), dtype=np.int64).reshape(-1) + else: + text = path.read_text(encoding="utf-8").replace(",", " ").split() + indices = np.asarray([int(value) for value in text], dtype=np.int64) + mode = "file" + elif ":" in spec or spec.endswith("+"): + if spec.endswith("+"): + start_text, stop_text = spec[:-1], "" + else: + start_text, _, stop_text = spec.partition(":") + start = int(start_text) if start_text.strip() else 0 + stop = int(stop_text) if stop_text.strip() else start + produced_rows + indices = np.arange(start, stop, dtype=np.int64) + mode = "slice" + else: + raise ValueError( + f"--verify-rows: expected 'full', 'prefix', ':', '+' or '@', got {spec!r}" + ) + + if indices.size != produced_rows: + raise ValueError( + f"--verify-rows {spec}: selects {indices.size} reference rows but the output has {produced_rows}" + ) + if indices.size and (indices.min() < 0 or indices.max() >= reference_rows): + raise ValueError( + f"--verify-rows {spec}: row indices {int(indices.min())}..{int(indices.max())} fall outside " + f"the {reference_rows} rows of the reference" + ) + if mode == "slice" and indices.size and int(indices[0]) == 0 and indices.size == reference_rows: + return None, "full" + return indices, mode + + +def _take_rows(reference: Any, indices: np.ndarray | None) -> Any: + if indices is None: + return reference + if isinstance(reference, np.ndarray): + return reference[indices] + return [reference[int(i)] for i in indices] + + +# --------------------------------------------------------------------------- # +# comparison +# --------------------------------------------------------------------------- # +def compare_vectors( + produced: np.ndarray, + reference: np.ndarray, + *, + rows: np.ndarray | None = None, + row_mode: str | None = None, +) -> dict[str, Any]: + """Elementwise and per-row agreement between two embedding matrices. + + Returns a dict with the shapes, the largest and mean absolute difference, whether the + arrays are equal, the per-row cosine similarity statistics and the number of rows + outside a few thresholds. Rows of zero norm are excluded from the cosine statistics. + + ``rows`` compares row ``i`` of ``produced`` with row ``rows[i]`` of ``reference``, which + is what a run over part of the conformers needs: the statistics are then computed over + those rows only, and ``compared_rows`` / ``row_selection`` say so. + """ + produced = np.asarray(produced) + reference = np.asarray(reference) + out: dict[str, Any] = { + "produced_shape": tuple(produced.shape), + "reference_shape": tuple(reference.shape), + "shape_match": produced.shape == reference.shape, + } + if rows is not None: + rows = np.asarray(rows, dtype=np.int64) + out["row_selection"] = row_mode or "rows" + out["compared_rows"] = int(rows.size) + out["reference_rows_used"] = ( + f"{int(rows[0])}..{int(rows[-1])}" + if rows.size and np.array_equal(rows, np.arange(rows[0], rows[0] + rows.size)) + else "listed" + ) + reference = reference[rows] + else: + out["row_selection"] = row_mode or "full" + out["compared_rows"] = int(produced.shape[0]) if produced.ndim else 0 + out["compared_shape_match"] = produced.shape == reference.shape + if not out["compared_shape_match"]: + return out + a = produced.astype(np.float64, copy=False) + b = reference.astype(np.float64, copy=False) + diff = np.abs(a - b) + out["array_equal"] = bool(np.array_equal(produced, reference)) + out["max_abs_diff"] = float(diff.max()) if diff.size else 0.0 + out["mean_abs_diff"] = float(diff.mean()) if diff.size else 0.0 + out["allclose_1e-5"] = bool(np.allclose(a, b, atol=1e-5, rtol=1e-4)) + out["allclose_1e-4"] = bool(np.allclose(a, b, atol=1e-4, rtol=1e-3)) + out["allclose_1e-3"] = bool(np.allclose(a, b, atol=1e-3, rtol=1e-3)) + per_row_close = np.isclose(a, b, atol=1e-5, rtol=1e-4).all(axis=1) if a.ndim == 2 else np.array([]) + out["rows_allclose_1e-5"] = int(per_row_close.sum()) + na = np.linalg.norm(a, axis=1) + nb = np.linalg.norm(b, axis=1) + ok = (na > 0) & (nb > 0) + cos = np.full(a.shape[0], np.nan) + cos[ok] = np.einsum("ij,ij->i", a[ok], b[ok]) / (na[ok] * nb[ok]) + out["zero_norm_rows"] = int((~ok).sum()) + if ok.any(): + out["cosine_mean"] = float(cos[ok].mean()) + out["cosine_min"] = float(cos[ok].min()) + out["rows_cosine_below_1-1e-6"] = int((cos[ok] < 1 - 1e-6).sum()) + out["rows_cosine_below_0.999"] = int((cos[ok] < 0.999).sum()) + present = np.flatnonzero(ok) + order = present[np.argsort(cos[present])] + out["worst_row"] = int(order[0]) + out["worst_rows"] = [int(i) for i in order[:10]] + return out + + +def compare_fingerprints( + produced: Sequence[Any], + reference: Sequence[Any], + *, + rows: np.ndarray | None = None, + row_mode: str | None = None, +) -> dict[str, Any]: + """Agreement between two lists of RDKit bit vectors (E3FP / Morgan fingerprints). + + ``rows`` has the same meaning as in :func:`compare_vectors`: fingerprint ``i`` of + ``produced`` is compared with fingerprint ``rows[i]`` of ``reference``. + """ + out: dict[str, Any] = { + "produced_len": len(produced), + "reference_len": len(reference), + "shape_match": len(produced) == len(reference), + } + if rows is not None: + rows = np.asarray(rows, dtype=np.int64) + out["row_selection"] = row_mode or "rows" + out["compared_rows"] = int(rows.size) + reference = _take_rows(list(reference), rows) + else: + out["row_selection"] = row_mode or "full" + out["compared_rows"] = len(produced) + out["compared_shape_match"] = len(produced) == len(reference) + if not out["compared_shape_match"]: + return out + from rdkit import DataStructs + + identical = 0 + tanimoto = np.empty(len(produced), dtype=np.float64) + mismatched: list[int] = [] + for i, (p, r) in enumerate(zip(produced, reference)): + tanimoto[i] = DataStructs.TanimotoSimilarity(p, r) + if list(p.GetOnBits()) == list(r.GetOnBits()): + identical += 1 + elif len(mismatched) < 20: + mismatched.append(i) + out["identical"] = identical + out["identical_fraction"] = identical / len(produced) if produced else 0.0 + out["tanimoto_mean"] = float(tanimoto.mean()) if len(tanimoto) else 0.0 + out["tanimoto_min"] = float(tanimoto.min()) if len(tanimoto) else 0.0 + out["first_mismatched_rows"] = mismatched + return out + + +def format_report(result: dict[str, Any]) -> str: + """Render the dict of :func:`compare_vectors` / :func:`compare_fingerprints`.""" + width = max((len(k) for k in result), default=0) + lines = [] + for key, value in result.items(): + if isinstance(value, float): + text = f"{value:.10g}" + else: + text = str(value) + lines.append(f" {key:<{width}} {text}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# the published reference files +# --------------------------------------------------------------------------- # +def resolve_reference( + spec: str, + *, + model: str | None = None, + cache_dir: str | Path | None = None, + revision: str | None = None, +) -> tuple[Path, str | None]: + """Resolve ``--verify`` into a local file and the array key to read from it. + + ``spec`` is one of + + * ``"published"`` -- the published chirality embedding of ``model``, downloaded from + ``EscheWang/3dcs-embeddings`` (or taken from the cache of ``huggingface_hub``); + * ``"hub:"`` -- any file of that dataset repository, e.g. + ``hub:chirality/mace/chirality.npz``; + * a local path, in which case the key is inferred by :func:`load_embedding`. + """ + if spec == "published": + if model is None or model not in PUBLISHED_CHIRALITY: + raise ValueError(f"'published' needs a known model, got {model!r}") + path, key = PUBLISHED_CHIRALITY[model] + return _download_published(path, cache_dir=cache_dir, revision=revision), key + if spec.startswith("hub:"): + path = spec[len("hub:") :] + key = next((k for p, k in PUBLISHED_CHIRALITY.values() if p == path), None) + return _download_published(path, cache_dir=cache_dir, revision=revision), key + return Path(spec), None + + +def _download_published(path: str, *, cache_dir: str | Path | None, revision: str | None) -> Path: + from huggingface_hub import hf_hub_download + + local = hf_hub_download( + repo_id=EMBEDDINGS_REPO_ID, + filename=path, + repo_type="dataset", + revision=revision, + cache_dir=str(cache_dir) if cache_dir else None, + ) + return Path(local) + + +def verify( + produced_path: str | Path, + *, + model: str | None = None, + reference: str = "published", + produced_key: str | None = None, + reference_key: str | None = None, + rows: str | None = None, + cache_dir: str | Path | None = None, + revision: str | None = None, + stream=None, +) -> dict[str, Any]: + """Compare a file written by one of these scripts against a reference file. + + Prints the SHA-256 of both files and the agreement statistics, and returns them. The + caller decides what to do with the result; the scripts print it and exit 0 either way, + because agreement to the last bit is not expected across machines and library versions. + + ``rows`` is the ``--verify-rows`` selection (see :func:`parse_row_selection`). A file + written by a run over part of the conformers -- ``--limit``, ``--start`` -- holds fewer + rows than the published reference; with the default selection its rows are compared with + the first rows of the reference and every statistic is computed over those rows. + """ + stream = stream or sys.stdout + produced_path = Path(produced_path) + ref_path, ref_key = resolve_reference(reference, model=model, cache_dir=cache_dir, revision=revision) + if reference_key is not None: + ref_key = reference_key + + result: dict[str, Any] = { + "produced": str(produced_path), + "reference": str(ref_path), + "produced_sha256": sha256_file(produced_path), + "reference_sha256": sha256_file(ref_path), + } + result["sha256_identical"] = result["produced_sha256"] == result["reference_sha256"] + + left = load_embedding(produced_path, produced_key) + right = load_embedding(ref_path, ref_key) + as_fingerprints = isinstance(right, (list, tuple)) or isinstance(left, (list, tuple)) + indices, mode = parse_row_selection(rows, produced_rows=len(left), reference_rows=len(right)) + if mode != "full": + print( + f"[verify] the output holds {len(left)} of the {len(right)} rows of the reference; " + f"comparing row selection '{mode}'", + file=stream, + ) + if as_fingerprints: + result.update(compare_fingerprints(list(left), list(right), rows=indices, row_mode=mode)) + else: + result.update(compare_vectors(left, right, rows=indices, row_mode=mode)) + + print("[verify] comparison with the reference file", file=stream) + print(format_report(result), file=stream) + return result + + +# --------------------------------------------------------------------------- # +# inputs +# --------------------------------------------------------------------------- # +#: ``--dataset`` help text, identical in every script. +DATASET_SPEC_HELP = ( + "conformer source, read in benchmark row order. One of: " + "'hf:EscheWang/3dcs:chirality' (a Hub dataset and its config), " + "'hfdisk:' or a plain path to a 'save_to_disk' directory of that config, " + "a bare Hub dataset id (config 'chirality'), " + "a pickle holding a list of RDKit molecules or a dict of such lists, or " + "'lmdb:' for a rotation shard" +) + +#: The kinds :func:`parse_dataset_spec` returns. +DATASET_KINDS = ("hub", "disk", "pickle", "lmdb") + +_PICKLE_SUFFIXES = {".pkl", ".pickle", ".pkl.gz"} +_LMDB_SUFFIXES = {".lmdb", ".mdb"} + + +def parse_dataset_spec(spec: str, *, hf_config: str = "chirality") -> tuple[str, str, str | None]: + """Resolve a ``--dataset`` value into ``(kind, target, config)``. + + ``kind`` is one of :data:`DATASET_KINDS`. This is pure string and filesystem + inspection: nothing is downloaded or opened here. + + ======================================== ========= ========================== + spec kind target + ======================================== ========= ========================== + ``hf:EscheWang/3dcs:chirality`` ``hub`` ``EscheWang/3dcs`` + ``hf:EscheWang/3dcs`` ``hub`` ``EscheWang/3dcs`` + ``EscheWang/3dcs`` ``hub`` ``EscheWang/3dcs`` + ``hfdisk:data/hf/chirality`` ``disk`` ``data/hf/chirality`` + ``data/hf/chirality`` (an existing dir) ``disk`` ``data/hf/chirality`` + ``conformers.pkl`` (an existing file) ``pickle`` ``conformers.pkl`` + ``lmdb:rotation_0.lmdb`` ``lmdb`` ``rotation_0.lmdb`` + ======================================== ========= ========================== + """ + if not isinstance(spec, str) or not spec.strip(): + raise ValueError(f"--dataset: expected a non-empty string. {DATASET_SPEC_HELP}") + spec = spec.strip() + + if spec.startswith("lmdb:"): + return "lmdb", spec[len("lmdb:") :], None + if spec.startswith("hfdisk:"): + return "disk", spec[len("hfdisk:") :], None + if spec.startswith("hf:"): + rest = spec[len("hf:") :] + repo, sep, config = rest.partition(":") + if not repo: + raise ValueError(f"--dataset {spec!r}: 'hf:' needs a repository id, e.g. hf:EscheWang/3dcs:chirality") + return "hub", repo, (config or hf_config) if sep else hf_config + + path = Path(spec) + if path.is_dir(): + return "disk", spec, None + if path.is_file(): + return ("lmdb", spec, None) if path.suffix.lower() in _LMDB_SUFFIXES else ("pickle", spec, None) + + # Not on disk: a Hub id is "/", anything else is a path that does not exist. + parts = spec.split("/") + if len(parts) == 2 and all(parts) and path.suffix.lower() not in _PICKLE_SUFFIXES: + return "hub", spec, hf_config + raise FileNotFoundError(f"--dataset {spec!r}: no such file or directory. {DATASET_SPEC_HELP}") + + +def load_conformers( + spec: str, + *, + hf_config: str = "chirality", + hf_split: str = "train", + revision: str | None = None, + sanitize: bool = True, + sanitize_fallback: bool = True, + remove_hs: bool = False, + limit: int | None = None, + start: int = 0, + stream=None, +) -> list: + """Return the conformers as RDKit molecules, in benchmark row order. + + ``spec`` takes every form of :func:`parse_dataset_spec`, which is the ``--dataset`` + syntax of every script in ``baselines/``. + + Rows of the Hugging Face config carry ``mol_blocks`` -- one MOL block per conformer of + that stereoisomer -- and the ``offset`` of the first of them, so reading ``mol_blocks`` + in ascending ``offset`` gives the row order of the published embedding files. MOL blocks + store coordinates with four decimals, so geometries read that way differ from the source + pickle by up to 5e-5 A. + + ``sanitize`` is passed to ``Chem.MolFromMolBlock``. With ``sanitize_fallback`` a block + that RDKit refuses to sanitise is re-read unsanitised rather than dropped, so the row + order never shifts; the number of such rows is printed. Set it to ``False`` where + sanitisation changes the result (E3FP reads bond orders and stereochemistry). + """ + return list( + iter_conformers( + spec, + hf_config=hf_config, + hf_split=hf_split, + revision=revision, + sanitize=sanitize, + sanitize_fallback=sanitize_fallback, + remove_hs=remove_hs, + limit=limit, + start=start, + stream=stream, + ) + ) + + +def iter_conformers( + spec: str, + *, + hf_config: str = "chirality", + hf_split: str = "train", + revision: str | None = None, + sanitize: bool = True, + sanitize_fallback: bool = True, + remove_hs: bool = False, + limit: int | None = None, + start: int = 0, + stream=None, +) -> Iterator[Any]: + """:func:`load_conformers` as a generator, for a script that does not want the list.""" + stream = stream or sys.stdout + kind, target, config = parse_dataset_spec(spec, hf_config=hf_config) + if kind == "pickle": + source: Iterable[Any] = _iter_pickle_molecules(target) + elif kind == "lmdb": + source = _iter_lmdb_molecules(target) + else: + source = _iter_hf_molecules( + kind, + target, + config=config or hf_config, + hf_split=hf_split, + revision=revision, + sanitize=sanitize, + sanitize_fallback=sanitize_fallback, + remove_hs=remove_hs, + stream=stream, + ) + + for index, mol in enumerate(source): + if index < start: + continue + if limit is not None and index - start >= limit: + return + yield mol + + +def _open_hf_dataset(kind: str, target: str, *, config: str, hf_split: str, revision: str | None): + if kind == "disk": + from datasets import load_from_disk + + return load_from_disk(target) + from datasets import load_dataset + + return load_dataset(target, name=config, split=hf_split, revision=revision) + + +def _iter_hf_molecules( + kind: str, + target: str, + *, + config: str, + hf_split: str, + revision: str | None, + sanitize: bool, + sanitize_fallback: bool, + remove_hs: bool, + stream, +) -> Iterator[Any]: + from rdkit import Chem + + ds = _open_hf_dataset(kind, target, config=config, hf_split=hf_split, revision=revision) + if "mol_blocks" not in ds.column_names: + raise KeyError( + f"{target}: expected a 'mol_blocks' column (the 'chirality' config of EscheWang/3dcs " + f"stores one list of MDL MOL blocks per stereoisomer), got {ds.column_names}" + ) + order: Sequence[int] + if "offset" in ds.column_names: + offsets = np.asarray(ds["offset"], dtype=np.int64) # one column read, not len(ds) row reads + order = np.argsort(offsets, kind="stable") + else: + offsets = None + order = np.arange(len(ds), dtype=np.int64) + blocks_column = ds["mol_blocks"] + + seen = 0 + first_offset: int | None = None + unsanitised = 0 + for row_idx in order: + row_idx = int(row_idx) + if offsets is not None: + offset = int(offsets[row_idx]) + if first_offset is None: + first_offset = offset + if offset: + print(f"[data] the rows of {target} start at conformer offset {offset}", file=stream) + if offset != first_offset + seen: + raise ValueError( + f"{target}: row {row_idx} has offset {offset}, expected {first_offset + seen}; " + "the rows do not form one contiguous block of the published row order" + ) + for block in blocks_column[row_idx]: + mol = Chem.MolFromMolBlock(block, removeHs=remove_hs, sanitize=sanitize) + if mol is None and sanitize and sanitize_fallback: + mol = Chem.MolFromMolBlock(block, removeHs=remove_hs, sanitize=False) + if mol is not None: + unsanitised += 1 + if mol is None: + raise ValueError(f"RDKit could not parse the MOL block of conformer {seen} (dataset row {row_idx})") + seen += 1 + yield mol + if unsanitised: + print(f"[data] {unsanitised} of {seen} conformers were read without sanitisation", file=stream) + + +def _iter_pickle_molecules(path: str) -> Iterator[Any]: + with open(path, "rb") as handle: + payload = pickle.load(handle) + yield from _flatten_molecules(payload) + + +def _iter_lmdb_molecules(path: str) -> Iterator[Any]: + """Rotation shards: each key holds a list of ``(Mol, energy, torsion_deg)``. + + Row order is the LMDB cursor order of the keys, then the position inside each list, + which is the row order of the published ``rotation//rotation_conformers_.npz``. + """ + import lmdb + + env = lmdb.open(path, subdir=False, readonly=True, lock=False, readahead=False, meminit=False) + try: + with env.begin() as txn: + for _key, value in txn.cursor(): + for item in pickle.loads(value): + yield item[0] if isinstance(item, tuple) else item + finally: + env.close() + + +def _flatten_molecules(payload: Any) -> list: + if isinstance(payload, dict): + out: list = [] + for value in payload.values(): + out.extend(value if isinstance(value, (list, tuple)) else [value]) + return out + if isinstance(payload, (list, tuple)): + return list(payload) + raise TypeError(f"expected a list or dict of RDKit molecules, got {type(payload).__name__}") + + +# --------------------------------------------------------------------------- # +# reporting +# --------------------------------------------------------------------------- # +def print_versions(modules: Sequence[str] = (), stream=None) -> dict[str, str]: + """Print the version of Python and of each importable module name in ``modules``.""" + import importlib + + stream = stream or sys.stdout + found = {"python": sys.version.split()[0], "numpy": np.__version__} + for name in modules: + try: + module = importlib.import_module(name) + except Exception as exc: # pragma: no cover - depends on the environment + found[name] = f"not importable ({exc.__class__.__name__})" + continue + found[name] = str(getattr(module, "__version__", "unknown")) + for name, version in found.items(): + print(f"[versions] {name:<14} {version}", file=stream) + return found diff --git a/baselines/e3fp/ENVIRONMENT.md b/baselines/e3fp/ENVIRONMENT.md new file mode 100644 index 0000000..bd90759 --- /dev/null +++ b/baselines/e3fp/ENVIRONMENT.md @@ -0,0 +1,72 @@ +# E3FP — environment + +`extract_chirality.py` needs only `e3fp` and RDKit. There are no model weights. + +## Upstream + +| item | source | version | +|---|---|---| +| E3FP | (LGPL-3.0) | `e3fp==1.2.7` on PyPI | + +## Install + +```bash +conda create -y -p ./env python=3.10 +./env/bin/pip install "e3fp==1.2.7" "rdkit==2024.9.6" "numpy<2" +# only for --dataset hf:... / hfdisk:... +./env/bin/pip install datasets +# only for --verify +./env/bin/pip install huggingface_hub +``` + +`e3fp` pulls in `sdaxen_python_utilities`, `smart_open` and `mmh3`. + +## Run + +```bash +python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --out sampled_chi.pkl --jobs 24 --verify +``` + +`--dataset` takes the input specification shared by every script in `baselines/` (see the +table in [`../README.md`](../README.md#running-one)): `hf:[:]`, `hfdisk:` or a plain +`save_to_disk` directory, a bare Hub dataset id, a pickle of RDKit molecules, or `lmdb:`. +`--limit`/`--start` run a slice of the conformers and `--verify-rows` says which rows of the +reference that slice covers. + +E3FP reads bond orders and stereochemistry, so molecules are only used sanitised: a MOL block +RDKit refuses to sanitise is an error here rather than a row read without sanitisation. +## Fingerprint parameters + +``` +bits=1024, level=5, radius_multiplier=1.5, stereo=True, +include_disconnected=True, rdkit_invariants=True, first=1, counts=False +``` + +Hydrogens are kept. These are the parameters of the published +`chirality/e3fp/sampled_chi.pkl`; the `e3fp` defaults (`bits=4096`, +`radius_multiplier=1.718`, `rdkit_invariants=False`) give different bits. + +## Agreement with the published file + +Run over the full-precision RDKit molecules with the parameters above, all 52,391 +fingerprints are bit-identical to the published `chirality/e3fp/sampled_chi.pkl` +(`--verify` reports `identical 52391`, `identical_fraction 1`, `tanimoto_min 1`), with +e3fp 1.2.7 and rdkit 2026.03.6. The pickle itself is not byte-identical, because the +container is rewritten. + +## Input precision + +The published fingerprint file and the agreement numbers in [`../README.md`](../README.md) were +computed from the source RDKit molecules, whose coordinates carry full float precision. The public +`EscheWang/3dcs` dataset stores those geometries as V2000 MOL blocks, which hold four decimals, so +a run from `--dataset hf:EscheWang/3dcs:chirality` starts up to 5e-5 A away from the coordinates +behind the published file. E3FP bins interatomic distances, so that rounding can move a shell +boundary: over 3,000 sampled conformers, fingerprints computed from the MOL blocks agree with the +published file for 2,846 of 3,000. Start from a pickle of the molecules when the bits have to match +exactly. + +## Cost + +CPU only. 52,391 conformers in 214 s with `--jobs 24` (~250 conformers/s). diff --git a/baselines/e3fp/extract_chirality.py b/baselines/e3fp/extract_chirality.py new file mode 100644 index 0000000..41c8a4a --- /dev/null +++ b/baselines/e3fp/extract_chirality.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python +"""Extract E3FP fingerprints for the 3DCS chirality benchmark. + +What this script computes +------------------------- +For every input conformer it computes one E3FP (Extended 3-Dimensional FingerPrint) +with the parameters below and converts it to an RDKit ``ExplicitBitVect``: + + bits=1024, level=5, radius_multiplier=1.5, stereo=True, + include_disconnected=True, rdkit_invariants=True, first=1, counts=False + +Hydrogens are kept: the conformer is passed to ``e3fp`` exactly as it is stored in the +dataset. These are not the ``e3fp`` defaults (which are ``bits=4096``, +``radius_multiplier=1.718``, ``rdkit_invariants=False``). + +The output is a pickle holding ``{"e3fp": [ExplicitBitVect, ...]}``, one entry per +conformer in benchmark row order -- the layout of the published +``chirality/e3fp/sampled_chi.pkl``, which the chirality evaluator reads with +``--embedding-key e3fp`` and scores with the Tanimoto distance. + +Third-party dependencies (not redistributed here) +------------------------------------------------- + * e3fp -- https://github.com/keiserlab/e3fp (LGPL-3.0). Pinned: 1.2.7 + * rdkit, numpy, smart_open, sdaxen_python_utilities (pulled in by e3fp) + +Example +------- + python extract_chirality.py \\ + --dataset hf:EscheWang/3dcs:chirality \\ + --out sampled_chi.pkl --jobs 8 --verify +""" + +from __future__ import annotations + +import argparse +import hashlib +import logging +import pickle +import sys +import time +from collections.abc import Sequence + +MODEL_NAME = "e3fp" +OUTPUT_KEY = "e3fp" + +#: The fingerprint parameters of the published file. +FPRINT_PARAMS = { + "bits": 1024, + "level": 5, + "radius_multiplier": 1.5, + "stereo": True, + "include_disconnected": True, + "rdkit_invariants": True, + "first": 1, + "counts": False, +} + +_PARAMS: dict = {} + + +def _common(): + """Load ``baselines/common.py`` (input and verification helpers) without touching sys.path.""" + import importlib.util + import os + + path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "common.py") + spec = importlib.util.spec_from_file_location("baselines_common", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_molecules(spec: str, *, limit: int | None = None, start: int = 0): + """Return the input conformers as RDKit molecules, in benchmark row order. + + ``spec`` takes the ``--dataset`` syntax shared by every script in ``baselines/`` + (see ``baselines/common.py``). E3FP reads bond orders and stereochemistry, so a MOL + block RDKit refuses to sanitise is an error rather than a row read unsanitised. + """ + return _common().load_conformers(spec, limit=limit, start=start, sanitize_fallback=False) + + +def fingerprint(mol): + """E3FP of conformer 0 of ``mol`` as an RDKit ``ExplicitBitVect``.""" + from e3fp.pipeline import fprints_from_mol + + if not mol.HasProp("_Name"): + mol.SetProp("_Name", "mol") # e3fp names the fingerprint after the molecule + return fprints_from_mol(mol, fprint_params=_PARAMS or FPRINT_PARAMS)[0].to_rdkit() + + +def _quiet_e3fp() -> None: + """e3fp logs one INFO line per molecule on the root logger; keep the run log readable.""" + logging.getLogger().setLevel(logging.WARNING) + + +def _init_worker(params: dict) -> None: + global _PARAMS + _PARAMS = params + _quiet_e3fp() + + +def run(mols, params: dict, jobs: int, log_every: int) -> list: + """Fingerprint every molecule, preserving the input order.""" + global _PARAMS + _PARAMS = params + n = len(mols) + started = time.time() + if jobs <= 1: + out = [] + for i, mol in enumerate(mols): + out.append(fingerprint(mol)) + if log_every and (i + 1) % log_every == 0: + print(f" {i + 1}/{n} {(i + 1) / (time.time() - started):.1f} conf/s", flush=True) + return out + + import multiprocessing as mp + + with mp.Pool(jobs, initializer=_init_worker, initargs=(params,)) as pool: + out = [] + for i, fp in enumerate(pool.imap(fingerprint, mols, chunksize=64)): + out.append(fp) + if log_every and (i + 1) % log_every == 0: + print(f" {i + 1}/{n} {(i + 1) / (time.time() - started):.1f} conf/s", flush=True) + return out + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + common = _common() + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dataset", required=True, help=common.DATASET_SPEC_HELP) + p.add_argument("--out", required=True, help="output .pkl") + p.add_argument("--key", default=OUTPUT_KEY, help="dict key written into the pickle (default: e3fp)") + p.add_argument("--bits", type=int, default=FPRINT_PARAMS["bits"]) + p.add_argument("--level", type=int, default=FPRINT_PARAMS["level"]) + p.add_argument("--radius-multiplier", type=float, default=FPRINT_PARAMS["radius_multiplier"]) + p.add_argument("--first", type=int, default=FPRINT_PARAMS["first"]) + p.add_argument( + "--no-stereo", + dest="stereo", + action="store_false", + help="drop the stereochemical atom invariants (the published file uses stereo=True)", + ) + p.add_argument("--no-include-disconnected", dest="include_disconnected", action="store_false") + p.add_argument("--no-rdkit-invariants", dest="rdkit_invariants", action="store_false") + p.add_argument("--counts", action="store_true", help="count fingerprint instead of a bit vector") + p.add_argument("--jobs", type=int, default=1, help="worker processes (default: 1)") + p.add_argument("--limit", type=int, default=None, help="only process the first N conformers") + p.add_argument("--start", type=int, default=0, help="skip the first N conformers") + p.add_argument("--log-every", type=int, default=5000) + p.add_argument( + "--verify", + nargs="?", + const="published", + default=None, + metavar="REFERENCE", + help="after writing, compare the output with a reference file: 'published' " + "(the published E3FP chirality fingerprints), hub:, or a local path", + ) + p.add_argument( + "--verify-key", default=None, help="dict key to read from the --verify reference (default: its published key)" + ) + p.add_argument("--verify-rows", default=None, metavar="ROWS", help=common.ROW_SELECTION_HELP) + return p.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + common = _common() + common.print_versions(("e3fp", "rdkit")) + _quiet_e3fp() + + params = { + "bits": args.bits, + "level": args.level, + "radius_multiplier": args.radius_multiplier, + "stereo": args.stereo, + "include_disconnected": args.include_disconnected, + "rdkit_invariants": args.rdkit_invariants, + "first": args.first, + "counts": args.counts, + } + print(f"[params] {params}") + + mols = load_molecules(args.dataset, limit=args.limit, start=args.start) + print(f"[data] {len(mols)} conformers from {args.dataset}", flush=True) + + started = time.time() + fprints = run(mols, params, args.jobs, args.log_every) + print(f"[run] {len(fprints)} fingerprints in {time.time() - started:.1f}s") + + with open(args.out, "wb") as handle: + pickle.dump({args.key: fprints}, handle) + with open(args.out, "rb") as handle: + digest = hashlib.sha256(handle.read()).hexdigest() + print(f"[out] {args.out} key={args.key} n={len(fprints)} sha256={digest}") + + if args.verify: + common.verify( + args.out, + model=MODEL_NAME, + reference=args.verify, + produced_key=args.key, + reference_key=args.verify_key, + rows=args.verify_rows or (f"{args.start}+" if args.start else None), + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/baselines/e3fp/requirements.txt b/baselines/e3fp/requirements.txt new file mode 100644 index 0000000..4b95f57 --- /dev/null +++ b/baselines/e3fp/requirements.txt @@ -0,0 +1,14 @@ +# Environment used to compute the E3FP chirality fingerprints (python 3.10 or newer). +# +# conda create -y -p ./env python=3.10 +# ./env/bin/pip install -r requirements.txt +# +# e3fp pulls in sdaxen_python_utilities, smart_open and mmh3. +e3fp==1.2.7 +rdkit==2024.9.6 +numpy<2 +# only for the Hugging Face --dataset forms (hf:[:], hfdisk:, a +# save_to_disk directory or a bare Hub id) +# datasets +# only for --verify +# huggingface_hub diff --git a/baselines/fmg/ENVIRONMENT.md b/baselines/fmg/ENVIRONMENT.md new file mode 100644 index 0000000..459b7f3 --- /dev/null +++ b/baselines/fmg/ENVIRONMENT.md @@ -0,0 +1,111 @@ +# FMG — environment + +`extract_chirality.py` is a wrapper. It imports `utils.align`, +`utils.create_gaussian_batch_pdf_values` and +`denoising_diffusion_pytorch.classifier_free_guidance.Unet3D` from a clone of the upstream +FMG repository and loads a checkpoint obtained from the FMG authors. Neither the FMG source +nor the FMG weights are redistributed here. + +> Dumitrescu, Korpela, Heinonen, Verma, Iadarola, Marttinen, Garg. +> *E(3)-equivariant models cannot learn chirality: Field-based molecular generation.* +> ICLR 2025. + +## Upstream + +| item | source | pin | +|---|---|---| +| FMG | | commit `13a0a7cc331d136a4028eda80c8b55b0adfd58c3` (2025-04-22) | + +## Weights + +| field | value | +|---|---| +| file | `model-120qm9_3rd_run.pt` (FMG QM9 3D U-Net) | +| size | 1,246,683,351 bytes | +| sha256 | `f55ec38f2b6c20ad3a2e4e6287efb77af3901d46548449543bdbab33357d2afa` | +| source | the Google Drive folder linked from the upstream FMG README ("Pre-trained model weights") | +| Drive folder | | +| Drive file id | `1g-EJOtJpY0eRx2UlEmvvcBM0LIrKq0NP` | + +```bash +pip install gdown +# the whole folder (three QM9 runs, ~3.5 GB), or just the file this script uses: +gdown --folder https://drive.google.com/drive/folders/1XpOfCPRvPu22dSgbWgfGRF0Lul7ygdC7 -O fmg_weights +gdown 1g-EJOtJpY0eRx2UlEmvvcBM0LIrKq0NP -O model-120qm9_3rd_run.pt +sha256sum model-120qm9_3rd_run.pt +# f55ec38f2b6c20ad3a2e4e6287efb77af3901d46548449543bdbab33357d2afa +``` + +The download needs no Google account. The folder also holds `model-120qm9_1st_run.pt` and +`model-120qm9_2nd_run.pt`, which are different runs of the same architecture; the published +embeddings come from the third. + +The script reads the 226 tensors of the `ema` state dict whose names start with +`online_model.model.`. + +## Install + +```bash +git clone https://github.com/Dumitrescu-Alexandru/FMG.git +git -C FMG checkout 13a0a7cc331d136a4028eda80c8b55b0adfd58c3 + +conda create -y -p ./env python=3.10 +./env/bin/pip install "pip==24.0" "setuptools==67.8.0" wheel +./env/bin/pip install torch==2.0.0 torchvision==0.15.1 --index-url https://download.pytorch.org/whl/cu118 +./env/bin/pip install -r requirements.txt +``` + +Resolved versions used for the numbers in [`../README.md`](../README.md): python 3.10.21, +torch 2.0.0+cu118 (cuDNN 8700), numpy 1.26.4, rdkit 2024.9.6, einops 0.7.0, datasets 4.0.0, +tqdm 4.70.1 (`datasets` 4.0.0 requires `tqdm>=4.66.3`, which is the floor `requirements.txt` +carries). + +RDKit must be 2023 or newer to read the released conformer pickles (they carry RDKit pickle +version 16); the `rdkit-pypi==2022.9.5` of the upstream `requirements.txt` raises +`Bad pickle format: ENDMOL tag not found`. The remaining pins in `requirements.txt` are +transitive dependencies of `import utils` in the FMG checkout. + +## Run + +```bash +python extract_chirality.py \ + --fmg-repo ./FMG \ + --checkpoint ./model-120qm9_3rd_run.pt \ + --dataset EscheWang/3dcs \ + --out chirality_fmg.npz \ + --batch-size 32 --device cuda:0 --verify +``` + +`--dataset` takes the `chirality` config of `EscheWang/3dcs` (a Hub id or a `save_to_disk` +directory), or a pickle holding a list of RDKit molecules with one conformer each. Both +forms give rows in the order of the published embedding file; each row's `offset` is +checked against the running conformer count. + +The output `.npz` holds `embeddings` (float32, N x 128) and `smiles`. The `smiles` array is +metadata written by the RDKit build in use: 108 of the 52,391 entries are written +differently by rdkit 2024.9.6 than in the published file (all of them `[O][Na]` against +`O[Na]`), and all 108 re-canonicalise to the same molecule. + +## Determinism + +The FMG forward pass runs in float32 with TF32 convolutions enabled by default, so two runs +of the same code on the same machine differ by up to ~6e-4 per element. `--deterministic` +forces deterministic cuDNN kernels and disables TF32; it is slower and moves the output +further from the published file, which was computed with the library defaults. + +## Input precision + +The published embedding file and the agreement numbers in [`../README.md`](../README.md) were +computed from the source RDKit molecules, whose coordinates carry full float precision. The public +`EscheWang/3dcs` dataset stores those geometries as V2000 MOL blocks, which hold four decimals. FMG +aligns each conformer onto its PCA axes, and for 216 of the 52,391 conformers (0.41 %) that rounding +is enough to swap near-degenerate principal axes, which rotates the field (lowest per-row cosine +0.9931 against the run from the full-precision molecules). The remaining 99.6 % agree to the 1e-6 +level, and the chirality metrics move by at most 5.3e-4. Start from a pickle of the molecules for +the closest agreement. + +## Cost + +52,391 conformers on one A100-80GB at `--batch-size 32`: 89 s to read the molecules and +510 s for the forward pass (102.8 conformers/s). Peak GPU memory 1.79 GiB allocated, 9.93 +GiB reserved by the caching allocator. diff --git a/baselines/fmg/extract_chirality.py b/baselines/fmg/extract_chirality.py new file mode 100644 index 0000000..16f3c4c --- /dev/null +++ b/baselines/fmg/extract_chirality.py @@ -0,0 +1,523 @@ +#!/usr/bin/env python3 +"""Extract FMG embeddings for the 3DCS chirality set. + +The released ``chirality/fmg/...npz`` embedding file holds, for every conformer of the +3DCS chirality set, a 128-dimensional vector read out of the field-based molecular +generation (FMG) 3D U-Net of + + Dumitrescu et al., "E(3)-equivariant models cannot learn chirality: Field-based + molecular generation", ICLR 2025. https://github.com/Dumitrescu-Alexandru/FMG + +This script is the extraction wrapper only. The FMG model code and the FMG QM9 +checkpoint are third party and are NOT redistributed here; ``--fmg-repo`` must point at +your own clone of the upstream repository and ``--checkpoint`` at the QM9 checkpoint +obtained from the FMG authors. See the accompanying README for the exact revision and +checksums that were used. + +What the script computes, per conformer: + + 1. atom coordinates are mean-centred and rotated onto their PCA axes + (``utils.align`` of the FMG repo; a reflection is turned into a rotation by + flipping the third axis, so handedness is preserved); + 2. only the atoms of ``--atom-channels`` (default ``C,O,N,F``) are kept. Hydrogen is + dropped, as are any other elements; + 3. the molecule is rendered onto a ``--grid-size``^3 cubic grid of spacing + ``--resolution`` A, centred at the origin, as a sum of isotropic Gaussians of + variance ``--gaussian-std``: one channel per kept element, plus one channel per bond + order (1.0, 2.0, 3.0 and, with ``--explicit-aromatic``, 1.5) with a Gaussian at each + bond midpoint. Values are thresholded at 0.1 and min-max normalised per channel + (``utils.create_gaussian_batch_pdf_values`` of the FMG repo); + 4. the field is passed through the class-conditional 3D U-Net at a fixed diffusion + timestep (``--timestep``), with no noise added to the field, class label 0 and no + classifier-free-guidance dropout; + 5. the output of the U-Net's ``final_res_block`` -- a (128, G, G, G) feature map -- is + averaged over the three spatial axes, giving the 128-d embedding. + +``--dataset`` takes the input syntax shared by every script in ``baselines/`` (see +``baselines/common.py``) and every form gives rows in the same order: + + * ``hf:EscheWang/3dcs:chirality`` -- the ``chirality`` config of the Hugging Face + dataset; a bare Hub id, ``hfdisk:`` and a plain ``save_to_disk`` directory read + the same rows. Conformers are taken from the ``mol_blocks`` column in ascending + ``offset``, and each row's ``offset`` is checked against the running conformer count. + * ```` -- a pickle holding a list of RDKit molecules with one conformer each. + +Output: a compressed ``.npz`` with ``embeddings`` (float32, N x 128) and ``smiles`` +(isomeric SMILES of each input molecule, as written by the RDKit build in use). + +Example +------- + python extract_chirality.py \ + --fmg-repo /path/to/FMG \ + --checkpoint /path/to/model-120qm9_3rd_run.pt \ + --dataset hf:EscheWang/3dcs:chirality \ + --out chirality_fmg.npz \ + --batch-size 32 --device cuda:0 +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import sys +import time +from collections.abc import Iterable, Sequence +from dataclasses import dataclass + +import numpy as np + +# --------------------------------------------------------------------------------- +# defaults -- these are the settings the released chirality embeddings were made with +# --------------------------------------------------------------------------------- +DEFAULT_GRID_SIZE = 24 +DEFAULT_RESOLUTION = 0.40 +DEFAULT_GAUSSIAN_STD = 0.08 +DEFAULT_TIMESTEP = 50 +DEFAULT_ATOM_CHANNELS = "C,O,N,F" +DEFAULT_UNET_DIM = 128 +DEFAULT_UNET_DIM_MULTS = "1,2,3" +DEFAULT_NUM_CLASSES = 17 + + +@dataclass +class MolRecord: + coords: np.ndarray + atoms: np.ndarray + bonds: np.ndarray # (n_bonds, 3): begin, end, bond order + smiles: str + + +# --------------------------------------------------------------------------------- +# input +# --------------------------------------------------------------------------------- +def _bond_order_map(): + from rdkit import Chem + + return { + Chem.rdchem.BondType.SINGLE: 1.0, + Chem.rdchem.BondType.DOUBLE: 2.0, + Chem.rdchem.BondType.TRIPLE: 3.0, + Chem.rdchem.BondType.AROMATIC: 1.5, + } + + +def _record_from_mol(mol, align, allowed, bond_order_map) -> MolRecord | None: + from rdkit import Chem + + if mol is None or mol.GetNumConformers() == 0: + return None + conf = mol.GetConformer() + coords = np.array( + [ + [conf.GetAtomPosition(i).x, conf.GetAtomPosition(i).y, conf.GetAtomPosition(i).z] + for i in range(mol.GetNumAtoms()) + ], + dtype=np.float32, + ) + coords = align(coords) + atom_symbols = np.array([atom.GetSymbol() for atom in mol.GetAtoms()]) + valid_mask = np.array([sym in allowed for sym in atom_symbols]) + if not valid_mask.any(): + return None + coords = coords[valid_mask] + atoms = atom_symbols[valid_mask] + index_map = {orig: new for new, orig in enumerate(np.where(valid_mask)[0])} + bonds = [] + for bond in mol.GetBonds(): + order = bond_order_map.get(bond.GetBondType()) + if order is None: + continue + begin, end = bond.GetBeginAtomIdx(), bond.GetEndAtomIdx() + if begin in index_map and end in index_map: + bonds.append((index_map[begin], index_map[end], order)) + smiles = Chem.MolToSmiles(Chem.Mol(mol), isomericSmiles=True) + bond_array = np.array(bonds, dtype=np.float32).reshape(-1, 3) if bonds else np.zeros((0, 3), dtype=np.float32) + return MolRecord(coords=coords, atoms=atoms, bonds=bond_array, smiles=smiles) + + +def load_molecules(spec: str, *, limit: int | None = None, start: int = 0, revision: str | None = None) -> list: + """The input conformers as a list of RDKit molecules, in benchmark row order.""" + return _common().load_conformers(spec, revision=revision, limit=limit, start=start) + + +def iter_molecules(args) -> Iterable: + """Yield the input conformers as RDKit molecules, in benchmark row order. + + ``--dataset`` takes the syntax shared by every script in ``baselines/`` + (see ``baselines/common.py``): ``hf:[:]``, ``hfdisk:`` or a plain + ``save_to_disk`` directory, a bare Hub dataset id, a pickle of RDKit molecules, or + ``lmdb:``. Hugging Face rows are read in ascending ``offset``, and the offsets + are checked against the running conformer count, so the row order is the published one. + """ + return _common().iter_conformers( + args.dataset, + revision=args.hf_revision, + limit=args.limit, + start=args.start, + ) + + +def load_records(args, align) -> list[MolRecord]: + allowed = set(s.strip() for s in args.atom_channels.split(",") if s.strip()) + bond_order_map = _bond_order_map() + records: list[MolRecord] = [] + skipped = 0 + for mol in iter_molecules(args): + rec = _record_from_mol(mol, align, allowed, bond_order_map) + if rec is None: + skipped += 1 + continue + records.append(rec) + if skipped: + print(f"[warn] skipped {skipped} molecules without a conformer or without any kept atom") + return records + + +# --------------------------------------------------------------------------------- +# field construction + model +# --------------------------------------------------------------------------------- +class MolFieldDataset: + """torch Dataset over MolRecords -- returns the Gaussian centres and their channels.""" + + def __init__(self, records, atom_channels: Sequence[str], bond_orders: Sequence[float]): + import torch + + self._torch = torch + self.records = records + self.atom_channels = list(atom_channels) + self.bond_orders = list(bond_orders) + self.atom_map = {sym: i for i, sym in enumerate(self.atom_channels)} + self.bond_map = {o: len(self.atom_channels) + i for i, o in enumerate(self.bond_orders)} + self.total_channels = len(self.atom_channels) + len(self.bond_orders) + + def __len__(self): + return len(self.records) + + def __getitem__(self, idx): + torch = self._torch + rec = self.records[idx] + points, ids, counts = [], [], [] + coords, atoms = rec.coords, rec.atoms + for sym in self.atom_channels: + mask = atoms == sym + n = int(mask.sum()) + counts.append(n) + if n: + points.append(coords[mask]) + ids.append(np.full(n, self.atom_map[sym], dtype=np.int64)) + bond_counts = [] + if rec.bonds.size: + for order in self.bond_orders: + mask = np.isclose(rec.bonds[:, 2], order) + n = int(mask.sum()) + bond_counts.append(n) + if n: + mids = 0.5 * (coords[rec.bonds[mask, 0].astype(int)] + coords[rec.bonds[mask, 1].astype(int)]) + points.append(mids) + ids.append(np.full(n, self.bond_map[order], dtype=np.int64)) + else: + bond_counts.extend([0] * len(self.bond_orders)) + if not points: + points = [coords] + ids = [np.zeros(len(coords), dtype=np.int64)] + counts = [len(coords)] + [0] * (len(self.atom_channels) - 1) + bond_counts = [0] * len(self.bond_orders) + coords_all = np.concatenate(points, axis=0).astype(np.float32) + ids_all = np.concatenate(ids, axis=0).astype(np.int64) + return ( + torch.from_numpy(coords_all), + torch.from_numpy(ids_all), + counts + bond_counts, + rec.smiles, + ) + + +def make_collate(total_channels: int): + import torch + + def _collate(batch): + coord_list, id_list, n_lists, smiles = [], [], [], [] + for sample_idx, (coords, ids, n_list, smi) in enumerate(batch): + coord_list.append(coords) + id_list.append(ids + sample_idx * total_channels) + n_lists.append(n_list) + smiles.append(smi) + return torch.cat(coord_list, 0), torch.cat(id_list, 0), n_lists, smiles + + return _collate + + +def prepare_grid(grid_size: int, resolution: float, device): + import torch + + half = 0.5 * resolution * (grid_size - 1) + axis = np.linspace(-half, half, grid_size, dtype=np.float32) + flat = np.stack(np.meshgrid(axis, axis, axis, indexing="ij"), axis=-1).reshape(-1, 3) + return torch.tensor(flat, dtype=torch.float32, device=device) + + +def build_encoder(args, total_channels: int, device): + import torch + from denoising_diffusion_pytorch.classifier_free_guidance import Unet3D + + dim_mults = tuple(int(m) for m in args.unet_dim_mults.split(",") if m.strip()) + + class DiffusionEncoder(torch.nn.Module): + def __init__(self): + super().__init__() + self.unet = Unet3D( + dim=args.unet_dim, + num_classes=args.num_classes, + cond_drop_prob=0.0, + dim_mults=dim_mults, + channels=total_channels, + legacy_attention=args.legacy_attention, + add_pe=args.add_pe, + ) + self._latent = None + self.unet.final_res_block.register_forward_hook(self._capture) + + def _capture(self, module, inputs, output): + self._latent = output + + def forward(self, fields, labels, timestep): + bsz = fields.shape[0] + t = torch.full((bsz,), float(timestep), dtype=torch.float32, device=fields.device) + self._latent = None + self.unet(fields, t, labels, cond_var=None, cond_drop_prob=0.0) + if self._latent is None: + raise RuntimeError("failed to capture the final_res_block feature map") + emb = self._latent.mean(dim=(2, 3, 4)) + self._latent = None + return emb + + model = DiffusionEncoder().to(device) + + state = torch.load(args.checkpoint, map_location=device) + if "ema" in state and "online_model.model.init_conv.weight" in state["ema"]: + weights, prefix = state["ema"], "online_model.model." + else: + weights, prefix = state.get("model", state), "module.model." + unet_state = {k[len(prefix) :]: v for k, v in weights.items() if k.startswith(prefix)} + if not unet_state: + raise RuntimeError(f"no weights with prefix {prefix!r} found in {args.checkpoint}") + missing, unexpected = model.unet.load_state_dict(unet_state, strict=False) + if missing: + raise RuntimeError(f"checkpoint is missing {len(missing)} keys, first: {missing[:5]}") + if unexpected: + print(f"[warn] {len(unexpected)} unexpected keys in the checkpoint, first: {unexpected[:5]}") + print(f"loaded {len(unet_state)} tensors from {args.checkpoint} (prefix {prefix!r})") + model.eval() + return model + + +# --------------------------------------------------------------------------------- +# main +# --------------------------------------------------------------------------------- +def sha256(path: str, chunk: int = 1 << 22) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for block in iter(lambda: fh.read(chunk), b""): + h.update(block) + return h.hexdigest() + + +MODEL_NAME = "fmg" +OUTPUT_KEY = "embeddings" + + +def _common(): + """Load ``baselines/common.py`` (verification helpers) without touching sys.path.""" + import importlib.util + import os + + path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "common.py") + spec = importlib.util.spec_from_file_location("baselines_common", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def parse_args(argv=None): + p = argparse.ArgumentParser( + description="Extract FMG embeddings for the 3DCS chirality set.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p.add_argument("--dataset", required=True, help=_common().DATASET_SPEC_HELP) + p.add_argument("--hf-revision", default=None, help="pin the Hub revision when --dataset is a Hub id") + p.add_argument("--out", required=True, help="output .npz") + p.add_argument( + "--fmg-repo", required=True, help="clone of https://github.com/Dumitrescu-Alexandru/FMG (prepended to sys.path)" + ) + p.add_argument("--checkpoint", required=True, help="FMG QM9 3D U-Net checkpoint (.pt)") + p.add_argument("--batch-size", type=int, default=32) + p.add_argument("--num-workers", type=int, default=4) + p.add_argument("--device", default="cuda:0" if os.environ.get("CUDA_VISIBLE_DEVICES", "") != "-1" else "cpu") + p.add_argument("--grid-size", type=int, default=DEFAULT_GRID_SIZE) + p.add_argument("--resolution", type=float, default=DEFAULT_RESOLUTION) + p.add_argument("--gaussian-std", type=float, default=DEFAULT_GAUSSIAN_STD) + p.add_argument("--timestep", type=int, default=DEFAULT_TIMESTEP) + p.add_argument("--atom-channels", default=DEFAULT_ATOM_CHANNELS) + p.add_argument( + "--no-explicit-aromatic", + dest="explicit_aromatic", + action="store_false", + help="drop the aromatic (order 1.5) bond channel; the released embeddings keep it", + ) + p.set_defaults(explicit_aromatic=True) + p.add_argument("--unet-dim", type=int, default=DEFAULT_UNET_DIM) + p.add_argument("--unet-dim-mults", default=DEFAULT_UNET_DIM_MULTS) + p.add_argument("--num-classes", type=int, default=DEFAULT_NUM_CLASSES) + p.add_argument("--legacy-attention", action="store_true") + p.add_argument("--add-pe", action="store_true") + p.add_argument("--class-label", type=int, default=0, help="class-conditioning label fed to the U-Net") + p.add_argument("--limit", type=int, default=None, help="only process the first N conformers") + p.add_argument("--start", type=int, default=0, help="skip the first N conformers") + p.add_argument( + "--deterministic", + action="store_true", + help="force deterministic cuDNN kernels and disable TF32; slower, and not the setting " + "the released file was produced with (library defaults)", + ) + p.add_argument("--log-interval", type=int, default=5000) + p.add_argument("--dtype", choices=["float32"], default="float32") + p.add_argument( + "--verify", + nargs="?", + const="published", + default=None, + metavar="REFERENCE", + help="after writing, compare the output with a reference file: 'published' " + "(the published FMG chirality embedding), hub:, or a local path", + ) + p.add_argument( + "--verify-key", default=None, help="array key to read from the --verify reference (default: its published key)" + ) + p.add_argument("--verify-rows", default=None, metavar="ROWS", help=_common().ROW_SELECTION_HELP) + return p.parse_args(argv) + + +def main(argv=None): + args = parse_args(argv) + + repo = os.path.abspath(os.path.expanduser(args.fmg_repo)) + if not os.path.isdir(repo): + sys.exit(f"--fmg-repo {repo} is not a directory") + sys.path.insert(0, repo) + + import torch + from rdkit import RDLogger, rdBase + + RDLogger.DisableLog("rdApp.*") + + from utils import align, create_gaussian_batch_pdf_values # FMG upstream + + versions = { + "python": platform.python_version(), + "numpy": np.__version__, + "torch": torch.__version__, + "torch_cuda": torch.version.cuda, + "cudnn": torch.backends.cudnn.version(), + "rdkit": rdBase.rdkitVersion, + "fmg_repo": repo, + "device": args.device, + "gpu": torch.cuda.get_device_name(0) if torch.cuda.is_available() else None, + } + print("versions:", json.dumps(versions)) + print("checkpoint sha256:", sha256(args.checkpoint)) + print("settings:", json.dumps({k: v for k, v in vars(args).items()}, default=str)) + + torch.manual_seed(0) + np.random.seed(0) + if args.deterministic: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + print( + f"cudnn.deterministic={torch.backends.cudnn.deterministic} benchmark={torch.backends.cudnn.benchmark} allow_tf32={torch.backends.cudnn.allow_tf32} matmul.allow_tf32={torch.backends.cuda.matmul.allow_tf32}" + ) + + device = torch.device(args.device) + + t0 = time.time() + records = load_records(args, align) + print(f"loaded {len(records)} conformers in {time.time() - t0:.1f} s") + if not records: + sys.exit("no usable conformers") + + atom_channels = [s.strip() for s in args.atom_channels.split(",") if s.strip()] + bond_orders = [1.0, 2.0, 3.0] + ([1.5] if args.explicit_aromatic else []) + dataset = MolFieldDataset(records, atom_channels, bond_orders) + print(f"field channels: {atom_channels} + bond orders {bond_orders} = {dataset.total_channels}") + + from torch.utils.data import DataLoader + + loader = DataLoader( + dataset, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True, + collate_fn=make_collate(dataset.total_channels), + ) + + model = build_encoder(args, dataset.total_channels, device) + grid = prepare_grid(args.grid_size, args.resolution, device) + + embeddings, smiles_all, processed = [], [], 0 + t0 = time.time() + with torch.no_grad(): + for coords, inds, n_lists, smiles in loader: + coords = coords.to(device, non_blocking=True) + inds = inds.to(device, non_blocking=True) + fields = create_gaussian_batch_pdf_values( + x=grid, + coords=coords, + N_list=n_lists, + std=args.gaussian_std, + device=device, + gaussian_indices=inds, + no_fields=dataset.total_channels, + grid_shapes=[args.grid_size] * 3, + ) + labels = torch.full((fields.shape[0],), args.class_label, dtype=torch.long, device=device) + emb = model(fields, labels, args.timestep) + embeddings.append(emb.float().cpu().numpy()) + smiles_all.extend(smiles) + prev, processed = processed, processed + len(smiles) + if args.log_interval > 0 and ( + processed == len(dataset) or processed // args.log_interval != prev // args.log_interval + ): + rate = processed / max(time.time() - t0, 1e-9) + print(f" {processed}/{len(dataset)} {rate:.1f} mol/s", flush=True) + dt = time.time() - t0 + embeddings = np.concatenate(embeddings, axis=0).astype(np.float32) + print(f"forward pass: {len(dataset)} conformers in {dt:.1f} s ({len(dataset) / dt:.1f} mol/s)") + if device.type == "cuda": + print( + f"peak GPU memory: {torch.cuda.max_memory_allocated(device) / 2**30:.2f} GiB allocated, {torch.cuda.max_memory_reserved(device) / 2**30:.2f} GiB reserved" + ) + + out = os.path.abspath(args.out) + os.makedirs(os.path.dirname(out) or ".", exist_ok=True) + np.savez_compressed(out, embeddings=embeddings, smiles=np.array(smiles_all)) + print(f"wrote {out} shape={embeddings.shape} sha256={sha256(out)}") + + if args.verify: + _common().verify( + out, + model=MODEL_NAME, + reference=args.verify, + produced_key=OUTPUT_KEY, + reference_key=args.verify_key, + rows=args.verify_rows or (f"{args.start}+" if args.start else None), + ) + + +if __name__ == "__main__": + main() diff --git a/baselines/fmg/requirements.txt b/baselines/fmg/requirements.txt new file mode 100644 index 0000000..05b4ab9 --- /dev/null +++ b/baselines/fmg/requirements.txt @@ -0,0 +1,38 @@ +# Environment for baselines/fmg/extract_chirality.py +# +# Install order matters: torch first (from the PyTorch CUDA 11.8 index), then the rest +# from PyPI. Python 3.10. +# +# conda create -y -p ./env python=3.10 +# ./env/bin/pip install "pip==24.0" "setuptools==67.8.0" wheel +# ./env/bin/pip install torch==2.0.0 torchvision==0.15.1 --index-url https://download.pytorch.org/whl/cu118 +# ./env/bin/pip install -r requirements.txt +# +# torch 2.0.0 + cuDNN 8.7 (the versions pinned by the upstream FMG requirements.txt) is +# what the released embeddings were computed with; the +cu118 wheels run unchanged on a +# CUDA 13 driver. The packages below the first block are needed only because importing +# the upstream FMG modules pulls them in. + +# --- direct dependencies of the extraction script ------------------------------- +numpy==1.26.4 +einops==0.7.0 +rdkit==2024.9.6 +datasets==4.0.0 # only for --dataset + +# --- pulled in by `import utils` / `denoising_diffusion_pytorch` of the FMG repo -- +scipy==1.10.1 +pandas==2.2.1 +networkx==3.2.1 +matplotlib==3.7.1 +pysmiles==1.1.2 +plotly==5.19.0 +py3Dmol==2.0.4 +omegaconf==2.3.0 +msgpack-python==0.5.6 +torch_geometric==2.3.0 +pytorch_fid==0.3.0 +ema-pytorch==0.4.2 +accelerate==0.18.0 +torch-dct==0.1.6 +scikit-learn==1.2.2 +tqdm>=4.66.3 # the floor datasets 4.0.0 requires; 4.70.1 resolves here diff --git a/baselines/gemnet/ENVIRONMENT.md b/baselines/gemnet/ENVIRONMENT.md new file mode 100644 index 0000000..ea56815 --- /dev/null +++ b/baselines/gemnet/ENVIRONMENT.md @@ -0,0 +1,123 @@ +# GemNet-Q — environment + +`extract_chirality.py` drives an unmodified `gemnet_pytorch` checkout: the graph is built by +upstream's own `DataContainer`, and the per-atom representation `h` of the last interaction +block is read through a `register_forward_hook`, so upstream's `gemnet.py` runs exactly as +published. No third-party code and no weights are redistributed here. + +## Upstream + +```bash +git clone https://github.com/TUM-DAML/gemnet_pytorch.git +git -C gemnet_pytorch checkout a0164f74217155232d39c35f0bb2c016bd3f44da +``` + +Upstream `gemnet_pytorch` is licensed under the Hippocratic License 2.0. Check that it +permits your use before running it. + +## Weights + +They are part of that checkout — the released GemNet-Q pretraining, no fine-tuned weights. + +| file in the checkout | sha256 | +|---|---| +| `pretrained/GemNet-Q/model.pth` | `d51e00af18cf9dd0097f0bc251386fdd539cfe82fdeb2790d0a0f9a241f169cd` | +| `pretrained/GemNet-Q/model_kwargs.json` | `a1283907379e62a9af447d7dc74410cdc9e0969265f87d1e842c5f72238825d0` | +| `pretrained/scaling_factors.json` | `f9c855d929b8774b003c20246733faa571befe9d999bda2025fe7618cc100635` | + +`model_kwargs.json` selects GemNet-Q: `num_blocks 4`, `emb_size_atom 128`, `cutoff 5.0`, +`int_cutoff 10.0`, `triplets_only false`, `direct_forces false`, `extensive true`, +`activation swish`, `scale_file scaling_factors.json`. + +## Install + +```bash +conda create -y -p ./env python=3.10 +./env/bin/pip install torch==2.1.0 --index-url https://download.pytorch.org/whl/cu121 +./env/bin/pip install torch_scatter -f https://data.pyg.org/whl/torch-2.1.0+cu121.html +./env/bin/pip install -r requirements.txt +``` + +Resolved versions: python 3.10.21, torch 2.1.0+cu121, torch_scatter 2.1.2+pt21cu121, +numpy 1.24.4, scipy 1.10.1, sympy 1.12, numba 0.58.1, rdkit 2025.03.5. The script prints all +of them at startup. + +Three compatibility points, all handled by the script: + +- **`torch_scatter` is optional.** If the compiled package is not importable, + `compat/torch_scatter.py` is used instead — a pure-PyTorch implementation of the + `add` / `mean` reductions GemNet needs. On this dataset the two agree to + `max|diff| = 7.4e-06` (`np.allclose(atol=1e-5, rtol=1e-4)` passes). +- **numpy aliases.** Upstream targets numpy < 1.24 and uses names later releases removed: + `np.bool` (`gemnet/training/data_container.py`) and `np.math.factorial` + (`gemnet/model/layers/basis_utils.py`). The script restores those aliases at import time + instead of editing the checkout. +- Upstream ships `gemnet/` without `__init__.py` files; the script creates the empty ones if + they are missing. + +## Run + +```bash +python extract_chirality.py \ + --gemnet-repo ./gemnet_pytorch \ + --dataset hf:EscheWang/3dcs:chirality \ + --out gemnet_chirality.npz \ + --batch-size 8 --device cuda --checkpoint-every 8000 --verify +``` + +`--dataset` takes the input specification shared by every script in `baselines/` (see the +table in [`../README.md`](../README.md#running-one)): `hf:[:]`, `hfdisk:` or a plain +`save_to_disk` directory, a bare Hub dataset id, a pickle of RDKit molecules, or `lmdb:`. +`--limit`/`--start` run a slice of the conformers and `--verify-rows` says which rows of the +reference that slice covers. + +Row order follows the dataset: for the Hugging Face forms, rows are ordered by `offset` and +then by position inside each row's `mol_blocks`. The script refuses to skip a conformer it +cannot featurise rather than shifting the rows. + +Options: `--pooling {mean,add}`, `--hydrogens {all,remove,keep}`, `--cutoff`, `--int-cutoff`, +`--triplets-only`, `--round-coords`, `--fp16`, `--tf32`, `--start/--limit`, `--batch-size`, +`--device`, `--seed`, `--num-threads`, `--oom-retries/--oom-wait` (retry a batch in smaller +pieces after a CUDA OOM) and `--checkpoint-every` (write the partial `.npz` as it goes). +## Settings + +`Chem.RemoveAllHs` (heavy atoms only, `--hydrogens all`), edges within 5 A, quadruplet +interactions within 10 A, float32, mean pooling of `h` over the atoms of the molecule, no +normalisation afterwards. + +## Determinism + +The script seeds python / numpy / torch, disables cuDNN autotuning and leaves TF32 off, so +repeated runs on the same machine agree bit for bit, and `--batch-size` does not change the +result (a run at `--batch-size 1` and one at `--batch-size 64` agreed to the last float32 bit +on the first 256 conformers). Results are not bit-identical across different GPUs and +CUDA / cuBLAS versions: the same code with TF32 on moves the output by `max|diff| = 2.0e-03`. + +## Input precision + +The published embedding file and the agreement numbers in [`../README.md`](../README.md) were +computed from the source RDKit molecules, whose coordinates carry full float precision. The public +`EscheWang/3dcs` dataset stores those geometries as V2000 MOL blocks, which hold four decimals. +Over the first 2,000 conformers, a run from `hf:EscheWang/3dcs:chirality` and a run from the +full-precision molecules, on the same machine in the same environment, differ by +`max|diff| = 5.1e-04` and `mean|diff| = 1.9e-05`, per-row cosine at least 0.999999996, no row below +1 - 1e-6. On the first 256 conformers the same comparison gives `max|diff| = 2.4e-04`, and +`--round-coords 4` on the full-precision molecules reproduces it, so the rounding of the MOL blocks +is the whole of the difference. GemNet-Q is, with MACE, among the least sensitive of the seven. +## Rotation shards + +The same settings cover the rotation track. Pass `--dataset lmdb:.lmdb`: each key of a +shard holds a list of `(Mol, energy, torsion_deg)`, and the row order is the LMDB cursor +order of the keys followed by the position inside each list. For shard 0 that gives 630,021 +conformers over 97,487 keys, which is the shape of the published +`rotation/gemnet/rotation_conformers_0.npz`. On the first 2,048 rows of that shard the script +agrees with the published file to a per-row cosine of at least 0.9999997 +(`max|diff| = 3.4e-03`). Requires `lmdb`. Write the output with `--key gemnet` and the file +name `rotation_conformers_.npz` to match the `--layout by-shard` reader. + +## Cost + +52,391 conformers on one A100-80GB: 44 min at `--batch-size 8` (19.8 conformers/s) on a busy +machine. Peak GPU memory is a few GB at `--batch-size 8`; a handful of large molecules spike +much higher, so on a shared GPU use a small `--batch-size` and rely on `--oom-retries`. +`--hydrogens keep` makes the quadruplet index sets explode and can exhaust an 80 GB GPU. diff --git a/baselines/gemnet/compat/torch_scatter.py b/baselines/gemnet/compat/torch_scatter.py new file mode 100644 index 0000000..c59fdf5 --- /dev/null +++ b/baselines/gemnet/compat/torch_scatter.py @@ -0,0 +1,60 @@ +"""Minimal drop-in replacement for the torch_scatter entry point GemNet uses. + +Mirrors ``torch_scatter.scatter(..., reduce="add"/"mean")`` with pure PyTorch ops, so the +extraction script does not need a compiled torch-scatter build. It is only used when the +real package is not importable; see README.md for the agreement between the two. +""" + +import torch + + +def broadcast(src: torch.Tensor, other: torch.Tensor, dim: int) -> torch.Tensor: + if dim < 0: + dim = other.dim() + dim + if src.dim() == 1: + for _ in range(0, dim): + src = src.unsqueeze(0) + for _ in range(src.dim(), other.dim()): + src = src.unsqueeze(-1) + return src.expand(other.size()) + + +def scatter_sum(src, index, dim=-1, out=None, dim_size=None): + index = broadcast(index, src, dim) + if out is None: + size = list(src.size()) + if dim_size is not None: + size[dim] = dim_size + elif index.numel() == 0: + size[dim] = 0 + else: + size[dim] = int(index.max()) + 1 + out = torch.zeros(size, dtype=src.dtype, device=src.device) + return out.scatter_add_(dim, index, src) + + +def scatter_mean(src, index, dim=-1, out=None, dim_size=None): + out = scatter_sum(src, index, dim, out, dim_size) + dim_size = out.size(dim) + index_dim = dim + if index_dim < 0: + index_dim = index_dim + src.dim() + if index.dim() <= index_dim: + index_dim = index.dim() - 1 + ones = torch.ones(index.size(), dtype=src.dtype, device=src.device) + count = scatter_sum(ones, index, index_dim, None, dim_size) + count[count < 1] = 1 + count = broadcast(count, out, dim) + if out.is_floating_point(): + out.true_divide_(count) + else: + out.div_(count, rounding_mode="floor") + return out + + +def scatter(src, index, dim=-1, out=None, dim_size=None, reduce="sum"): + if reduce in ("sum", "add"): + return scatter_sum(src, index, dim, out, dim_size) + if reduce == "mean": + return scatter_mean(src, index, dim, out, dim_size) + raise ValueError(f"scatter shim does not implement reduce='{reduce}'") diff --git a/baselines/gemnet/extract_chirality.py b/baselines/gemnet/extract_chirality.py new file mode 100644 index 0000000..6065f16 --- /dev/null +++ b/baselines/gemnet/extract_chirality.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +"""Extract GemNet-Q graph embeddings for the 3DCS chirality conformer set. + +What this computes +------------------ +For every conformer, the molecule is reduced to its heavy atoms +(``rdkit.Chem.RemoveAllHs``), turned into a GemNet graph (5 A edge cutoff, +10 A quadruplet interaction cutoff), and pushed through the pretrained GemNet-Q +backbone. The per-atom representation ``h`` produced by the last interaction +block is averaged over the atoms of each molecule, giving one 128-d vector per +conformer. The result is written to a ``.npz`` whose single array is keyed +``gemnet``, in the row order of the input dataset. + +Third-party code and weights +---------------------------- +The GemNet architecture and the GemNet-Q weights are not part of this +repository. Fetch them from the upstream project and point ``--gemnet-repo`` at +the checkout: + + git clone https://github.com/TUM-DAML/gemnet_pytorch.git + git -C gemnet_pytorch checkout a0164f74217155232d39c35f0bb2c016bd3f44da + + # pretrained/GemNet-Q/model.pth sha256 d51e00af18cf9dd0097f0bc251386fdd539cfe82fdeb2790d0a0f9a241f169cd + # pretrained/scaling_factors.json sha256 f9c855d929b8774b003c20246733faa571befe9d999bda2025fe7618cc100635 + +This script does not modify that checkout and contains no code derived from it. +The graph (edges, triplets, quadruplets) is built by upstream's own +``gemnet.training.data_container.DataContainer``, and ``h`` is read with a +forward hook on the last interaction block, so upstream's ``gemnet.py`` runs +exactly as published. Upstream is licensed under the Hippocratic License 2.0; +check that it permits your use before running it. + +Environment (see README.md for the exact install commands) +---------------------------------------------------------- +python 3.10, torch 2.1.0+cu121, torch_scatter 2.1.2, numpy 1.24.4, scipy 1.10.1, +sympy 1.12, numba 0.58.1, rdkit 2025.03.5. ``torch_scatter`` is optional: without it the +pure-PyTorch fallback in ``compat/torch_scatter.py`` is used, which agrees with +the compiled package to 7.4e-06 on this dataset. + +Examples +-------- + # from the released HuggingFace dataset + python extract_chirality.py --gemnet-repo ./gemnet_pytorch \ + --dataset hf:EscheWang/3dcs:chirality \ + --out gemnet_chirality.npz --batch-size 32 --device cuda + + # from a local pickle holding a list of RDKit Mol objects + python extract_chirality.py --gemnet-repo ./gemnet_pytorch \ + --dataset chirality_bench_conformers_noised_only_aslist.pkl \ + --out gemnet_chirality.npz +""" + +from __future__ import annotations + +import argparse +import json +import os +import random +import sys +import time +from pathlib import Path + +import numpy as np + +HERE = Path(__file__).resolve().parent + +MODEL_NAME = "gemnet" + + +def _common(): + """Load ``baselines/common.py`` (verification helpers) without touching sys.path.""" + import importlib.util + import os + + path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "common.py") + spec = importlib.util.spec_from_file_location("baselines_common", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# --------------------------------------------------------------------------- CLI +def parse_args(argv=None): + p = argparse.ArgumentParser( + description="Extract GemNet-Q chirality embeddings.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--gemnet-repo", + default="gemnet_pytorch", + help="Checkout of github.com/TUM-DAML/gemnet_pytorch (commit a0164f7).", + ) + p.add_argument( + "--pretrained-dir", + default=None, + help="Directory with model.pth + model_kwargs.json " + "(default: /pretrained/GemNet-Q). scaling_factors.json is " + "resolved relative to its parent, as upstream does.", + ) + p.add_argument("--dataset", required=True, help=_common().DATASET_SPEC_HELP) + p.add_argument("--out", required=True, help="Output .npz path.") + p.add_argument("--key", default="gemnet", help="Array key inside the .npz (default: gemnet).") + p.add_argument("--batch-size", type=int, default=32, help="Conformers per forward pass.") + p.add_argument("--device", default="cuda", help="cuda, cuda:0 or cpu.") + p.add_argument( + "--pooling", + default="mean", + choices=["mean", "add"], + help="Per-molecule reduction of the final atom features h.", + ) + p.add_argument( + "--hydrogens", + default="all", + choices=["all", "remove", "keep"], + help="all = Chem.RemoveAllHs (heavy atoms only); remove = Chem.RemoveHs, which " + "keeps the hydrogens RDKit treats as stereo-defining; keep = no removal.", + ) + p.add_argument("--cutoff", type=float, default=5.0, help="GemNet edge cutoff in Angstrom.") + p.add_argument("--int-cutoff", type=float, default=10.0, help="GemNet quadruplet interaction cutoff in Angstrom.") + p.add_argument("--triplets-only", action="store_true", help="Disable quadruplet interactions.") + p.add_argument( + "--round-coords", + type=int, + default=0, + help="Round coordinates to N decimals before featurising (0 = off). The MDL mol " + "blocks of the released dataset carry 4 decimals; this makes a run on a " + "full-precision pickle comparable with one on the dataset.", + ) + p.add_argument( + "--max-atoms", type=int, default=0, help="Truncate conformers to this many atoms (0 = no truncation)." + ) + p.add_argument("--fp16", action="store_true", help="Run the forward pass under autocast(float16).") + p.add_argument( + "--tf32", + action="store_true", + help="Allow TensorFloat-32 matmuls on Ampere+ GPUs. Off by default: it costs " + "about three mantissa bits and makes the output machine-dependent.", + ) + p.add_argument("--limit", type=int, default=0, help="Process only the first N conformers.") + p.add_argument("--start", type=int, default=0, help="Skip the first N conformers.") + p.add_argument("--seed", type=int, default=0) + p.add_argument("--num-threads", type=int, default=8) + p.add_argument("--log-interval", type=int, default=5000, help="Progress line every N conformers.") + p.add_argument( + "--oom-retries", + type=int, + default=3, + help="On CUDA OOM, wait and retry the batch, halving it each time down to 1 " + "conformer. A few large molecules need far more memory than the median, " + "and a shared GPU can be busy. 0 disables.", + ) + p.add_argument("--oom-wait", type=float, default=30.0, help="Seconds to wait before each OOM retry.") + p.add_argument( + "--checkpoint-every", + type=int, + default=0, + help="Write the partial output every N conformers (0 = only at the end). The " + "partial file is the same .npz with the rows computed so far.", + ) + p.add_argument( + "--work-dir", + default=None, + help="Where to put the temporary .npz that upstream's DataContainer reads (default: next to --out).", + ) + p.add_argument( + "--verify", + nargs="?", + const="published", + default=None, + metavar="REFERENCE", + help="after writing, compare the output with a reference file: 'published' " + "(the published GemNet chirality embedding), hub:, or a local path", + ) + p.add_argument( + "--verify-key", default=None, help="array key to read from the --verify reference (default: its published key)" + ) + p.add_argument("--verify-rows", default=None, metavar="ROWS", help=_common().ROW_SELECTION_HELP) + return p.parse_args(argv) + + +# ------------------------------------------------------------------- environment +def ensure_imports(gemnet_repo: Path): + """Put the upstream checkout on sys.path and make its two hard edges work: + torch_scatter (falls back to compat/) and the numpy aliases later releases removed.""" + notes = [] + repo = gemnet_repo.resolve() + if not (repo / "gemnet" / "model" / "gemnet.py").exists(): + raise SystemExit(f"--gemnet-repo {repo} does not look like a gemnet_pytorch checkout") + # Upstream ships gemnet/ without __init__.py files; add them if they are missing. + for sub in ("gemnet", "gemnet/model", "gemnet/model/layers", "gemnet/training"): + init = repo / sub / "__init__.py" + if (repo / sub).is_dir() and not init.exists(): + init.touch() + notes.append(f"created {init.relative_to(repo)}") + if str(repo) not in sys.path: + sys.path.insert(0, str(repo)) + try: + import torch_scatter # noqa: F401 + + notes.append("torch_scatter: installed package") + except ImportError: + sys.path.append(str(HERE / "compat")) + import torch_scatter # noqa: F401 + + notes.append("torch_scatter: compat shim") + import math + + # Upstream targets numpy < 1.24 and uses names later numpy releases removed: + # np.bool (data_container.py) and np.math.factorial (basis_utils.py). Restore the + # aliases here rather than editing the upstream checkout. + restored = [] + for name, value in (("bool", bool), ("int", int), ("float", float), ("object", object)): + if not hasattr(np, name): + setattr(np, name, value) + restored.append(f"np.{name}") + if not hasattr(np, "math"): + np.math = math + restored.append("np.math") + if restored: + notes.append("restored removed numpy aliases for upstream: " + ", ".join(restored)) + return notes + + +def set_deterministic(seed: int, tf32: bool = False): + random.seed(seed) + np.random.seed(seed) + import torch + + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.backends.cuda.matmul.allow_tf32 = tf32 + torch.backends.cudnn.allow_tf32 = tf32 + + +def print_versions(): + import numba + import rdkit + import scipy + import sympy + import torch + import torch_scatter + + print("[versions] python ", sys.version.split()[0]) + print("[versions] numpy ", np.__version__) + print("[versions] scipy ", scipy.__version__) + print("[versions] sympy ", sympy.__version__) + print("[versions] numba ", numba.__version__) + print("[versions] rdkit ", rdkit.__version__) + print("[versions] torch ", torch.__version__, "cuda", torch.version.cuda) + print( + "[versions] torch_scatter ", getattr(torch_scatter, "__version__", "compat shim"), "at", torch_scatter.__file__ + ) + + +# ------------------------------------------------------------------------- data +def load_molecules(spec: str, *, limit: int | None = None, start: int = 0): + """Return the conformers as RDKit Mol objects, in dataset row order. + + ``spec`` takes the ``--dataset`` syntax shared by every script in ``baselines/`` + (see ``baselines/common.py``): ``hf:[:]``, ``hfdisk:`` or a plain + ``save_to_disk`` directory, a bare Hub dataset id, a pickle of RDKit molecules, or + ``lmdb:`` for a rotation shard. GemNet reads atomic numbers and coordinates only. + """ + from rdkit import RDLogger + + RDLogger.DisableLog("rdApp.*") + return _common().load_conformers(spec, limit=limit, start=start) + + +def mol_to_arrays(mol, hydrogens: str, max_atoms: int, round_coords: int = 0): + from rdkit import Chem + + if mol is None: + return None + mol = Chem.Mol(mol) + if hydrogens == "all": + mol = Chem.RemoveAllHs(mol) + elif hydrogens == "remove": + mol = Chem.RemoveHs(mol) + if mol.GetNumConformers() == 0: + return None + conf = mol.GetConformer(0) + idx = [a.GetIdx() for a in mol.GetAtoms()] + Z = np.array([mol.GetAtomWithIdx(i).GetAtomicNum() for i in idx], dtype=np.int64) + R = np.array( + [[conf.GetAtomPosition(i).x, conf.GetAtomPosition(i).y, conf.GetAtomPosition(i).z] for i in idx], + dtype=np.float32, + ) + if round_coords: + R = np.round(R.astype(np.float64), round_coords).astype(np.float32) + if max_atoms and len(Z) > max_atoms: + Z, R = Z[:max_atoms], R[:max_atoms] + return {"Z": Z, "R": R} + + +def build_container(confs, work_dir: Path, cutoff: float, int_cutoff: float, triplets_only: bool): + """Hand the conformers to upstream's own DataContainer, which builds the edge, triplet + and quadruplet index sets. It reads an .npz off disk, so write one first.""" + from gemnet.training.data_container import DataContainer + + work_dir.mkdir(parents=True, exist_ok=True) + N = np.array([c["Z"].size for c in confs], dtype=np.int64) + Z = np.concatenate([c["Z"] for c in confs]).astype(np.int64) + R = np.concatenate([c["R"] for c in confs]).astype(np.float32) + path = work_dir / f"_gemnet_inputs_{len(confs)}_{os.getpid()}.npz" + np.savez(path, N=N, Z=Z, R=R, E=np.zeros((len(confs), 1), dtype=np.float32), F=np.zeros_like(R)) + return DataContainer(str(path), cutoff, int_cutoff, triplets_only), path + + +# ------------------------------------------------------------------------ model +def load_model(pretrained_dir: Path, device, triplets_only: bool): + import torch + from gemnet.model.gemnet import GemNet + + with open(pretrained_dir / "model_kwargs.json") as fh: + kwargs = json.load(fh) + kwargs["triplets_only"] = triplets_only + if "scale_file" in kwargs: + kwargs["scale_file"] = str(pretrained_dir.parent / kwargs["scale_file"]) + model = GemNet(**kwargs).to(device) + state = torch.load(pretrained_dir / "model.pth", map_location=device) + if isinstance(state, dict): + state = state.get("state_dict", state.get("model", state)) + missing, unexpected = model.load_state_dict(state, strict=False) + if missing: + print("[load] missing keys:", missing) + if unexpected: + print("[load] unexpected keys:", unexpected) + model.eval() + return model + + +class _StopForward(Exception): + """Raised from the hook once h has been captured, so the unused energy/force tail of + the upstream forward (an autograd.grad call) never runs.""" + + +class AtomFeatureReader: + """Capture h, the atom representation leaving the last interaction block. + + Upstream's own `forward` pools exactly this tensor when it builds a graph + embedding, so hooking it needs no change to the upstream file. + """ + + def __init__(self, model): + self.h = None + self._handle = model.int_blocks[-1].register_forward_hook(self._hook) + + def _hook(self, module, args, output): + self.h = output[0] + raise _StopForward + + def __call__(self, model, inputs, pooling: str): + import torch + from torch_scatter import scatter + + self.h = None + try: + model(inputs) + except _StopForward: + pass + if self.h is None: + raise RuntimeError("the forward hook did not fire; check the gemnet_pytorch commit") + batch_seg = inputs["batch_seg"] + n_mol = int(torch.max(batch_seg)) + 1 + return scatter(self.h, batch_seg, dim=0, dim_size=n_mol, reduce=pooling) + + def close(self): + self._handle.remove() + + +def _forward(model, reader, container, idx, device, use_cuda, args): + import torch + + inputs = container[idx] + inputs = {k: v.to(device, non_blocking=use_cuda) for k, v in inputs.items() if isinstance(v, torch.Tensor)} + with torch.no_grad(): + with torch.autocast("cuda", dtype=torch.float16, enabled=args.fp16 and use_cuda): + emb = reader(model, inputs, args.pooling) + return emb.float().detach().cpu().numpy() + + +def run_batch(model, reader, container, idx, device, use_cuda, args): + """One forward pass, retrying on CUDA OOM with progressively smaller sub-batches. + + Splitting a batch does not change the result: the graph is block-diagonal, so each + conformer's embedding depends only on its own atoms. + """ + import torch + + try: + return _forward(model, reader, container, idx, device, use_cuda, args) + except torch.cuda.OutOfMemoryError: + if not use_cuda or args.oom_retries <= 0 or len(idx) == 1: + raise + size = max(1, len(idx) // 2) + for attempt in range(1, args.oom_retries + 1): + torch.cuda.empty_cache() + print( + f"[oom] rows {idx[0]}..{idx[-1]}: retry {attempt}/{args.oom_retries} " + f"in sub-batches of {size} after {args.oom_wait:.0f}s", + flush=True, + ) + time.sleep(args.oom_wait) + try: + parts = [ + _forward(model, reader, container, idx[i : i + size], device, use_cuda, args) + for i in range(0, len(idx), size) + ] + return np.concatenate(parts, axis=0) + except torch.cuda.OutOfMemoryError: + if size == 1: + continue + size = max(1, size // 2) + raise RuntimeError( + f"rows {idx[0]}..{idx[-1]} still OOM after {args.oom_retries} retries; " + f"lower --batch-size or wait for the GPU to free up" + ) + + +# ------------------------------------------------------------------------- main +def main(argv=None): + args = parse_args(argv) + repo = Path(args.gemnet_repo) + notes = ensure_imports(repo) + import torch + + torch.set_num_threads(args.num_threads) + set_deterministic(args.seed, tf32=args.tf32) + for note in notes: + print("[compat]", note) + print_versions() + print("[args]", json.dumps(vars(args))) + + pdir = Path(args.pretrained_dir) if args.pretrained_dir else repo / "pretrained" / "GemNet-Q" + mols = load_molecules(args.dataset, limit=args.limit or None, start=args.start) + lo = args.start + hi = lo + len(mols) + print(f"[data] {len(mols)} conformers (rows {lo}..{hi - 1}) from {args.dataset}") + + confs = [mol_to_arrays(m, args.hydrogens, args.max_atoms, args.round_coords) for m in mols] + bad = [lo + i for i, c in enumerate(confs) if c is None] + if bad: + raise RuntimeError(f"conformer(s) {bad} could not be featurised; refusing to silently shift the row order") + work = Path(args.work_dir) if args.work_dir else Path(args.out).resolve().parent + container, npz_path = build_container(confs, work, args.cutoff, args.int_cutoff, args.triplets_only) + print( + f"[graph] upstream DataContainer over {len(container)} conformers " + f"({sum(c['Z'].size for c in confs)} atoms), cutoff={args.cutoff} int_cutoff={args.int_cutoff}" + ) + + use_cuda = "cuda" in args.device and torch.cuda.is_available() + device = torch.device(args.device if use_cuda else "cpu") + model = load_model(pdir, device, args.triplets_only) + reader = AtomFeatureReader(model) + + out = None + t0 = time.time() + done = 0 + next_log = 0 + next_ckpt = args.checkpoint_every + for start in range(0, len(confs), args.batch_size): + idx = np.arange(start, min(start + args.batch_size, len(confs))) + emb = run_batch(model, reader, container, idx, device, use_cuda, args) + if out is None: + out = np.empty((len(confs), emb.shape[1]), dtype=np.float32) + out[start : start + len(idx)] = emb + done += len(idx) + if done >= next_log: + print(f"[run] {done}/{len(confs)} {done / max(time.time() - t0, 1e-9):.1f} conf/s", flush=True) + next_log += args.log_interval + if args.checkpoint_every and done >= next_ckpt: + np.savez(args.out, **{args.key: out[:done]}) + print(f"[ckpt] wrote {done} rows to {args.out}", flush=True) + next_ckpt += args.checkpoint_every + reader.close() + npz_path.unlink(missing_ok=True) + dt = time.time() - t0 + print(f"[run] done {done} conformers in {dt:.1f}s ({done / dt:.1f} conf/s)") + + Path(args.out).parent.mkdir(parents=True, exist_ok=True) + np.savez(args.out, **{args.key: out}) + print(f"[out] {args.out} key={args.key} shape={out.shape} dtype={out.dtype}") + print(f"[out] sha256={_common().sha256_file(args.out)}") + + if args.verify: + _common().verify( + args.out, + model=MODEL_NAME, + reference=args.verify, + produced_key=args.key, + reference_key=args.verify_key, + rows=args.verify_rows or (f"{args.start}+" if args.start else None), + ) + + +if __name__ == "__main__": + main() diff --git a/baselines/gemnet/requirements.txt b/baselines/gemnet/requirements.txt new file mode 100644 index 0000000..a94a547 --- /dev/null +++ b/baselines/gemnet/requirements.txt @@ -0,0 +1,13 @@ +# GemNet-Q chirality embedding extraction — pinned environment (python 3.10). +# torch and torch_scatter need their own index URLs, see README.md: +# pip install torch==2.1.0 --index-url https://download.pytorch.org/whl/cu121 +# pip install torch_scatter -f https://data.pyg.org/whl/torch-2.1.0+cu121.html +numpy==1.24.4 +scipy==1.10.1 +sympy==1.12 +numba==0.58.1 +rdkit==2025.03.5 +# only for --dataset hf:: or a save_to_disk directory +datasets>=2.14 +# only for --dataset lmdb: +lmdb>=1.4 diff --git a/baselines/mace/ENVIRONMENT.md b/baselines/mace/ENVIRONMENT.md new file mode 100644 index 0000000..b25d25a --- /dev/null +++ b/baselines/mace/ENVIRONMENT.md @@ -0,0 +1,89 @@ +# MACE — environment + +`extract_chirality.py` calls `mace-torch` directly. No third-party code or weights are +redistributed here: `mace_mp()` fetches the foundation model from the upstream GitHub +release and caches it under `~/.cache/mace/`, and the script reports what it used. + +## Upstream + +| item | source | pin | +|---|---|---| +| MACE | (MIT) | `mace-torch==0.3.15` on PyPI | + +## Weights + +`mace_mp()` downloads these; the sha256 below were computed from the downloaded files. + +| `--model` | file | URL | bytes | sha256 | +|---|---|---|---|---| +| `medium` (default) | `2023-12-03-mace-128-L1_epoch-199.model` | | 44,422,970 | `01bfe22100139f424713cf921144e5509cbe353d67aa9fa1be9c6e1e0ed35845` | +| `small` | `2023-12-10-mace-128-L0_energy_epoch-249.model` | | 32,581,838 | `2ddb079cee0e131eaaf6912ba581b394551ead283e95c99cfe78c605d10b5736` | +| `large` | `MACE_MPtrj_2022.9.model` | | 133,803,220 | `f80e992b65ab8f88fdf26964511357c022e92704e4d9bcd086652635a8495b32` | +| `medium-mpa-0` | `mace-mpa-0-medium.model` | | 79,462,305 | `75428afe3a1d7d8062e19bcaabd5c433623cabf308242ec9fb493e38604fb638` | + +`mace-torch` strips punctuation from the cached file name, e.g. +`~/.cache/mace/20231203mace128L1_epoch199model`. Descriptor width per model (invariants of +all interaction layers, concatenated): `small` 256, `medium` 256, `large` 512, +`medium-mpa-0` 256. The MACE-MP-0 weights are released by the MACE authors under the terms +stated in that repository; check them before redistributing. + +## Install + +```bash +conda create -y -p ./env python=3.11 +./env/bin/pip install "torch==2.5.1" --index-url https://download.pytorch.org/whl/cu124 +./env/bin/pip install -r requirements.txt +./env/bin/pip install datasets # only for --dataset hf:... / hfdisk:... +``` + +Resolved versions used for the numbers in [`../README.md`](../README.md): python 3.11, +torch 2.5.1+cu124, mace-torch 0.3.15, e3nn 0.4.4, ase 3.29.0, numpy 1.26.4, rdkit 2024.09.6, +scipy 1.17.1, opt-einsum 3.4.0, torchmetrics 1.9.0. Box driver 595.71.05. + +## Run + +```bash +python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --out chirality.npz \ + --device cuda --batch-size 1 --compress --verify +``` + +`--dataset` takes the input specification shared by every script in `baselines/` (see the +table in [`../README.md`](../README.md#running-one)): `hf:[:]`, `hfdisk:` or a plain +`save_to_disk` directory, a bare Hub dataset id, a pickle of RDKit molecules, or `lmdb:`. +`--limit`/`--start` run a slice of the conformers and `--verify-rows` says which rows of the +reference that slice covers. + +Other switches: `--model`, `--aggregation mean|sum`, `--num-layers`, `--full-features`, +`--conf-id`, `--dtype`. The script prints the versions of every numerically relevant package +and the sha256 of the file it wrote. +## Settings + +`mace_mp(model="medium")` (MACE-MP-0 medium, 128 channels, two interaction layers) -> +`get_descriptors(invariants_only=True, num_layers=-1)`, which concatenates the l = 0 part of +the node features of both layers (256 values per atom) -> mean over **all** atoms, hydrogens +included, conformer 0, float32 -> `np.savez_compressed` under `arr_0`, input row order kept. +Being a Materials-Project model it covers every element in the benchmark, including the +Se / Si / B / As / Na / K / Ca / Mg / Zn atoms that appear in 459 of the 52,391 conformers. + +`--batch-size 1` runs one conformer per forward pass and is the default. Larger batches are +faster but reorder the scatter reductions, so results can differ in the last float32 digits +(measured: at most 1.4e-7 on a 128-conformer sample, the same order as the difference +between CPU and GPU). float32 and float64 runs agree to ~1e-7, the float32 spacing at these +magnitudes. + +## Input precision + +The published embedding file and the agreement numbers in [`../README.md`](../README.md) were +computed from the source RDKit molecules, whose coordinates carry full float precision. The public +`EscheWang/3dcs` dataset stores those geometries as V2000 MOL blocks, which hold four decimals. Atom +order and symbols are identical either way and coordinates differ by at most 5e-05 A; embeddings +extracted through the Hugging Face path differ from the published file by at most 3.78e-05, per-row +cosine at least 0.9999998. MACE is the least sensitive of the seven to this rounding. + +## Cost + +52,391 conformers at `--batch-size 1`: 5,378 s on an A100 80GB shared with three other jobs +(11.8 GB of GPU memory), or about 24 minutes split over 24 CPU processes at +`--batch-size 16`. diff --git a/baselines/mace/extract_chirality.py b/baselines/mace/extract_chirality.py new file mode 100644 index 0000000..718c98a --- /dev/null +++ b/baselines/mace/extract_chirality.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python +"""Extract MACE descriptors for the 3DCS chirality benchmark. + +What this script computes +------------------------- +For every input conformer it runs a pretrained MACE foundation model, takes the +O(3)-invariant (l = 0) part of the node features of every interaction layer, +concatenates the per-layer invariants, and averages them over the atoms of the +molecule. With the default model (``mace_mp`` "medium", 128 channels, two +interaction layers) this yields a 256-dimensional vector per conformer: +``[layer-1 invariants (128) | layer-2 invariants (128)]``. + +All atoms are used, including hydrogens. Conformer 0 of every RDKit ``Mol`` is +used. The output ``.npz`` stores a single float32 array of shape +``(n_conformers, 256)`` under the key ``arr_0``, in the same row order as the +input, matching the layout of the released +``3dcs-embeddings/chirality/mace/chirality.npz``. + +Third-party dependencies (not redistributed here) +------------------------------------------------- + * mace-torch -- https://github.com/ACEsuit/mace (MIT). Pinned: 0.3.15 + * MACE-MP-0 "medium" weights, downloaded by ``mace`` itself from + https://github.com/ACEsuit/mace-mp/releases/download/mace_mp_0/2023-12-03-mace-128-L1_epoch-199.model + (cached as ``~/.cache/mace/20231203mace128L1_epoch199model``; + sha256 01bfe22100139f424713cf921144e5509cbe353d67aa9fa1be9c6e1e0ed35845) + * torch, e3nn, ase, rdkit, numpy -- see requirements.txt + +Example +------- + python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --out chirality.npz --device cuda --batch-size 1 --compress +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import logging +import sys +import time +from collections.abc import Sequence +from pathlib import Path + +import numpy as np + +LOGGER = logging.getLogger("extract_chirality") + +MODEL_NAME = "mace" +OUTPUT_KEY = "arr_0" + + +def _common(): + """Load ``baselines/common.py`` (verification helpers) without touching sys.path.""" + import importlib.util + import os + + path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "common.py") + spec = importlib.util.spec_from_file_location("baselines_common", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# --------------------------------------------------------------------------- # +# input +# --------------------------------------------------------------------------- # +def load_molecules(dataset: str, *, limit: int | None = None, start: int = 0): + """Return a list of RDKit Mol objects with 3D conformers, in benchmark row order. + + ``dataset`` takes the ``--dataset`` syntax shared by every script in ``baselines/`` + (see ``baselines/common.py``): ``hf:[:]``, ``hfdisk:`` or a plain + ``save_to_disk`` directory, a bare Hub dataset id, a pickle of RDKit molecules, or + ``lmdb:``. + + The Hugging Face rows store one ``mol_blocks`` list per stereoisomer plus the + ``offset`` of its first conformer in the embedding matrix, so concatenating + ``mol_blocks`` in ascending ``offset`` order reproduces the row order of the + released embedding files. MOL blocks carry coordinates with four decimals, + so geometries read this way differ from the pickle by up to 5e-5 A. MACE reads + element symbols and coordinates, which do not depend on RDKit sanitisation. + """ + return _common().load_conformers(dataset, limit=limit, start=start) + + +def mol_to_atoms(mol, conf_id: int): + from ase import Atoms + + if mol.GetNumConformers() <= conf_id: + raise ValueError(f"molecule has {mol.GetNumConformers()} conformers, need {conf_id + 1}") + conf = mol.GetConformer(conf_id) + return Atoms( + symbols=[atom.GetSymbol() for atom in mol.GetAtoms()], + positions=np.asarray(conf.GetPositions(), dtype=np.float64), + ) + + +# --------------------------------------------------------------------------- # +# model +# --------------------------------------------------------------------------- # +def build_calculator(model: str, device: str, dtype: str): + from mace.calculators import foundations_models + + return foundations_models.mace_mp(model=model, device=device, default_dtype=dtype, return_raw_model=False) + + +def aggregate(descriptors: np.ndarray, method: str) -> np.ndarray: + if method == "mean": + return descriptors.mean(axis=0) + if method == "sum": + return descriptors.sum(axis=0) + raise ValueError(f"unknown aggregation {method}") + + +def run_single(calc, mols, args) -> np.ndarray: + """One molecule per forward pass. This is the reference path.""" + out: list[np.ndarray] = [] + n = len(mols) + for idx, mol in enumerate(mols): + atoms = mol_to_atoms(mol, args.conf_id) + desc = calc.get_descriptors(atoms=atoms, invariants_only=not args.full_features, num_layers=args.num_layers) + desc = np.asarray(desc, dtype=np.float32) + out.append(np.nan_to_num(aggregate(desc, args.aggregation), copy=False)) + if args.log_every and (idx + 1) % args.log_every == 0: + LOGGER.info("… %d/%d", idx + 1, n) + return np.stack(out).astype(np.float32, copy=False) + + +def run_batched(calc, mols, args) -> np.ndarray: + """Several molecules per forward pass. + + Faster, but the scatter reductions run in a different order, so results can + differ from ``run_batched`` in the last float32 digits. ``--batch-size 1`` + is the reference path. + """ + import torch + from e3nn import o3 + from mace import data as mace_data + from mace.modules.utils import extract_invariant + from mace.tools import torch_geometric, utils + + model = calc.models[0] + z_table = utils.AtomicNumberTable([int(z) for z in model.atomic_numbers]) + r_max = float(model.r_max) + heads = getattr(model, "heads", ["Default"]) + num_interactions = int(model.num_interactions) + num_layers = num_interactions if args.num_layers == -1 else args.num_layers + irreps_out = o3.Irreps(str(model.products[0].linear.irreps_out)) + l_max = irreps_out.lmax + num_invariant_features = irreps_out.dim // (l_max + 1) ** 2 + + configs = [] + for mol in mols: + atoms = mol_to_atoms(mol, args.conf_id) + configs.append(mace_data.config_from_atoms(atoms)) + dataset = [mace_data.AtomicData.from_config(c, z_table=z_table, cutoff=r_max, heads=heads) for c in configs] + loader = torch_geometric.dataloader.DataLoader( + dataset=dataset, batch_size=args.batch_size, shuffle=False, drop_last=False + ) + out: list[np.ndarray] = [] + done = 0 + for batch in loader: + batch = batch.to(args.device) + res = model(batch.to_dict(), compute_force=False, compute_virials=False, compute_stress=False) + node_feats = res["node_feats"] + if args.full_features: + feats = node_feats + else: + feats = extract_invariant( + node_feats, num_layers=num_layers, num_features=num_invariant_features, l_max=l_max + ) + keep = num_invariant_features * num_layers + feats = feats[:, :keep] + idx = batch.batch + nb = int(idx.max().item()) + 1 + summed = torch.zeros(nb, feats.shape[1], dtype=feats.dtype, device=feats.device) + summed.index_add_(0, idx, feats) + if args.aggregation == "mean": + counts = torch.bincount(idx, minlength=nb).clamp(min=1).unsqueeze(1).to(feats.dtype) + pooled = summed / counts + else: + pooled = summed + arr = pooled.detach().cpu().numpy().astype(np.float32) + out.append(np.nan_to_num(arr, copy=False)) + done += arr.shape[0] + if args.log_every and done % args.log_every < args.batch_size: + LOGGER.info("… %d/%d", done, len(mols)) + return np.concatenate(out, axis=0).astype(np.float32, copy=False) + + +# --------------------------------------------------------------------------- # +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + common = _common() + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dataset", required=True, help=common.DATASET_SPEC_HELP) + p.add_argument("--out", required=True, type=Path, help="output .npz path") + p.add_argument( + "--model", + default="medium", + help="mace_mp model name: small | medium | large | medium-mpa-0 … (default: medium)", + ) + p.add_argument( + "--aggregation", + choices=("mean", "sum"), + default="mean", + help="how per-atom descriptors are pooled (default: mean)", + ) + p.add_argument( + "--num-layers", type=int, default=-1, help="number of interaction layers to keep, -1 = all (default: -1)" + ) + p.add_argument( + "--full-features", + action="store_true", + help="keep the full equivariant node features instead of the invariant part", + ) + p.add_argument("--conf-id", type=int, default=0, help="conformer id (default: 0)") + p.add_argument( + "--batch-size", type=int, default=1, help="molecules per forward pass; 1 is the reference path (default: 1)" + ) + p.add_argument("--device", default="cuda", help="torch device (default: cuda)") + p.add_argument("--dtype", choices=("float32", "float64"), default="float32") + p.add_argument("--limit", type=int, default=None, help="only process the first N molecules") + p.add_argument("--start", type=int, default=0, help="skip the first N molecules") + p.add_argument( + "--compress", action="store_true", help="write with np.savez_compressed (the released file is compressed)" + ) + p.add_argument("--log-every", type=int, default=2000) + p.add_argument("--seed", type=int, default=0) + p.add_argument( + "--verify", + nargs="?", + const="published", + default=None, + metavar="REFERENCE", + help="after writing, compare the output with a reference file: 'published' " + "(the published MACE chirality embedding), hub:, or a local path", + ) + p.add_argument( + "--verify-key", default=None, help="array key to read from the --verify reference (default: its published key)" + ) + p.add_argument("--verify-rows", default=None, metavar="ROWS", help=common.ROW_SELECTION_HELP) + return p.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + + import ase + import e3nn + import mace + import rdkit + import torch + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + torch.use_deterministic_algorithms(False) + torch.backends.cudnn.benchmark = False + + LOGGER.info("python %s", sys.version.split()[0]) + LOGGER.info( + "numpy %s | torch %s (cuda %s) | e3nn %s | mace-torch %s | ase %s | rdkit %s", + np.__version__, + torch.__version__, + torch.version.cuda, + e3nn.__version__, + mace.__version__, + ase.__version__, + rdkit.__version__, + ) + LOGGER.info("args: %s", json.dumps({k: str(v) for k, v in vars(args).items()}, sort_keys=True)) + + mols = load_molecules(args.dataset, limit=args.limit, start=args.start) + LOGGER.info("loaded %d molecules from %s", len(mols), args.dataset) + + t0 = time.time() + calc = build_calculator(args.model, args.device, args.dtype) + LOGGER.info("model ready in %.1fs", time.time() - t0) + + t0 = time.time() + arr = run_single(calc, mols, args) if args.batch_size == 1 else run_batched(calc, mols, args) + LOGGER.info("extracted %s in %.1fs", arr.shape, time.time() - t0) + + args.out.parent.mkdir(parents=True, exist_ok=True) + writer = np.savez_compressed if args.compress else np.savez + writer(args.out, arr) + digest = hashlib.sha256(args.out.read_bytes()).hexdigest() + LOGGER.info("wrote %s (%d bytes) sha256=%s", args.out, args.out.stat().st_size, digest) + + if args.verify: + _common().verify( + args.out, + model=MODEL_NAME, + reference=args.verify, + produced_key=OUTPUT_KEY, + reference_key=args.verify_key, + rows=args.verify_rows or (f"{args.start}+" if args.start else None), + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/baselines/mace/requirements.txt b/baselines/mace/requirements.txt new file mode 100644 index 0000000..d2e618c --- /dev/null +++ b/baselines/mace/requirements.txt @@ -0,0 +1,18 @@ +# Environment used to produce the MACE chirality embeddings. +# Python 3.11. Install torch first, from the CUDA 12.4 wheel index: +# +# conda create -y -p ./env python=3.11 +# ./env/bin/pip install "torch==2.5.1" --index-url https://download.pytorch.org/whl/cu124 +# ./env/bin/pip install -r requirements.txt +# +# mace-torch pulls e3nn==0.4.4, ase, opt-einsum, torchmetrics and friends. +torch==2.5.1 +mace-torch==0.3.15 +rdkit==2024.9.6 +numpy<2 +tqdm +# only for the Hugging Face --dataset forms (hf:[:], hfdisk:, a +# save_to_disk directory or a bare Hub id) +# datasets>=3.0 +# only for --verify +# huggingface_hub diff --git a/baselines/molae/ENVIRONMENT.md b/baselines/molae/ENVIRONMENT.md new file mode 100644 index 0000000..eef44d1 --- /dev/null +++ b/baselines/molae/ENVIRONMENT.md @@ -0,0 +1,142 @@ +# Mol-AE — environment + +`extract_chirality.py` is a wrapper around the Uni-Mol inference pipeline: it writes the +LMDB that `unimol.tasks.UniMolTask` reads, builds `unimol_base`, loads the Mol-AE +pre-trained checkpoint into it and keeps the `[CLS]` representation (`mol_repr_cls`). No +third-party code and no weights are redistributed here. + +## Upstream + +| component | source | pin | +|---|---|---| +| Uni-Mol (inference pipeline: `unimol` task, `unimol_infer` loss, `unimol_base` model, dataset classes) | | commit `90f52c41299a1a582da0f9765e9f87aa21faa16a` | +| Uni-Core (framework: options, tasks, collators, checkpoint loading) | | commit `ace6fae1c8479a9751f2bb1e1d6e4047427bc134` | +| Mol-AE (reference implementation of the pre-training architecture; not needed at inference time) | | commit `2992f2d5862e104745dce7f4af3ca58fe366a4b6` | + +## Weights and dictionary + +| field | value | +|---|---| +| file | `checkpoint_7_1000000.pt` (Mol-AE pre-training) | +| size | 760,699,523 bytes | +| sha256 | `b4ca21a63799976fbf435a1c7275d5ef6e93a854cc7d90955dbaec50ef89b8c0` | +| source | the Google Drive link in the Mol-AE README () | +| Drive file id | `1NKObZCfE80GCLS9yJ7hqMGzjfGol4LLo` | +| `args.arch` / `args.loss` recorded inside | `unimol_MAE_padding` / `unimol_MAE` | +| encoder / decoder | 15 layers, 512 embed, 2048 ffn, 64 heads / 5 layers, 2048 ffn, 64 heads | +| `remove_hydrogen` / `only_polar` / `dict_name` / `max_atoms` | `True` / `0` / `dict.txt` / 256 | +| `num_updates` | 1,000,000 | + +### Downloading it + +The Drive download is a **ZIP archive** that carries the name `checkpoint_7_1000000.pt` +(611,355,678 bytes, sha256 +`d226201c0f20cb5b2f38042412e9d1b366fcd45d62832b928d37ec555e834c63`). It holds one member, +also called `checkpoint_7_1000000.pt`, which is the 760,699,523-byte checkpoint in the table +above. Unpack it before `torch.load`: on the archive itself `torch.load` stops with +`file in archive is not in a subdirectory`. + +```bash +pip install gdown +gdown 1NKObZCfE80GCLS9yJ7hqMGzjfGol4LLo -O molae_download.zip # the file id form works on every gdown +unzip -o molae_download.zip -d molae_ckpt +sha256sum molae_ckpt/checkpoint_7_1000000.pt +# b4ca21a63799976fbf435a1c7275d5ef6e93a854cc7d90955dbaec50ef89b8c0 +``` + +`gdown --fuzzy ` needs a recent gdown; the bare file id above does not. + +The atom dictionary is Uni-Mol's `unimol/example_data/molecule/dict.txt`, sha256 +`94135cb9a9198f988de684cb61e2c372882a3bd59b8320effbae704c38057127` (30 symbols; `[MASK]` is +added by the task, giving 31). + +### How the encoder is loaded + +The checkpoint's parameter names are a superset of Uni-Mol `unimol_base`: `embed_tokens`, +`encoder.*`, `gbf` and `gbf_proj` match one for one, while `decoder.*`, `lm_head.*`, +`dist_head.*` and `pair2coord_proj.*` are extra. Uni-Mol's inference entry point loads with +`load_state_dict(..., strict=False)`, so the Mol-AE encoder is used and the MAE decoder and +pre-training heads are dropped. The script prints the counts: 0 missing, 82 unexpected. + +## Install + +Built on an A100-80GB, CUDA driver 595.71.05. + +```bash +conda create -y -p ./env python=3.10 +./env/bin/python -m pip install torch==2.4.1 --index-url https://download.pytorch.org/whl/cu121 +./env/bin/python -m pip install numpy==1.26.4 lmdb==1.4.1 scipy pandas tqdm \ + ml_collections tensorboardX iopath tokenizers==0.19.1 scikit-learn==1.5.2 rdkit==2025.3.6 +git clone https://github.com/dptech-corp/Uni-Core.git && git -C Uni-Core checkout ace6fae +./env/bin/python -m pip install --no-build-isolation ./Uni-Core # no CUDA extensions +git clone https://github.com/deepmodeling/Uni-Mol.git && git -C Uni-Mol checkout 90f52c4 +./env/bin/python -m pip install --no-deps -e ./Uni-Mol/unimol +``` + +Resolved versions printed by the script at run time: python 3.10.21, numpy 1.26.4, +torch 2.4.1+cu121 (cuda 12.1), rdkit 2025.03.6, lmdb 1.4.1, unicore 0.0.1. + +Uni-Core's fused CUDA kernels are optional (`setup.py` sets `DISABLE_CUDA_EXTENSION = True` +unless `--enable-cuda-ext` is passed). They were not built, so `fused_layer_norm`, +`fused_softmax`, `fused_rms_norm`, `fused_multi_tensor` and `fused_rounding` fall back to +their PyTorch implementations. + +RDKit is used only to read the input molecules and to canonicalise a per-row SMILES *name* +that never reaches the model. A version new enough to unpickle the released molecule +pickles is required (RDKit pickle version 16.2, i.e. RDKit >= 2024.09). + +## Run + +```bash +python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --weights checkpoint_7_1000000.pt \ + --unimol-dir /path/to/Uni-Mol/unimol/unimol \ + --dict /path/to/Uni-Mol/unimol/example_data/molecule/dict.txt \ + --out molae_chirality.npz \ + --batch-size 256 --device cuda:0 --num-workers 8 --verify +``` + +`--dataset` takes the input specification shared by every script in `baselines/` (see the +table in [`../README.md`](../README.md#running-one)): `hf:[:]`, `hfdisk:` or +a plain `save_to_disk` directory, a bare Hub dataset id, a pickle of RDKit molecules, or +`lmdb:`. The Hugging Face forms need `datasets` in the environment (the line is in +`requirements.txt`). `--limit`/`--start` run a slice of the conformers and `--verify-rows` +says which rows of the reference that slice covers: + +```bash +# the first 2,000 conformers, compared with the first 2,000 rows of the published file +python extract_chirality.py --dataset hf:EscheWang/3dcs:chirality --limit 2000 ... --verify +``` + +## Settings + +`infer.py --task unimol --loss unimol_infer --arch unimol_base --only-polar 0 --conf-size 1 +--random-token-prob 0 --leave-unmasked-prob 1.0 --mode infer`, output `mol_repr_cls` (the +512-d `[CLS]` vector), float32, no `--fp16`. `--leave-unmasked-prob 1.0` with +`--random-token-prob 0` makes `MaskPointsDataset` an identity (no mask, no coordinate +noise), and `valid_step` calls `model.eval()`, so the pipeline is deterministic up to +floating-point reduction order. + +Row order is preserved end to end: the script writes LMDB records under the ascii keys +`"0" … "N-1"`, `LMDBDataset.__getitem__` looks a record up by `str(idx)`, +`UnicoreDataset.ordered_indices` returns `arange(N)`, and `next_epoch_itr(shuffle=False)` +keeps the batches sequential. + +## Input precision + +The published embedding file and the agreement numbers in [`../README.md`](../README.md) were +computed from the source RDKit molecules, whose coordinates carry full float precision. The public +`EscheWang/3dcs` dataset stores those geometries as V2000 MOL blocks, which hold four decimals. +Over the first 2,000 conformers, a run from `hf:EscheWang/3dcs:chirality` and a run from the +full-precision molecules, same machine and same environment, differ by `max|diff| = 4.4e-01` and +`mean|diff| = 3.2e-04`, per-row cosine mean 0.9999948 and minimum 0.9921: 56 of the 2,000 rows +(2.8 %) are below 1 - 1e-6 and one is below 0.999. The `[CLS]` vector reacts to the rounding +through the pair-distance encoding, so this is the larger of the two effects for Mol-AE — against +the published file the same run gives per-row cosine mean 0.9999948 and minimum 0.9921, i.e. the +same numbers. Start from a pickle of the molecules for the closest agreement. + +## Cost + +84 s for 52,391 conformers on one A100-80GB (~624 conformers/s), under 2 GB of GPU memory at +`--batch-size 256`. diff --git a/baselines/molae/extract_chirality.py b/baselines/molae/extract_chirality.py new file mode 100644 index 0000000..c16664f --- /dev/null +++ b/baselines/molae/extract_chirality.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +"""Extract Mol-AE (MolAE) molecule-level embeddings for the 3DCS chirality benchmark. + +What this script computes +------------------------- +For every input conformer it returns the 512-d encoder representation of the +``[CLS]`` token of the Mol-AE encoder, evaluated with the Uni-Mol inference +pipeline (``--task unimol --loss unimol_infer --arch unimol_base --mode infer``). + +The Mol-AE pre-trained checkpoint (``arch: unimol_MAE_padding``) contains a +15-layer / 512-d encoder whose parameter names and shapes are identical to +Uni-Mol ``unimol_base`` plus an extra 5-layer MAE decoder. The decoder is not +used at inference time: the checkpoint is loaded with ``strict=False`` into a +``unimol_base`` model, so only the encoder / embedding / GBF parameters are +consumed. The script prints the missing / unexpected key counts so the load is +auditable. + +Pre-processing (inherited unchanged from upstream Uni-Mol): + * all hydrogens removed (``--only-polar 0`` -> ``remove_hydrogen=True``) + * molecules cropped to at most ``--max-atoms`` heavy atoms + * coordinates centred (mean subtracted); no rotation, no scaling + * ``--random-token-prob 0 --leave-unmasked-prob 1.0`` makes ``MaskPointsDataset`` + a deterministic identity: no token is replaced and no coordinate noise is added + * ``[CLS]`` prepended, ``[SEP]`` appended, both with coordinate (0, 0, 0) + * ``--conf-size 1``: one conformer per input record, no test-time augmentation + +Output: an ``.npz`` written with ``numpy.savez(out, embeddings)``, i.e. a single +array under the key ``arr_0`` of shape ``(n_conformers, 512)``, float32, in the +same order as the input records. + +Third-party code and weights are NOT redistributed with this script. See +``ENVIRONMENT.md`` for the upstream repositories, commits and the checkpoint the +script expects. + +Example +------- + python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --weights checkpoint_7_1000000.pt \ + --unimol-dir /path/to/Uni-Mol/unimol/unimol \ + --dict /path/to/Uni-Mol/unimol/example_data/molecule/dict.txt \ + --out molae_chirality.npz \ + --batch-size 256 --device cuda:0 +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import pickle +import random +import shutil +import sys +import tempfile +import time + +import numpy as np + + +# --------------------------------------------------------------------------- # +# input handling +# --------------------------------------------------------------------------- # +def _common(): + """Load ``baselines/common.py`` (input and verification helpers) without touching sys.path.""" + import importlib.util + + path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "common.py") + spec = importlib.util.spec_from_file_location("baselines_common", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_molecules( + spec: str, + hf_split: str = "train", + hf_config: str = "chirality", + *, + limit: int | None = None, + start: int = 0, +): + """Return a list of RDKit molecules, in benchmark row order. + + ``spec`` takes the ``--dataset`` syntax shared by every script in ``baselines/`` + (see ``baselines/common.py``): ``hf:[:]``, ``hfdisk:`` or a plain + ``save_to_disk`` directory, a bare Hub dataset id, a pickle of RDKit molecules (a list, + or a dict of lists flattened in insertion order), or ``lmdb:``. + + The ``chirality`` config of ``EscheWang/3dcs`` stores one ``mol_blocks`` list per + stereoisomer -- one MDL MOL block per conformer -- and the ``offset`` of the first of + them, so reading ``mol_blocks`` in ascending ``offset`` gives the 52,391 conformers in + the row order of the published embedding file. Mol-AE reads element symbols and + coordinates only. + """ + return _common().load_conformers(spec, hf_config=hf_config, hf_split=hf_split, limit=limit, start=start) + + +def build_lmdb(mols, lmdb_path: str) -> int: + """Write the molecules in the Uni-Mol inference LMDB format. + + Each record is ``{"atoms": [symbol, ...], "coordinates": [array(n, 3), ...], + "smi": canonical_smiles}`` stored under the ascii key ``str(index)``. + ``LMDBDataset.__getitem__`` looks records up by ``str(idx)``, so the row order + of the output is exactly the order of ``mols``. + """ + import lmdb + from rdkit import Chem + + env = lmdb.open( + lmdb_path, + subdir=False, + readonly=False, + lock=False, + readahead=False, + meminit=False, + max_readers=1, + map_size=int(100e9), + ) + txn = env.begin(write=True) + n = 0 + for idx, mol in enumerate(mols): + atoms = [atom.GetSymbol() for atom in mol.GetAtoms()] + coordinates = [conf.GetPositions().astype(np.float32) for conf in mol.GetConformers()] + if not coordinates: + raise ValueError(f"molecule {idx} has no conformer") + # `smi` is only a per-row name carried through to the logging output; it + # never enters the model input, so a molecule RDKit refuses to + # re-sanitise still yields the correct embedding. + try: + smi = Chem.MolToSmiles(Chem.RemoveHs(mol)) + except Exception: + try: + smi = Chem.MolToSmiles(mol) + except Exception: + smi = f"row{idx}" + txn.put( + f"{idx}".encode("ascii"), + pickle.dumps({"atoms": atoms, "coordinates": coordinates, "smi": smi}, protocol=-1), + ) + n += 1 + if n % 20000 == 0: + txn.commit() + txn = env.begin(write=True) + txn.commit() + env.close() + return n + + +# --------------------------------------------------------------------------- # +# extraction +# --------------------------------------------------------------------------- # +def extract(args) -> np.ndarray: + import torch + from unicore import checkpoint_utils, options, tasks, utils + + parser = options.get_validation_parser() + options.add_model_args(parser) + argv = [ + args.data_dir, + "--user-dir", + args.unimol_dir, + "--valid-subset", + args.subset, + "--results-path", + args.data_dir, + "--num-workers", + str(args.num_workers), + "--ddp-backend", + "c10d", + "--batch-size", + str(args.batch_size), + "--task", + "unimol", + "--loss", + "unimol_infer", + "--arch", + "unimol_base", + "--path", + args.weights, + "--only-polar", + "0", + "--dict-name", + os.path.basename(args.dict), + "--conf-size", + str(args.conf_size), + "--max-atoms", + str(args.max_atoms), + "--log-interval", + "50", + "--log-format", + "simple", + "--random-token-prob", + "0", + "--leave-unmasked-prob", + "1.0", + "--mode", + "infer", + "--seed", + str(args.seed), + ] + model_args = options.parse_args_and_arch(parser, argv) + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + random.seed(args.seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + if args.tf32 is False: + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False + + device = torch.device(args.device) + state = checkpoint_utils.load_checkpoint_to_cpu(model_args.path) + ckpt_arch = getattr(state.get("args", None), "arch", None) + print(f"[info] checkpoint arch recorded in file: {ckpt_arch}") + + task = tasks.setup_task(model_args) + model = task.build_model(model_args) + incompatible = model.load_state_dict(state["model"], strict=False) + print( + f"[info] load_state_dict(strict=False): " + f"{len(incompatible.missing_keys)} missing, " + f"{len(incompatible.unexpected_keys)} unexpected" + ) + if incompatible.missing_keys: + print(f"[info] missing (sample): {incompatible.missing_keys[:8]}") + if incompatible.unexpected_keys: + prefixes = sorted({k.split(".")[0] for k in incompatible.unexpected_keys}) + print(f"[info] unexpected key prefixes (unused): {prefixes}") + model = model.to(device) + if args.fp16: + model = model.half() + model.eval() + + task.load_dataset(args.subset, combine=False, epoch=1) + dataset = task.dataset(args.subset) + itr = task.get_batch_iterator( + dataset=dataset, + batch_size=model_args.batch_size, + ignore_invalid_inputs=True, + required_batch_size_multiple=model_args.required_batch_size_multiple, + seed=model_args.seed, + num_shards=1, + shard_id=0, + num_workers=model_args.num_workers, + data_buffer_size=model_args.data_buffer_size, + ).next_epoch_itr(shuffle=False) + + reps = [] + names = [] + t0 = time.time() + n_done = 0 + with torch.no_grad(): + for sample in itr: + if len(sample) == 0: + continue + sample = utils.move_to_cuda(sample) if device.type == "cuda" else sample + encoder_rep, _ = model(**sample["net_input"], features_only=True) + if args.pooling == "cls": + pooled = encoder_rep[:, 0, :] + else: + tokens = sample["net_input"]["src_tokens"] + pad = task.dictionary.pad() + bos = task.dictionary.bos() + eos = task.dictionary.eos() + mask = (tokens.ne(pad) & tokens.ne(bos) & tokens.ne(eos)).unsqueeze(-1).to(encoder_rep.dtype) + if args.pooling == "mean": + pooled = (encoder_rep * mask).sum(1) / mask.sum(1).clamp(min=1) + elif args.pooling == "sum": + pooled = (encoder_rep * mask).sum(1) + else: + raise ValueError(args.pooling) + reps.append(pooled.float().data.cpu().numpy()) + names.extend(sample["target"]["smi_name"]) + n_done += pooled.shape[0] + if n_done % (args.batch_size * 20) == 0: + print(f"[info] {n_done} conformers, {time.time() - t0:.1f}s", flush=True) + + embeddings = np.concatenate(reps, axis=0).astype(np.float32) + print( + f"[info] done: {embeddings.shape} in {time.time() - t0:.1f}s " + f"({embeddings.shape[0] / max(time.time() - t0, 1e-9):.1f} conf/s)" + ) + if args.names_out: + np.save(args.names_out, np.array(names, dtype=object), allow_pickle=True) + return embeddings + + +# --------------------------------------------------------------------------- # +MODEL_NAME = "molae" +OUTPUT_KEY = "arr_0" + + +def sha256(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def main() -> int: + common = _common() + ap = argparse.ArgumentParser(description="Mol-AE [CLS] embedding extraction for the 3DCS chirality set") + ap.add_argument("--dataset", required=True, help=common.DATASET_SPEC_HELP) + ap.add_argument("--hf-config", default="chirality", help="config to read when --dataset names a Hub dataset") + ap.add_argument("--hf-split", default="train") + ap.add_argument("--limit", type=int, default=None, help="only process the first N conformers") + ap.add_argument("--start", type=int, default=0, help="skip the first N conformers") + ap.add_argument("--out", required=True, help="output .npz (key arr_0)") + ap.add_argument("--weights", required=True, help="Mol-AE pre-trained checkpoint") + ap.add_argument( + "--unimol-dir", + required=True, + help="path to the Uni-Mol 'unimol' python package (used as --user-dir)", + ) + ap.add_argument("--dict", required=True, help="path to Uni-Mol dict.txt") + ap.add_argument("--batch-size", type=int, default=256) + ap.add_argument("--device", default="cuda:0") + ap.add_argument("--num-workers", type=int, default=8) + ap.add_argument("--conf-size", type=int, default=1) + ap.add_argument("--max-atoms", type=int, default=256) + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--pooling", default="cls", choices=["cls", "mean", "sum"]) + ap.add_argument("--fp16", action="store_true", help="run the model in half precision") + ap.add_argument( + "--tf32", + action="store_true", + help="allow TF32 matmuls (off by default for reproducibility)", + ) + ap.add_argument( + "--work-dir", + default=None, + help="where the intermediate LMDB is written (default: a temp dir)", + ) + ap.add_argument("--keep-work-dir", action="store_true") + ap.add_argument("--names-out", default=None, help="optional .npy of per-row SMILES") + ap.add_argument( + "--verify", + nargs="?", + const="published", + default=None, + metavar="REFERENCE", + help="after writing, compare the output with a reference file: 'published' " + "(the published Mol-AE chirality embedding), hub:, or a local path", + ) + ap.add_argument( + "--verify-key", default=None, help="array key to read from the --verify reference (default: its published key)" + ) + ap.add_argument("--verify-rows", default=None, metavar="ROWS", help=common.ROW_SELECTION_HELP) + args = ap.parse_args() + + work_dir = args.work_dir or tempfile.mkdtemp(prefix="molae_extract_") + os.makedirs(work_dir, exist_ok=True) + args.data_dir = work_dir + args.subset = "molae_input" + + shutil.copyfile(args.dict, os.path.join(work_dir, os.path.basename(args.dict))) + + lmdb_path = os.path.join(work_dir, args.subset + ".lmdb") + if not os.path.exists(lmdb_path): + mols = load_molecules(args.dataset, args.hf_split, args.hf_config, limit=args.limit, start=args.start) + n = build_lmdb(mols, lmdb_path) + print(f"[info] wrote {n} records to {lmdb_path}") + else: + print(f"[info] reusing existing LMDB {lmdb_path}") + + # versions + import lmdb as lmdb_mod + import rdkit + import torch + import unicore + + print(f"[versions] python {sys.version.split()[0]}") + print(f"[versions] numpy {np.__version__}") + print(f"[versions] torch {torch.__version__} (cuda {torch.version.cuda})") + print(f"[versions] rdkit {rdkit.__version__}") + print(f"[versions] lmdb {lmdb_mod.__version__}") + print(f"[versions] unicore {unicore.__version__}") + print(f"[versions] weights {os.path.basename(args.weights)}") + print(f"[versions] weights sha {sha256(args.weights)}") + print(f"[versions] dict sha {sha256(args.dict)}") + + embeddings = extract(args) + np.savez(args.out, embeddings) + print(f"[info] wrote {args.out} shape={embeddings.shape} dtype={embeddings.dtype}") + print(f"[info] output sha256 {sha256(args.out)}") + + if args.verify: + common.verify( + args.out, + model=MODEL_NAME, + reference=args.verify, + produced_key=OUTPUT_KEY, + reference_key=args.verify_key, + rows=args.verify_rows or (f"{args.start}+" if args.start else None), + ) + + if not args.keep_work_dir and args.work_dir is None: + shutil.rmtree(work_dir, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/baselines/molae/requirements.txt b/baselines/molae/requirements.txt new file mode 100644 index 0000000..9cb18c6 --- /dev/null +++ b/baselines/molae/requirements.txt @@ -0,0 +1,25 @@ +# Environment used to produce the Mol-AE chirality embeddings (python 3.10). +# Install torch first, from the CUDA 12.1 wheel index, then Uni-Core and Uni-Mol from the +# two upstream checkouts pinned in ENVIRONMENT.md: +# +# conda create -y -p ./env python=3.10 +# ./env/bin/python -m pip install torch==2.4.1 --index-url https://download.pytorch.org/whl/cu121 +# ./env/bin/python -m pip install -r requirements.txt +# ./env/bin/python -m pip install --no-build-isolation ./Uni-Core +# ./env/bin/python -m pip install --no-deps -e ./Uni-Mol/unimol +numpy==1.26.4 +lmdb==1.4.1 +tokenizers==0.19.1 +scikit-learn==1.5.2 +rdkit==2025.3.6 +scipy +pandas +tqdm +ml_collections +tensorboardX +iopath +# only for the Hugging Face --dataset forms (hf:[:], hfdisk:, a +# save_to_disk directory or a bare Hub id) +# datasets +# only for --verify +# huggingface_hub diff --git a/baselines/molspectra/ENVIRONMENT.md b/baselines/molspectra/ENVIRONMENT.md new file mode 100644 index 0000000..58930b6 --- /dev/null +++ b/baselines/molspectra/ENVIRONMENT.md @@ -0,0 +1,109 @@ +# MolSpectra — environment + +`extract_chirality.py` needs (a) a clone of the upstream MolSpectra repository, (b) an +equivariant-Transformer checkpoint with `embedding_dimension = 256`, and (c) the environment +below. No third-party code or weights are redistributed here. + +The MolSpectra authors publish their QM9S dataset but no pre-trained checkpoint, so the +weights have to be supplied by the caller. The output depends entirely on which checkpoint is +used: with a checkpoint of your own this script produces a MolSpectra-architecture embedding, +not a copy of the published file. The published +`chirality/molspectra/sampled_mol_feature.npz` in +[`EscheWang/3dcs-embeddings`](https://huggingface.co/datasets/EscheWang/3dcs-embeddings) is +the file the paper's MolSpectra values were computed from. + +## Upstream + +| item | source | pin | +|---|---|---| +| MolSpectra reference implementation | | commit `8846530e573a1dc2834eab738d20278b0c90e601` (2025-04-19, `main`) | +| architecture | TorchMD-NET equivariant Transformer, as forked by MolSpectra | — | +| checkpoint used for the runs reported in [`../README.md`](../README.md) | `checkpoints/denoised-pcqm4mv2.ckpt` of | sha256 `f6b387ba3632e03d273939257969161b2615a7fbf41e9f514fd25e1b5d345e66`, 86,684,181 B, epoch 25 / step 400000 | + +That checkpoint's `hyper_parameters` are the architecture MolSpectra pre-trains with: +`model=equivariant-transformer`, `embedding_dimension=256`, `num_layers=8`, `num_rbf=64`, +`rbf_type=expnorm`, `trainable_rbf=False`, `cutoff_lower=0.0`, `cutoff_upper=5.0`, +`max_z=100`, `max_num_neighbors=32`, `num_heads=8`, `distance_influence=both`, +`activation=silu`, `attn_activation=silu`, `neighbor_embedding=True`, +`layernorm_on_vec=whitened`, `reduce_op=add`. + +## Install + +```bash +uv venv --python 3.10 ./env +source ./env/bin/activate +uv pip install torch==2.3.1 --index-url https://download.pytorch.org/whl/cu121 +uv pip install torch_scatter==2.1.2 torch_cluster==1.6.3 \ + -f https://data.pyg.org/whl/torch-2.3.1+cu121.html +uv pip install -r requirements.txt +git clone https://github.com/AzureLeon1/MolSpectra.git upstream +git -C upstream checkout 8846530e573a1dc2834eab738d20278b0c90e601 +``` + +Verified versions on an A100 80GB (driver 595.71.05): python 3.10.21, torch 2.3.1+cu121, +torch_scatter 2.1.2+pt23cu121, torch_cluster 1.6.3+pt23cu121, torch_geometric 2.6.1, +numpy 1.26.3, rdkit 2024.03.5. + +Differences from the upstream `requirements.txt`, and why: + +* `pytorch_lightning==2.3.3` instead of `1.3.8`. PL 1.3.8 pins `pyyaml<=5.4.1`, which no + longer builds against Cython 3. PL is only needed to unpickle a Lightning checkpoint; the + extraction instantiates `torchmdnet.models.torchmd_et.TorchMD_ET` directly and never + touches the Lightning `LightningModule`. +* `setuptools<81` is required by `lightning_fabric`, which still calls `pkg_resources`. +* `rdkit`, `scipy` and `scikit-learn` are not in the upstream `requirements.txt`; they are + needed to read the conformers and to run the 3DCS evaluator. + +## Run + +```bash +python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --repo ./upstream \ + --checkpoint ./denoised-pcqm4mv2.ckpt \ + --out molspectra_chirality.npz \ + --batch-size 128 --device cuda \ + --arch torchmdnet --hydrogens remove --pool add --verify +``` + +`--dataset` takes the input specification shared by every script in `baselines/` (see the +table in [`../README.md`](../README.md#running-one)): `hf:[:]`, `hfdisk:` or a plain +`save_to_disk` directory, a bare Hub dataset id, a pickle of RDKit molecules, or `lmdb:`. +`--limit`/`--start` run a slice of the conformers and `--verify-rows` says which rows of the +reference that slice covers. +## What the flags mean + +* `--arch molspectra` builds MolSpectra's `TorchMD_ET`, which inserts a per-layer + `x_norms` / `vec_norms` pair that upstream TorchMD-NET does not have. A checkpoint that + predates MolSpectra has no weights for those 24 tensors and they stay at LayerNorm init, + which changes the function. `--arch torchmdnet` disables them (`use_dataset_md17=True`), so + an upstream checkpoint loads with 0 missing tensors. Use `molspectra` with a MolSpectra + checkpoint and `torchmdnet` with an upstream one; the script prints the missing-key count, + so a mismatch is visible. +* `--hydrogens remove` (default) drops every atom with `Z == 1` before the forward pass; + `keep` feeds the conformer as stored. The chirality conformers carry explicit hydrogens + (75.0 atoms per conformer, 38.6 of them heavy). +* `--pool add` (default) is MolSpectra's `reduce_op: add`. + +## Determinism + +The configuration is fixed by the CLI flags, but the GPU forward pass is not bit-reproducible: +`scatter(..., reduce="add")` and the attention reductions use atomics, so two runs of the +identical command differ in the last float32 bits. Measured over two runs of the same +command: largest absolute element difference 2.7e-3 on values up to 377 (~7e-6 relative to +the row norm), per-row cosine at least 0.999999999. Use `--device cpu` for a bit-reproducible +output. + +## Input precision + +The published embedding file and the agreement numbers in [`../README.md`](../README.md) were +computed from the source RDKit molecules, whose coordinates carry full float precision. The public +`EscheWang/3dcs` dataset stores those geometries as V2000 MOL blocks, which hold four decimals. For +a model with a 5 A cutoff that rounding can add or drop an edge: 1.19 % of rows have a per-row +cosine below 0.9999 between a run from the MOL blocks and a run from the full-precision molecules. +This applies to every cutoff-graph model here. + +## Cost + +119 s for 52,391 conformers on an otherwise idle A100 80GB (~440 conformers/s), about 290 s +when the GPU is shared. Under 3 GB of GPU memory at `--batch-size 128`. diff --git a/baselines/molspectra/extract_chirality.py b/baselines/molspectra/extract_chirality.py new file mode 100644 index 0000000..9a01cc6 --- /dev/null +++ b/baselines/molspectra/extract_chirality.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python +"""Extract MolSpectra-style molecule-level embeddings for the 3DCS chirality set. + +The released 3DCS file ``chirality/molspectra/sampled_mol_feature.npz`` holds one +256-d vector per conformer. In the MolSpectra reference implementation +(https://github.com/AzureLeon1/MolSpectra, an equivariant-Transformer / +TorchMD-NET fork) that quantity is produced inside ``TorchMD_Net.forward`` as:: + + x, v, z, pos, batch = self.representation_model(z, pos, batch=batch) + mol_feature = scatter(x, batch, dim=0, reduce=self.reduce_op) # reduce_op = "add" + +i.e. the per-atom scalar representation of the equivariant Transformer, summed +over the atoms of each molecule. This script computes exactly that quantity. +It does not vendor any third-party code or weights: point --repo at a local +clone of the upstream MolSpectra repository and --checkpoint at a TorchMD-NET +equivariant-Transformer checkpoint (256-d, 8 layers). + +The output depends entirely on which checkpoint is supplied. The MolSpectra +authors release their QM9S dataset but no pre-trained checkpoint, so the weights +have to be supplied by the caller; with a different checkpoint this script +produces a valid MolSpectra-architecture embedding, not a copy of the released +3DCS file. --arch records which of the two encoder variants is in use and the +script prints how many checkpoint tensors were missing, so a mismatch is +visible in the log. + +Usage +----- + python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --repo /path/to/MolSpectra \ + --checkpoint /path/to/et-256.ckpt \ + --out molspectra_chirality.npz \ + --batch-size 128 --device cuda + +Outputs an .npz with key ``arr_0`` of shape (n_conformers, 256), float32, rows +in the order of the input dataset. +""" + +import argparse +import hashlib +import os +import sys +import time + +MODEL_NAME = "molspectra" + + +def _common(): + """Load ``baselines/common.py`` (verification helpers) without touching sys.path.""" + import importlib.util + import os + + path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "common.py") + spec = importlib.util.spec_from_file_location("baselines_common", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def parse_args(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dataset", required=True, help=_common().DATASET_SPEC_HELP) + p.add_argument("--repo", required=True, help="Local clone of https://github.com/AzureLeon1/MolSpectra") + p.add_argument("--checkpoint", required=True, help="TorchMD-NET equivariant-Transformer checkpoint (.ckpt).") + p.add_argument("--out", required=True, help="Output .npz path.") + p.add_argument("--batch-size", type=int, default=128, help="Molecules per forward pass (default: 128).") + p.add_argument("--device", default="cuda", help="cuda | cuda:N | cpu (default: cuda).") + p.add_argument("--key", default="arr_0", help="npz key to write (default: arr_0).") + # --- recipe knobs ------------------------------------------------------- + p.add_argument( + "--hydrogens", + choices=["remove", "keep"], + default="remove", + help="Whether explicit hydrogens are fed to the encoder. " + "remove (default) drops every atom with Z == 1, i.e. the encoder " + "sees heavy atoms only; keep feeds the molecule as stored.", + ) + p.add_argument( + "--pool", + choices=["add", "mean", "max"], + default="add", + help="Pooling of per-atom features (default: add, = MolSpectra reduce_op).", + ) + p.add_argument( + "--arch", + choices=["molspectra", "torchmdnet"], + default="molspectra", + help="molspectra: MolSpectra's ET, with the per-layer x_norms / " + "vec_norms it adds (use_dataset_md17=False). " + "torchmdnet: those extra norms disabled (use_dataset_md17=True), " + "which is the plain upstream TorchMD-NET / denoising ET.", + ) + p.add_argument( + "--layernorm-on-vec", + default="whitened", + choices=["whitened", "none"], + help="Equivariant output layer norm (default: whitened).", + ) + p.add_argument("--normalize", action="store_true", help="L2-normalise each output row (off by default).") + p.add_argument("--dtype", choices=["float32", "float64"], default="float32") + p.add_argument("--seed", type=int, default=0) + p.add_argument( + "--verify", + nargs="?", + const="published", + default=None, + metavar="REFERENCE", + help="after writing, compare the output with a reference file: 'published' " + "(the published MolSpectra chirality embedding), hub:, or a local path", + ) + p.add_argument( + "--verify-key", default=None, help="array key to read from the --verify reference (default: its published key)" + ) + p.add_argument("--verify-rows", default=None, metavar="ROWS", help=_common().ROW_SELECTION_HELP) + p.add_argument("--limit", type=int, default=None, help="only process the first N conformers") + p.add_argument("--start", type=int, default=0, help="skip the first N conformers") + return p.parse_args() + + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def load_molecules(spec, *, limit=None, start=0): + """Return a flat list of RDKit Mol in the canonical 3DCS row order. + + ``spec`` takes the ``--dataset`` syntax shared by every script in ``baselines/`` + (see ``baselines/common.py``): ``hf:[:]``, ``hfdisk:`` or a plain + ``save_to_disk`` directory, a bare Hub dataset id, a pickle of RDKit molecules (a list, + or a dict of lists flattened in insertion order), or ``lmdb:``. MolSpectra reads + atomic numbers and coordinates only. + """ + return _common().load_conformers(spec, limit=limit, start=start) + + +def main(): + args = parse_args() + + import numpy as np + import torch + from rdkit import RDLogger + + RDLogger.DisableLog("rdApp.*") + + sys.path.insert(0, os.path.abspath(args.repo)) + import rdkit + import torch_cluster + import torch_geometric + from torch_scatter import scatter + from torchmdnet.models.torchmd_et import TorchMD_ET + + torch.manual_seed(args.seed) + np.random.seed(args.seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.use_deterministic_algorithms(False) # scatter-add has no deterministic kernel + + print("# ---- versions ----", flush=True) + print(f"# python {sys.version.split()[0]}") + print(f"# torch {torch.__version__} (cuda {torch.version.cuda})") + print(f"# torch_scatter {getattr(__import__('torch_scatter'), '__version__', '?')}") + print(f"# torch_cluster {torch_cluster.__version__}") + print(f"# torch_geometric {torch_geometric.__version__}") + print(f"# numpy {np.__version__}") + print(f"# rdkit {rdkit.__version__}") + print(f"# repo {os.path.abspath(args.repo)}") + print(f"# checkpoint {args.checkpoint}") + print(f"# checkpoint sha256 {sha256(args.checkpoint)}") + print(f"# dataset {args.dataset}") + if os.path.isfile(args.dataset): + print(f"# dataset sha256 {sha256(args.dataset)}") + print( + f"# recipe arch={args.arch} hydrogens={args.hydrogens} " + f"pool={args.pool} layernorm_on_vec={args.layernorm_on_vec} " + f"normalize={args.normalize} dtype={args.dtype}" + ) + + # ---- model ------------------------------------------------------------ + ckpt = torch.load(args.checkpoint, map_location="cpu", weights_only=False) + hp = dict(ckpt.get("hyper_parameters", {})) + + def h(k, default): + v = hp.get(k, default) + return default if v is None else v + + model = TorchMD_ET( + hidden_channels=h("embedding_dimension", 256), + num_layers=h("num_layers", 8), + num_rbf=h("num_rbf", 64), + rbf_type=h("rbf_type", "expnorm"), + trainable_rbf=h("trainable_rbf", False), + activation=h("activation", "silu"), + attn_activation=h("attn_activation", "silu"), + neighbor_embedding=h("neighbor_embedding", True), + num_heads=h("num_heads", 8), + distance_influence=h("distance_influence", "both"), + cutoff_lower=h("cutoff_lower", 0.0), + cutoff_upper=h("cutoff_upper", 5.0), + max_z=h("max_z", 100), + max_num_neighbors=h("max_num_neighbors", 32), + layernorm_on_vec=None if args.layernorm_on_vec == "none" else args.layernorm_on_vec, + use_dataset_md17=(args.arch == "torchmdnet"), + ) + + # the checkpoint stores the whole TorchMD_Net; keep the representation trunk only + prefix = "model.representation_model." + sd = {k[len(prefix) :]: v for k, v in ckpt["state_dict"].items() if k.startswith(prefix)} + if not sd: + prefix = "representation_model." + sd = {k[len(prefix) :]: v for k, v in ckpt["state_dict"].items() if k.startswith(prefix)} + ret = model.load_state_dict(sd, strict=False) + print(f"# loaded {len(sd)} tensors; missing={len(ret.missing_keys)} unexpected={len(ret.unexpected_keys)}") + if ret.missing_keys: + print(f"# missing (left at init): {sorted(set(k.split('.')[0] for k in ret.missing_keys))}") + if ret.unexpected_keys: + print(f"# unexpected: {ret.unexpected_keys[:8]}") + if ret.missing_keys and args.arch == "molspectra": + print( + "# WARNING: this checkpoint has no weights for MolSpectra's per-layer " + "x_norms / vec_norms, so they stay at LayerNorm init and the encoder is not " + "the one the checkpoint was trained as. Pass --arch torchmdnet for an " + "upstream TorchMD-NET checkpoint; --arch molspectra needs a MolSpectra one.", + flush=True, + ) + + device = torch.device(args.device if torch.cuda.is_available() or args.device == "cpu" else "cpu") + dtype = torch.float32 if args.dtype == "float32" else torch.float64 + model = model.to(device=device, dtype=dtype).eval() + + # ---- data ------------------------------------------------------------- + mols = load_molecules(args.dataset, limit=args.limit, start=args.start) + n = len(mols) + print(f"# conformers {n}", flush=True) + + zs, poss = [], [] + for mol in mols: + z = np.array([a.GetAtomicNum() for a in mol.GetAtoms()], dtype=np.int64) + xyz = np.asarray(mol.GetConformer().GetPositions(), dtype=np.float64) + if args.hydrogens == "remove": + keep = z != 1 + z, xyz = z[keep], xyz[keep] + zs.append(z) + poss.append(xyz) + n_atoms_fed = int(sum(len(z) for z in zs)) + print(f"# atoms fed {n_atoms_fed} ({n_atoms_fed / max(len(mols), 1):.2f} per conformer)") + + out = np.zeros((n, model.hidden_channels), dtype=np.float32) + t0 = time.time() + with torch.no_grad(): + for start in range(0, n, args.batch_size): + stop = min(start + args.batch_size, n) + zb = np.concatenate(zs[start:stop]) + pb = np.concatenate(poss[start:stop], axis=0) + bb = np.concatenate([np.full(len(zs[i]), i - start, dtype=np.int64) for i in range(start, stop)]) + z = torch.from_numpy(zb).to(device) + pos = torch.from_numpy(pb).to(device=device, dtype=dtype) + batch = torch.from_numpy(bb).to(device) + x, _v, _z, _pos, _batch = model(z, pos, batch) + feat = scatter(x, batch, dim=0, dim_size=stop - start, reduce=args.pool) + if args.normalize: + feat = feat / feat.norm(dim=1, keepdim=True).clamp_min(1e-12) + out[start:stop] = feat.float().cpu().numpy() + if (start // args.batch_size) % 50 == 0: + print(f"# {stop}/{n} {time.time() - t0:.1f}s", flush=True) + dt = time.time() - t0 + print(f"# forward wall time {dt:.1f}s ({n / dt:.1f} conformers/s)") + + os.makedirs(os.path.dirname(os.path.abspath(args.out)) or ".", exist_ok=True) + np.savez(args.out, **{args.key: out}) + print(f"# wrote {args.out} shape={out.shape} sha256={sha256(args.out)}") + + if args.verify: + _common().verify( + args.out, + model=MODEL_NAME, + reference=args.verify, + produced_key=args.key, + reference_key=args.verify_key, + rows=args.verify_rows or (f"{args.start}+" if args.start else None), + ) + + +if __name__ == "__main__": + main() diff --git a/baselines/molspectra/requirements.txt b/baselines/molspectra/requirements.txt new file mode 100644 index 0000000..7a8e12e --- /dev/null +++ b/baselines/molspectra/requirements.txt @@ -0,0 +1,21 @@ +# Environment used to run the MolSpectra extraction (python 3.10). +# torch and the two PyG extensions need their own index URLs, see ENVIRONMENT.md: +# uv pip install torch==2.3.1 --index-url https://download.pytorch.org/whl/cu121 +# uv pip install torch_scatter==2.1.2 torch_cluster==1.6.3 \ +# -f https://data.pyg.org/whl/torch-2.3.1+cu121.html +torch_geometric==2.6.1 +pytorch_lightning==2.3.3 +ase==3.23.0 +h5py==3.11.0 +numpy==1.26.3 +PyYAML>=6.0 +tqdm==4.66.5 +rdkit==2024.3.5 +scipy +scikit-learn +setuptools<81 +# only for the Hugging Face --dataset forms (hf:[:], hfdisk:, a +# save_to_disk directory or a bare Hub id) +# datasets +# only for --verify +# huggingface_hub diff --git a/baselines/unimol/ENVIRONMENT.md b/baselines/unimol/ENVIRONMENT.md new file mode 100644 index 0000000..486d1a1 --- /dev/null +++ b/baselines/unimol/ENVIRONMENT.md @@ -0,0 +1,99 @@ +# Uni-Mol — environment + +`extract_chirality.py` drives an unmodified Uni-Mol checkout: it writes the LMDB that +`unimol.tasks.UniMolTask` reads, parses the same command line that `unimol/infer.py` parses, +and runs the encoder forward pass, keeping the `[CLS]` row. No third-party code and no model +weights are redistributed here. + +## Upstream + +| item | source | pin | +|---|---|---| +| Uni-Core | | commit `ace6fae1c8479a9751f2bb1e1d6e4047427bc134` | +| Uni-Mol | | commit `90f52c41299a1a582da0f9765e9f87aa21faa16a`, the `unimol/` project | + +## Weights and dictionary + +| file | source | sha256 | +|---|---|---| +| `mol_pre_no_h_220816.pt` (190,540,187 bytes) | | `da27196af09a8c6d089e10b7764b6a716bcc33da227fc118f5b45b0e484585e9` | +| `dict.txt` | `Uni-Mol/unimol/example_data/molecule/dict.txt` of the checkout | `94135cb9a9198f988de684cb61e2c372882a3bd59b8320effbae704c38057127` | + +## Install + +Tested on Linux x86-64, one NVIDIA A100 80GB, CUDA 12.x driver. + +```bash +conda create -y -p ./env python=3.10 +./env/bin/python -m pip install --upgrade pip setuptools wheel +./env/bin/python -m pip install torch==2.5.1 --index-url https://download.pytorch.org/whl/cu124 +./env/bin/python -m pip install \ + "numpy==1.26.4" "pandas==2.2.3" "scipy==1.14.1" "rdkit==2024.03.6" "lmdb==1.5.1" \ + tqdm ml_collections tensorboardX tokenizers iopath scikit-learn +./env/bin/python -m pip install datasets # only for --dataset hf:... / a save_to_disk dir + +git clone https://github.com/dptech-corp/Uni-Core.git +git -C Uni-Core checkout ace6fae1c8479a9751f2bb1e1d6e4047427bc134 +./env/bin/python -m pip install --no-build-isolation --no-deps ./Uni-Core + +git clone https://github.com/deepmodeling/Uni-Mol.git +git -C Uni-Mol checkout 90f52c41299a1a582da0f9765e9f87aa21faa16a +./env/bin/python -m pip install --no-build-isolation --no-deps -e ./Uni-Mol/unimol + +curl -L -o mol_pre_no_h_220816.pt \ + https://github.com/deepmodeling/Uni-Mol/releases/download/v0.1/mol_pre_no_h_220816.pt +``` + +`Uni-Core`'s `setup.py` builds its fused CUDA kernels only with `--enable-cuda-ext`. The +install above uses the PyTorch reference implementations of layer norm and softmax-dropout, +which is what the numbers in [`../README.md`](../README.md) were produced with. The full +resolved environment is in `requirements.txt`. + +## Run + +```bash +python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --unimol-repo /path/to/Uni-Mol/unimol \ + --weights /path/to/mol_pre_no_h_220816.pt \ + --out chirality_unimol.npz \ + --batch-size 256 --device cuda:0 --verify +``` + +`--dataset` takes the input specification shared by every script in `baselines/` (see the +table in [`../README.md`](../README.md#running-one)): `hf:[:]`, `hfdisk:` or a plain +`save_to_disk` directory, a bare Hub dataset id, a pickle of RDKit molecules, or `lmdb:`. +`--limit`/`--start` run a slice of the conformers and `--verify-rows` says which rows of the +reference that slice covers. + +Other options: `--dict`, `--num-workers`, `--seed`, `--work-dir`, `--keep-work`. + +The script prints the versions of Python, PyTorch, Uni-Core, NumPy, RDKit and LMDB, the GPU +name, the sha256 of the weights and of `dict.txt`, the full Uni-Mol command line it parses, +and the shape and sha256 of the output. +## Settings + +The command line the script parses is +`--only-polar 0 --conf-size 1 --random-token-prob 0 --leave-unmasked-prob 1.0 --mode infer` +on `unimol_base` with `mol_pre_no_h_220816.pt`, fp32, no `--fp16`, and the embedding is +`encoder_rep[:, 0, :]` (the `[CLS]` token of the last encoder layer) with no normalisation. +With `--random-token-prob 0 --leave-unmasked-prob 1.0`, Uni-Mol's `MaskPointsDataset` +computes `num_mask == 0`, so no token is replaced and no coordinate noise is added, and the +pipeline draws no effective randomness: `--seed` does not change the output. `LMDBDataset` +looks records up by `str(idx)` and the iterator runs with `shuffle=False`, so the row order +of the input is preserved. + +## Input precision + +The published embedding file and the agreement numbers in [`../README.md`](../README.md) were +computed from the source RDKit molecules, whose coordinates carry full float precision. The public +`EscheWang/3dcs` dataset stores those geometries as V2000 MOL blocks, which hold four decimals. +Running the script on the full-precision molecules instead of the MOL blocks gives a mean per-row +cosine of 0.9999994 between the two runs, with 62 conformers of 52,391 below 0.9999 and a worst +case of 0.9945 (a 65-atom molecule); each chirality metric moves by less than 0.0002. + +## Cost + +52,391 conformers on one A100 80GB at `--batch-size 256`: 64 s for the forward pass from a +pickle input, 159 s when the input is the Hugging Face config (the MOL blocks are parsed by +RDKit first), plus about a minute to write the LMDB. diff --git a/baselines/unimol/extract_chirality.py b/baselines/unimol/extract_chirality.py new file mode 100644 index 0000000..6f8a6d7 --- /dev/null +++ b/baselines/unimol/extract_chirality.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Extract Uni-Mol molecule embeddings for the 3DCS chirality benchmark. + +What this computes +------------------ +For every conformer of the benchmark, in dataset order, the 512-d representation +of the ``[CLS]`` token produced by the pretrained Uni-Mol molecular encoder +(``unimol_base``, checkpoint ``mol_pre_no_h_220816.pt``, hydrogens removed, one +conformer per input, no masking and no coordinate noise). The output is a +``.npz`` holding a single ``(n_conformers, 512)`` float32 array under the key +``arr_0``. + +The Uni-Mol data pipeline is used unmodified: this script builds the LMDB that +``unimol.tasks.UniMolTask`` expects, parses the same command line that +``unimol/infer.py`` parses, and then runs the encoder forward pass itself so that +only the ``[CLS]`` row is kept (``infer.py`` additionally materialises the +per-atom and per-pair representations, which for 52k conformers is hundreds of +gigabytes of intermediate pickle). + +Requirements (third-party code and weights are NOT shipped with this script) +--------------------------------------------------------------------------- + * Uni-Core https://github.com/dptech-corp/Uni-Core + * Uni-Mol https://github.com/deepmodeling/Uni-Mol (the ``unimol/`` project) + * weights mol_pre_no_h_220816.pt, from the Uni-Mol release assets + * dict.txt Uni-Mol/unimol/example_data/molecule/dict.txt (shipped with Uni-Mol) +See README_unimol.md for exact versions and install commands. + +Usage +----- + python extract_chirality.py \ + --dataset hf:EscheWang/3dcs:chirality \ + --unimol-repo /path/to/Uni-Mol/unimol \ + --weights /path/to/mol_pre_no_h_220816.pt \ + --out chirality_unimol.npz + +``--dataset`` takes the syntax shared by every script in ``baselines/``: +``hf:EscheWang/3dcs:chirality``, ``hfdisk:`` or a plain ``save_to_disk`` +directory, a bare Hub dataset id, or a pickle holding RDKit molecules (a list, or +a dict whose values are lists; dict values are concatenated in insertion order). +""" + +from __future__ import annotations + +import argparse +import hashlib +import pickle +import shutil +import sys +import tempfile +import time +from pathlib import Path + +import numpy as np + +CONF_SIZE = 1 # one conformer per LMDB record; TTADataset takes index 0 +MAX_ATOMS = 256 # Uni-Mol default; the chirality set has <= 112 heavy atoms +DEFAULT_BATCH_SIZE = 256 + + +# --------------------------------------------------------------------------- # +# inputs +# --------------------------------------------------------------------------- # +def load_molecules(spec: str, *, limit: int | None = None, start: int = 0): + """Return the flat, dataset-ordered list of RDKit molecules. + + ``spec`` takes the ``--dataset`` syntax shared by every script in ``baselines/`` + (see ``baselines/common.py``): ``hf:[:]``, ``hfdisk:`` or a plain + ``save_to_disk`` directory, a bare Hub dataset id, a pickle of RDKit molecules, or + ``lmdb:``. Uni-Mol reads element symbols and coordinates only. + """ + mols = _common().load_conformers(spec, limit=limit, start=start) + print(f"[data] {len(mols)} conformers from {spec}") + return mols + + +def build_lmdb(mols, lmdb_path: Path) -> int: + """Write the records ``unimol.data.LMDBDataset`` reads, keyed by row index. + + Record: {"atoms": [element symbols, hydrogens included], + "coordinates": [float32 (n_atoms, 3) array], + "smi": canonical SMILES (a name; it never enters the model input)} + Hydrogens are stripped later by the Uni-Mol pipeline (``--only-polar 0``). + """ + import lmdb + from rdkit import Chem + + env = lmdb.open( + str(lmdb_path), + subdir=False, + readonly=False, + lock=False, + readahead=False, + meminit=False, + max_readers=1, + map_size=int(100e9), + ) + txn = env.begin(write=True) + for i, mol in enumerate(mols): + atoms = [a.GetSymbol() for a in mol.GetAtoms()] + coords = [c.GetPositions().astype(np.float32) for c in mol.GetConformers()] + if not coords: + raise ValueError(f"molecule {i} has no conformer") + try: + smi = Chem.MolToSmiles(Chem.RemoveHs(mol)) + except Exception: + smi = Chem.MolToSmiles(Chem.RemoveHs(mol, sanitize=False), canonical=False) + txn.put(f"{i}".encode("ascii"), pickle.dumps({"atoms": atoms, "coordinates": coords, "smi": smi}, protocol=-1)) + if (i + 1) % 20000 == 0: + txn.commit() + txn = env.begin(write=True) + txn.commit() + env.close() + return len(mols) + + +# --------------------------------------------------------------------------- # +# model +# --------------------------------------------------------------------------- # +def unimol_cli( + data_dir: Path, subset: str, user_dir: Path, weights: Path, batch_size: int, num_workers: int, seed: int +): + """The Uni-Mol inference command line, as a list of arguments. + + ``--only-polar 0`` removes every hydrogen (the checkpoint is the no-hydrogen + model); ``--conf-size 1`` takes the single stored conformer; + ``--random-token-prob 0`` together with ``--leave-unmasked-prob 1.0`` turns + Uni-Mol's masking into a no-op, so no token is replaced and no coordinate + noise is added. + """ + return [ + str(data_dir), + "--user-dir", + str(user_dir), + "--valid-subset", + subset, + "--results-path", + str(data_dir), + "--num-workers", + str(num_workers), + "--ddp-backend=c10d", + "--batch-size", + str(batch_size), + "--task", + "unimol", + "--loss", + "unimol_infer", + "--arch", + "unimol_base", + "--path", + str(weights), + "--only-polar", + "0", + "--dict-name", + "dict.txt", + "--conf-size", + str(CONF_SIZE), + "--max-atoms", + str(MAX_ATOMS), + "--log-interval", + "50", + "--log-format", + "simple", + "--random-token-prob", + "0", + "--leave-unmasked-prob", + "1.0", + "--mode", + "infer", + "--seed", + str(seed), + ] + + +def run_encoder(argv, subset: str, device: str): + import torch + from unicore import checkpoint_utils, options, tasks, utils + + parser = options.get_validation_parser() + options.add_model_args(parser) + args = options.parse_args_and_arch(parser, input_args=argv) + + state = checkpoint_utils.load_checkpoint_to_cpu(args.path) + task = tasks.setup_task(args) + model = task.build_model(args) + missing, _ = model.load_state_dict(state["model"], strict=False) + if missing: + print(f"[model] parameters not in the checkpoint: {sorted(missing)}") + use_cuda = device != "cpu" + if use_cuda: + torch.cuda.set_device(torch.device(device).index or 0) + model.cuda() + model.eval() + + task.load_dataset(subset, combine=False, epoch=1) + dataset = task.dataset(subset) + itr = task.get_batch_iterator( + dataset=dataset, + batch_size=args.batch_size, + ignore_invalid_inputs=True, + required_batch_size_multiple=args.required_batch_size_multiple, + seed=args.seed, + num_shards=1, + shard_id=0, + num_workers=args.num_workers, + data_buffer_size=args.data_buffer_size, + ).next_epoch_itr(shuffle=False) + + chunks, n_done, t0 = [], 0, time.time() + with torch.no_grad(): + for sample in itr: + if len(sample) == 0: + continue + if use_cuda: + sample = utils.move_to_cuda(sample) + # This is the tensor UniMolInferLoss records as "mol_repr_cls". + encoder_rep, _ = model(**sample["net_input"], features_only=True) + chunks.append(encoder_rep[:, 0, :].float().cpu().numpy()) + n_done += chunks[-1].shape[0] + if len(chunks) % 50 == 0: + print(f"[infer] {n_done} conformers, {time.time() - t0:.0f}s", flush=True) + out = np.concatenate(chunks, axis=0).astype(np.float32) + print(f"[infer] {out.shape[0]} conformers in {time.time() - t0:.0f}s") + return out + + +# --------------------------------------------------------------------------- # +MODEL_NAME = "unimol" +OUTPUT_KEY = "arr_0" + + +def _common(): + """Load ``baselines/common.py`` (verification helpers) without touching sys.path.""" + import importlib.util + import os + + path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "common.py") + spec = importlib.util.spec_from_file_location("baselines_common", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as fh: + for block in iter(lambda: fh.read(1 << 22), b""): + h.update(block) + return h.hexdigest() + + +def print_versions(weights: Path): + import lmdb + import rdkit + import torch + import unicore + + print(f"[env] python {sys.version.split()[0]}") + print(f"[env] torch {torch.__version__} (cuda {torch.version.cuda})") + print("[env] unicore {}".format(getattr(unicore, "__version__", "unknown"))) + print(f"[env] numpy {np.__version__}") + print(f"[env] rdkit {rdkit.__version__}") + print(f"[env] lmdb {lmdb.__version__}") + if torch.cuda.is_available(): + print(f"[env] gpu {torch.cuda.get_device_name(0)}") + print(f"[env] weights {weights.name} sha256={sha256(weights)}") + + +def main() -> int: + common = _common() + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--dataset", required=True, help=common.DATASET_SPEC_HELP) + ap.add_argument("--out", required=True, help="output .npz (single array under key arr_0)") + ap.add_argument("--unimol-repo", required=True, help="the unimol/ project directory of a Uni-Mol checkout") + ap.add_argument("--weights", required=True, help="mol_pre_no_h_220816.pt") + ap.add_argument("--dict", default=None, help="dict.txt (default: /example_data/molecule/dict.txt)") + ap.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE) + ap.add_argument("--device", default="cuda:0", help="cuda:N or cpu") + ap.add_argument("--num-workers", type=int, default=8) + ap.add_argument( + "--seed", + type=int, + default=1, + help="Uni-Core seed; the inference pipeline is deterministic, so it does not " + "change the output (masking is disabled and one conformer is stored)", + ) + ap.add_argument("--limit", type=int, default=None, help="only process the first N conformers") + ap.add_argument("--start", type=int, default=0, help="skip the first N conformers") + ap.add_argument("--work-dir", default=None, help="scratch directory for the LMDB") + ap.add_argument("--keep-work", action="store_true") + ap.add_argument( + "--verify", + nargs="?", + const="published", + default=None, + metavar="REFERENCE", + help="after writing, compare the output with a reference file: 'published' " + "(the published Uni-Mol chirality embedding), hub:, or a local path", + ) + ap.add_argument( + "--verify-key", default=None, help="array key to read from the --verify reference (default: its published key)" + ) + ap.add_argument("--verify-rows", default=None, metavar="ROWS", help=common.ROW_SELECTION_HELP) + a = ap.parse_args() + + repo = Path(a.unimol_repo).resolve() + weights = Path(a.weights).resolve() + dict_txt = Path(a.dict).resolve() if a.dict else repo / "example_data" / "molecule" / "dict.txt" + for p, what in ((repo / "unimol", "Uni-Mol package"), (weights, "weights"), (dict_txt, "dict.txt")): + if not p.exists(): + sys.exit(f"{what} not found: {p}") + sys.path.insert(0, str(repo)) + + print_versions(weights) + print(f"[env] dict.txt {dict_txt} sha256={sha256(dict_txt)}") + + work = Path(a.work_dir).resolve() if a.work_dir else Path(tempfile.mkdtemp(prefix="unimol_chi_")) + data_dir = work / "data" + data_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(dict_txt, data_dir / "dict.txt") + subset = "mols" + + try: + mols = load_molecules(a.dataset, limit=a.limit, start=a.start) + n = build_lmdb(mols, data_dir / (subset + ".lmdb")) + del mols + lmdb_name = data_dir / (subset + ".lmdb") + print(f"[data] wrote {n} records to {lmdb_name}") + + argv = unimol_cli(data_dir, subset, repo / "unimol", weights, a.batch_size, a.num_workers, a.seed) + print("[run ] " + " ".join(argv)) + emb = run_encoder(argv, subset, a.device) + if emb.shape[0] != n: + sys.exit(f"got {emb.shape[0]} rows for {n} inputs") + + out = Path(a.out).resolve() + out.parent.mkdir(parents=True, exist_ok=True) + np.savez(out, emb) + print(f"[out ] {out} shape={emb.shape} dtype={emb.dtype} key=arr_0") + print(f"[out ] sha256={sha256(out)}") + + if a.verify: + common.verify( + out, + model=MODEL_NAME, + reference=a.verify, + produced_key=OUTPUT_KEY, + reference_key=a.verify_key, + rows=a.verify_rows or (f"{a.start}+" if a.start else None), + ) + finally: + if not a.keep_work and a.work_dir is None: + shutil.rmtree(work, ignore_errors=True) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/baselines/unimol/requirements.txt b/baselines/unimol/requirements.txt new file mode 100644 index 0000000..f6579d2 --- /dev/null +++ b/baselines/unimol/requirements.txt @@ -0,0 +1,78 @@ +# Resolved environment used to produce the Uni-Mol chirality embeddings. +# See ENVIRONMENT.md for the install commands; unicore and unimol are installed from +# the two upstream checkouts pinned there, not from PyPI. +# +absl-py==2.5.0 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +anyio==4.15.1 +async-timeout==5.0.1 +attrs==26.1.0 +certifi==2026.7.22 +charset-normalizer==3.5.1 +click==8.5.0 +cloudpickle==3.1.2 +datasets==5.0.1 +dill==0.3.8 +exceptiongroup==1.3.1 +filelock==3.32.3 +frozenlist==1.8.0 +fsspec==2024.9.0 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.32.0 +idna==3.20 +iopath==0.1.10 +Jinja2==3.1.6 +joblib==1.6.0 +lmdb==1.5.1 +MarkupSafe==3.0.3 +ml_collections==1.1.0 +mpmath==1.3.0 +multidict==6.8.0 +multiprocess==0.70.16 +networkx==3.4.2 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +pandas==2.2.3 +pillow==12.3.0 +portalocker==4.3.2 +propcache==0.5.4 +protobuf==7.36.2 +pyarrow==25.0.1 +python-dateutil==2.9.0.post0 +pytz==2026.3.post1 +PyYAML==6.0.3 +rdkit==2024.3.6 +requests==2.34.2 +scikit-learn==1.7.2 +scipy==1.14.1 +six==1.17.0 +sympy==1.13.1 +tensorboardX==2.6.5 +threadpoolctl==3.7.0 +tokenizers==0.23.2 +tomli==2.4.1 +torch==2.5.1+cu124 +tqdm==4.70.1 +triton==3.1.0 +typing_extensions==4.16.0 +tzdata==2026.4 +urllib3==2.8.0 +xxhash==4.0.1 +yarl==1.25.1 +packaging diff --git a/docs/EMBEDDINGS.md b/docs/EMBEDDINGS.md index 0f1cc56..2351593 100644 --- a/docs/EMBEDDINGS.md +++ b/docs/EMBEDDINGS.md @@ -45,10 +45,16 @@ for shard in range(16): vectors = [] for row in part: for block in row["mol_blocks"]: - vectors.append(your_model(mol_from_block(block))) # shape (dim,) + vectors.append(encode(mol_from_block(block))) # your model, shape (dim,) np.savez(f"embeddings/my_model/rotation_conformers_{shard}.npz", arr_0=np.stack(vectors)) ``` +[`baselines/`](../baselines/README.md) holds a worked version of this for each of the seven +baseline models on the chirality set: input handling, the model call and the write, with the +environment each one needs. All seven read the dataset through the same `--dataset` syntax +(`hf:[:]`, `hfdisk:` or a `save_to_disk` directory, a bare Hub id, a pickle of +RDKit molecules, or `lmdb:`). + Check alignment before evaluating: for flat arrays the number of rows must equal the sum of `n_conformers`; for by-shard files, each file must have `max(offset + n_conformers)` rows of its shard. @@ -79,20 +85,31 @@ Rotation embeddings for E3FP, UniMol, MolAE and MolSpectra are not available. Fo per-molecule metric outputs of the original runs (`results/rotation/metrics_all_0.1_1.json.gz`, `results/rotation/metrics_sup_100.json.gz`) are published instead. -### Extraction status - -The extraction scripts used for the paper are not part of this repository. What is known about each -set of embeddings: +### What each published file contains -| Model | Dimension | Status | What is known | -|---|---|---|---| -| E3FP | 1024 bits | known | `e3fp` 1.2.7, `fprints_from_mol(mol, fprint_params=dict(bits=1024, level=5, radius_multiplier=1.5, stereo=True, include_disconnected=True, rdkit_invariants=True, first=1, counts=False))`, hydrogens kept. Recomputing from the original RDKit molecules reproduces 3,000/3,000 sampled chirality fingerprints; starting from the HF MolBlocks, about 5 % of fingerprints differ. | -| GemNet (GemNet-Q) | 128 | partially known | A GemNet implementation in the authors' files returns 128-d molecule embeddings by averaging final-layer atom features, but it postdates the published embeddings and its use for these files is not confirmed. The GemNet-Q weights are not released. | -| UniMol | 512 | unknown | Output format only. | -| MolAE | 512 | unknown | Output format only. | -| MolSpectra | 256 | unknown | Output format only. | -| MACE | 256 | unknown | Output format only. | -| FMG | 128 | unknown | Output format only (third-party model: Dumitrescu et al., ICLR 2025). | - -Embeddings produced with a re-implemented extractor may differ from these files; compare against the -published files before using them to reproduce the tables. +| Model | Dimension | Notes | +|---|---|---| +| E3FP | 1024 bits | `e3fp` 1.2.7, `fprints_from_mol(mol, fprint_params=dict(bits=1024, level=5, radius_multiplier=1.5, stereo=True, include_disconnected=True, rdkit_invariants=True, first=1, counts=False))`, hydrogens kept. Recomputing with these parameters from the original RDKit molecules reproduces all 52,391 chirality fingerprints bit for bit; starting from the HF MolBlocks, about 5 % of fingerprints differ. | +| GemNet (GemNet-Q) | 128 | one 128-d vector per conformer, key `gemnet`. The GemNet-Q weights are not part of this release. | +| UniMol | 512 | one 512-d vector per conformer, key `arr_0`. | +| MolAE | 512 | one 512-d vector per conformer, key `arr_0`. | +| MolSpectra | 256 | one 256-d vector per conformer, key `arr_0`. | +| MACE | 256 | one 256-d vector per conformer, key `arr_0`. | +| FMG | 128 | one 128-d vector per conformer, key `embeddings` (third-party model: Dumitrescu et al., ICLR 2025). | + +The reference values in [`reproduce/`](../reproduce/README.md) are computed from these files. +Embeddings produced with a different extractor are evaluated the same way, but their values are not +expected to match these reference values. + +### Extracting these embeddings + +[`baselines//extract_chirality.py`](../baselines/README.md) computes the chirality embedding +of each model from the dataset, in the same row order and under the same array key as the file +above. Each script drives an upstream checkout that you install yourself — no third-party code or +weights are redistributed — and `baselines//ENVIRONMENT.md` gives the upstream repository and +commit, the weight file with its SHA-256 and where to download it, and the exact install commands. +`--verify` compares a freshly extracted file with the published one and prints the checksums, the +elementwise differences and the per-row cosine similarity — over the rows the run covers, so a run +with `--limit` on a slice is compared with the matching rows of the published file; +[`baselines/README.md`](../baselines/README.md) tabulates those numbers for a full run of every +script. diff --git a/docs/METRICS.md b/docs/METRICS.md index 3c98108..a4b8829 100644 --- a/docs/METRICS.md +++ b/docs/METRICS.md @@ -3,11 +3,10 @@ 3DCS evaluates a molecular representation along three axes. Each evaluator supports two metric versions: -- `--metric-version paper` (default): the definitions that produced the published numbers. Where - the implementation used for the paper differs from the text of the paper, this version follows - the implementation, so that the tables can be regenerated. -- `--metric-version v2`: definitions that follow the text of the paper (or correct a degenerate - case), with the rationale given on each page. +- `--metric-version paper` (default): the definitions used for the published tables, so that those + tables can be regenerated. +- `--metric-version v2`: an alternative set of definitions (full-precision inputs, separate kernel + bandwidths, neighbour and population choices), described metric by metric on each page. | Axis | Dataset (`EscheWang/3dcs` config) | Paper tables | Definitions | |---|---|---|---| diff --git a/docs/USAGE.md b/docs/USAGE.md index 448c591..2e0f2eb 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -87,7 +87,7 @@ Options (see [metrics/energy.md](metrics/energy.md)): | `--metric-version {paper,v2}` | `paper` | metric definitions | | `--molecules` | all | subset of rMD17 molecules | | `--energy-precision-check {error,warn,ignore}` | `error` | action when the energies look float32-quantized | -| `--time-ordered` | off | treat the frames as a time series (enables the `v2` TS / smoothness metrics; rMD17 frames are not ordered) | +| `--time-ordered` | off | treat the frames as a time series (enables the `v2` TS / smoothness metrics; rMD17 frames are not stored in time order) | | `--legacy-traj-len` | 100000 | trajectory length assumed by the `legacy` window scheme | | `--block-size` | 4096 | block size of the pairwise distance computation | | `--n-jobs` | 1 | worker processes (`-1` = all CPUs) | @@ -125,11 +125,11 @@ Options: | `--extra-metrics` | off | also distance correlation, Mantel, stress, triplet order (slow) | | `--shards` | all shards with embeddings | shard ids to evaluate | | `--molecule-list FILE` | – | evaluate only these keys (one per line, `.gz` allowed), e.g. `reproduce/table1_geometry/sampled_molecules_seed2027.txt` | -| `--sample-ratio R --sample-seed S` | – , 2027 | per-shard random sample (`default_rng(S + shard)`); does not regenerate the published 10 % sample | +| `--sample-ratio R --sample-seed S` | – , 2027 | per-shard random sample (`default_rng(S + shard)`); for the molecules of the published 10 % sample use `--molecule-list` | | `--min-conformers` | 2 | skip molecules with fewer conformers | | `--max-keys` | – | evaluate at most N molecules (testing) | | `--n-jobs` | 1 | worker processes (Linux, fork) | -| `--replicate-offset-drift` | off | by-shard only: also compute `__offset_drift` columns with the embedding shift of the published full run (Table 1 LIE@k / AS) | +| `--replicate-offset-drift` | off | by-shard only: also compute `__offset_drift` columns with the embedding indexing of the original full run (Table 1 LIE@k / AS) | Outputs: `_per_key.parquet` (one row per molecule and distance space), `summary.csv` (mean, median, number of finite values per metric) and `config.json` (definitions, selection counts, diff --git a/docs/metrics/chirality.md b/docs/metrics/chirality.md index 7a43b00..d4e93b8 100644 --- a/docs/metrics/chirality.md +++ b/docs/metrics/chirality.md @@ -1,7 +1,7 @@ # Chirality metrics (Table 2) This page documents how `python -m three_dbench evaluate chirality` computes the zero-shot chirality -metrics, which settings produced the published Table 2, and what the `v2` definitions change. +metrics, which settings produced the published Table 2, and what the `v2` definitions compute. - Code: `src/three_dbench/chirality/evaluation.py` (metrics, per-molecule loop) and `src/three_dbench/benchmarks/chirality.py` (dataset loading, validation, output files). @@ -63,32 +63,30 @@ Molecule populations in the published protocol (`paper`, identical for all 7 mod | `cosine` | `Delta_ij = 1 - / ((||z_i|| + 1e-12)(||z_j|| + 1e-12))` in float64, with the diagonal set to exactly 0 | `cosine_distances` | | RDKit fingerprints | Tanimoto distance `1 - |F_i & F_j| / |F_i | F_j|` (RDKit `BulkTanimotoSimilarity`), whatever `--distance` is | `tanimoto_distance_matrix` | -- **The published Table 2 was computed with Euclidean distance.** The paper text (§4.1 and App. C.2) - says cosine distance is the default for learned representations. The released evaluator - (`chirality/evaluation.py`, `distance_matrix_for_subset`) used Euclidean distance. Recomputing - Table 2 with Euclidean distance matches all published cells to within 0.001, except for the sign of - GemNet SCI (section 7). Cosine distance does not match: for example, MolAE SCI is 0.189 against the - printed 0.115, and GemNet ES-AUC is 0.602 against 0.577. -- App. C.2 also says distance matrices are normalised to [0, 1]. Normalisation is not applied, and it - would not change any chirality metric. Each metric is computed within one molecule, and each is - invariant to rescaling that molecule's `Delta` by a positive constant: ES-AUC is rank based, NN@1 - uses an argmin, and silhouette, DBI and clarity are ratios. -- The diagonal of the cosine matrix is set to 0. Without that, the 1e-12 guard leaves diagonal values - around 1e-12, and `sklearn.metrics.silhouette_score(metric="precomputed")` rejects the matrix. The - code then silently falls back to a hand-written silhouette that skips points in singleton classes - instead of scoring them 0. Without the fix, cosine SCI changes for 6 (MolSpectra, large norms) to 959 - (FMG) molecules per model; for example, MolAE gives 0.1938 instead of 0.1895. Cosine SCI is not - published. -- In `paper` mode, `--distance` changes ES-AUC, NN@1-Acc, SCI, DBI and clarity. Hopkins and SCI_unsup - are still computed on the raw vectors with Euclidean distance, as in the published code. In `v2` - they follow the selected distance (section 4). +- **The evaluator uses Euclidean distances between conformer embeddings by default** + (`chirality/evaluation.py`, `distance_matrix_for_subset`). This is the setting behind the published + Table 2 and behind the `euclidean` reference values in `reproduce/table2_chirality/expected.csv`. + `--distance cosine` computes the same metrics from cosine distances; those values are listed as the + `cosine` variant in the same file (for example MolAE SCI 0.189 and GemNet ES-AUC 0.602, against + 0.115 and 0.577 with Euclidean distance). +- Distance matrices are not rescaled. Each metric is computed within one molecule and is invariant to + rescaling that molecule's `Delta` by a positive constant: ES-AUC is rank based, NN@1 uses an argmin, + and silhouette, DBI and clarity are ratios. +- The diagonal of the cosine matrix is set to exactly 0. Otherwise the 1e-12 guard leaves diagonal + values around 1e-12, which `sklearn.metrics.silhouette_score(metric="precomputed")` rejects, and the + code falls back to a hand-written silhouette that skips points in singleton classes instead of + scoring them 0. The two paths differ for 6 (MolSpectra, large norms) to 959 (FMG) molecules per + model; for example MolAE cosine SCI is 0.1895 with the zeroed diagonal and 0.1938 without it. Cosine + SCI is not part of Table 2. +- With `--metric-version paper`, `--distance` changes ES-AUC, NN@1-Acc, SCI, DBI and clarity. Hopkins + and SCI_unsup are computed on the raw vectors with Euclidean distance. In `v2` they follow the + selected distance (section 4). ## 4. Metric definitions: `paper` and `v2` -`paper` is the code path that produced the published numbers. `v2` changes a definition only where -the published code departs from App. C.5 of the paper, or where a per-molecule value is fixed by -construction rather than by the representation. Every such case is listed below with the measured -evidence. The remaining metrics are identical in both versions. +`paper` is the code path behind the published numbers. `v2` differs from it in the definitions listed +below; the remaining metrics are identical in both versions. Each entry states what the two versions +compute and the measured effect on the published embeddings. Notation: a molecule has `n` conformers with labels `y`, representation distances `Delta`, and vectors `X` (rows L2-normalised when `v2` is combined with `--distance cosine`). @@ -141,9 +139,9 @@ Code: `silhouette_with_labels_from_D`. - This matches `sklearn.metrics.davies_bouldin_score` on non-degenerate input. - Fingerprints have no centroid under Tanimoto, so they keep the medoid version, with the 1e-12 term. - A molecule in which no class has two or more members is NaN. -- **Rationale:** App. C.5 defines DBI with Euclidean centroids. In `paper` mode, the 753 molecules - with `n >= 3` and only singleton classes get `S = 0` and therefore `DBI = 0`, the best possible - score, whatever the representation. +- **Rationale:** `v2` follows the centroid definition of App. C.5. In `paper` mode, the 753 molecules + with `n >= 3` and only singleton classes have `S = 0` and therefore `DBI = 0`, independently of the + representation. ### Hopkins statistic @@ -212,48 +210,48 @@ All values below come from the published embeddings (section 6) with the current numpy 2.4.6, scikit-learn 1.9.1, rdkit 2026.03.6 and an AMD EPYC 7513. `expected.csv` holds 6-decimal values for all 7 models × 4 variants, with a tolerance of 0.001. -The `euclidean` column is the published protocol. Every cell is within 0.001 of the printed value, -except for GemNet SCI (sign). E3FP is a fingerprint, so its cosine rows equal its Euclidean rows and its -Hopkins value is NaN. In `v2`, NN@1-Acc is averaged over 2,823 molecules instead of 3,842 and Hopkins -over 1,753 instead of 1,764. The other columns use the same molecules in both versions. - -| Metric | Model | Paper | `euclidean` (published protocol) | `cosine` | `v2_euclidean` | `v2_cosine` | -|---|---|---|---|---|---|---| -| ES-AUC | E3FP | 0.486 | 0.485935 | 0.485935 | 0.485935 | 0.485935 | -| ES-AUC | GemNet | 0.577 | 0.577410 | 0.601837 | 0.577410 | 0.601837 | -| ES-AUC | MolAE | 0.782 | 0.782544 | 0.781942 | 0.782544 | 0.781942 | -| ES-AUC | MolSpectra | 0.545 | 0.544744 | 0.543114 | 0.544744 | 0.543114 | -| ES-AUC | UniMol | 0.622 | 0.622998 | 0.622620 | 0.622998 | 0.622620 | -| ES-AUC | FMG | 0.706 | 0.705652 | 0.712555 | 0.705652 | 0.712555 | -| ES-AUC | MACE | 0.485 | 0.485611 | 0.484971 | 0.485611 | 0.484971 | -| NN@1-Acc | E3FP | 0.178 | 0.177939 | 0.177939 | 0.256400 | 0.256400 | -| NN@1-Acc | GemNet | 0.292 | 0.291641 | 0.312251 | 0.415357 | 0.444804 | -| NN@1-Acc | MolAE | 0.497 | 0.497546 | 0.497357 | 0.709640 | 0.709405 | -| NN@1-Acc | MolSpectra | 0.235 | 0.235250 | 0.235108 | 0.337786 | 0.337668 | -| NN@1-Acc | UniMol | 0.339 | 0.339582 | 0.338899 | 0.481977 | 0.480923 | -| NN@1-Acc | FMG | 0.412 | 0.411765 | 0.422004 | 0.587976 | 0.601655 | -| NN@1-Acc | MACE | 0.199 | 0.199113 | 0.200708 | 0.283468 | 0.285157 | -| Hopkins | E3FP | – | NaN | NaN | NaN | NaN | -| Hopkins | GemNet | 0.593 | 0.592812 | 0.592812 | 0.592823 | 0.577979 | -| Hopkins | MolAE | 0.602 | 0.602659 | 0.602659 | 0.602748 | 0.602690 | -| Hopkins | MolSpectra | 0.533 | 0.532642 | 0.532642 | 0.532736 | 0.533677 | -| Hopkins | UniMol | 0.559 | 0.559572 | 0.559572 | 0.559674 | 0.559664 | -| Hopkins | FMG | 0.752 | 0.751741 | 0.751741 | 0.752106 | 0.751049 | -| Hopkins | MACE | 0.637 | 0.637379 | 0.637379 | 0.637289 | 0.637111 | -| SCI | E3FP | -0.012 | -0.012543 | -0.012543 | -0.012543 | -0.012543 | -| SCI | GemNet | 0.015 (sign typo) | -0.015151 | 0.004663 | -0.015151 | 0.004663 | -| SCI | MolAE | 0.115 | 0.115274 | 0.189480 | 0.115274 | 0.189480 | -| SCI | MolSpectra | -0.020 | -0.020326 | -0.037315 | -0.020326 | -0.037315 | -| SCI | UniMol | 0.012 | 0.011623 | 0.021010 | 0.011623 | 0.021010 | -| SCI | FMG | 0.117 | 0.117174 | 0.158030 | 0.117174 | 0.158030 | -| SCI | MACE | -0.094 | -0.093877 | -0.151823 | -0.093877 | -0.151823 | -| SCI_unsup | E3FP | 0.033 | 0.033825 | 0.033825 | 0.033825 | 0.033825 | -| SCI_unsup | GemNet | 0.272 | 0.271961 | 0.271961 | 0.271961 | 0.380451 | -| SCI_unsup | MolAE | 0.247 | 0.247099 | 0.247099 | 0.247099 | 0.408498 | -| SCI_unsup | MolSpectra | 0.127 | 0.127115 | 0.127115 | 0.127115 | 0.232817 | -| SCI_unsup | UniMol | 0.152 | 0.152108 | 0.152108 | 0.152108 | 0.270063 | -| SCI_unsup | FMG | 0.509 | 0.509364 | 0.509364 | 0.509364 | 0.682689 | -| SCI_unsup | MACE | 0.369 | 0.369336 | 0.369336 | 0.369336 | 0.566272 | +The `euclidean` column is the published protocol. E3FP is a fingerprint, so its cosine rows equal its +Euclidean rows and its Hopkins value is NaN. In `v2`, NN@1-Acc is averaged over 2,823 molecules instead +of 3,842 and Hopkins over 1,753 instead of 1,764. The other columns use the same molecules in both +versions. + +| Metric | Model | `euclidean` (published protocol) | `cosine` | `v2_euclidean` | `v2_cosine` | +|---|---|---|---|---|---| +| ES-AUC | E3FP | 0.485935 | 0.485935 | 0.485935 | 0.485935 | +| ES-AUC | GemNet | 0.577410 | 0.601837 | 0.577410 | 0.601837 | +| ES-AUC | MolAE | 0.782544 | 0.781942 | 0.782544 | 0.781942 | +| ES-AUC | MolSpectra | 0.544744 | 0.543114 | 0.544744 | 0.543114 | +| ES-AUC | UniMol | 0.622998 | 0.622620 | 0.622998 | 0.622620 | +| ES-AUC | FMG | 0.705652 | 0.712555 | 0.705652 | 0.712555 | +| ES-AUC | MACE | 0.485611 | 0.484971 | 0.485611 | 0.484971 | +| NN@1-Acc | E3FP | 0.177939 | 0.177939 | 0.256400 | 0.256400 | +| NN@1-Acc | GemNet | 0.291641 | 0.312251 | 0.415357 | 0.444804 | +| NN@1-Acc | MolAE | 0.497546 | 0.497357 | 0.709640 | 0.709405 | +| NN@1-Acc | MolSpectra | 0.235250 | 0.235108 | 0.337786 | 0.337668 | +| NN@1-Acc | UniMol | 0.339582 | 0.338899 | 0.481977 | 0.480923 | +| NN@1-Acc | FMG | 0.411765 | 0.422004 | 0.587976 | 0.601655 | +| NN@1-Acc | MACE | 0.199113 | 0.200708 | 0.283468 | 0.285157 | +| Hopkins | E3FP | NaN | NaN | NaN | NaN | +| Hopkins | GemNet | 0.592812 | 0.592812 | 0.592823 | 0.577979 | +| Hopkins | MolAE | 0.602659 | 0.602659 | 0.602748 | 0.602690 | +| Hopkins | MolSpectra | 0.532642 | 0.532642 | 0.532736 | 0.533677 | +| Hopkins | UniMol | 0.559572 | 0.559572 | 0.559674 | 0.559664 | +| Hopkins | FMG | 0.751741 | 0.751741 | 0.752106 | 0.751049 | +| Hopkins | MACE | 0.637379 | 0.637379 | 0.637289 | 0.637111 | +| SCI | E3FP | -0.012543 | -0.012543 | -0.012543 | -0.012543 | +| SCI | GemNet | -0.015151 | 0.004663 | -0.015151 | 0.004663 | +| SCI | MolAE | 0.115274 | 0.189480 | 0.115274 | 0.189480 | +| SCI | MolSpectra | -0.020326 | -0.037315 | -0.020326 | -0.037315 | +| SCI | UniMol | 0.011623 | 0.021010 | 0.011623 | 0.021010 | +| SCI | FMG | 0.117174 | 0.158030 | 0.117174 | 0.158030 | +| SCI | MACE | -0.093877 | -0.151823 | -0.093877 | -0.151823 | +| SCI_unsup | E3FP | 0.033825 | 0.033825 | 0.033825 | 0.033825 | +| SCI_unsup | GemNet | 0.271961 | 0.271961 | 0.271961 | 0.380451 | +| SCI_unsup | MolAE | 0.247099 | 0.247099 | 0.247099 | 0.408498 | +| SCI_unsup | MolSpectra | 0.127115 | 0.127115 | 0.127115 | 0.232817 | +| SCI_unsup | UniMol | 0.152108 | 0.152108 | 0.152108 | 0.270063 | +| SCI_unsup | FMG | 0.509364 | 0.509364 | 0.509364 | 0.682689 | +| SCI_unsup | MACE | 0.369336 | 0.369336 | 0.369336 | 0.566272 | ## 6. Published embeddings and keys @@ -279,29 +277,25 @@ The files are the original bytes from the authors' backup, published at `EscheWa ``` Starting from the HF MolBlocks instead gives the same bits for 2,846 of 3,000 conformers. -- **Other models.** The extraction settings (checkpoint, layer, pooling) of the other published - embeddings have not been recovered. The files are provided so that the published numbers can be - recomputed. They do not document how the embeddings were produced. - -## 7. Known differences between the paper and the recomputed values - -- **GemNet SCI.** The paper prints `0.015`. The recomputed value, and the Sept-2025 output, is - `-0.015151`, so the minus sign is missing in the table. -- **Rounding.** The printed values mix rounding and truncation to three decimals. The largest gap is - UniMol ES-AUC: 0.622998 is printed as 0.622. Every other cell is within 0.001 of the recomputed value. -- **Hopkins for E3FP** is printed as "–". Hopkins is not defined for fingerprints, and the evaluator - returns NaN. -- **Sept-2025 outputs.** For E3FP, GemNet, MolAE, MolSpectra and UniMol, the backup holds the original - per-molecule outputs (`en_sep_results/*.json`). With the defaults, ES-AUC, NN@1-Acc, SCI, DBI and - Hopkins are bit-identical for every molecule of all five models. - - SCI_unsup is bit-identical for E3FP, MolAE and MolSpectra. - - It differs for 1 GemNet molecule and 2 UniMol molecules, which changes the means by +2.4e-6 and - +2.0e-6. On these near-tied silhouette landscapes the KMeans local optimum depends on the OpenBLAS - kernel. - - The unpublished clarity columns differ by less than 2e-5, because of a numpy version difference - (they match under numpy 2.2 and 2.3). -- **FMG and MACE** have no original metric output in the backup. Their expected values are recomputed - from the published embeddings. +- **Other models.** Each file holds one vector per conformer in dataset row order, with the key and + shape listed above. The reference values in `reproduce/table2_chirality/expected.csv` are computed + from these files. + +## 7. Agreement with the original per-molecule outputs + +For E3FP, GemNet, MolAE, MolSpectra and UniMol, the original per-molecule outputs of the Sept-2025 run +are published as `results/chirality/en_sep_results/*.json` in `EscheWang/3dcs-embeddings`. Recomputed +with the defaults of this evaluator: + +- ES-AUC, NN@1-Acc, SCI, DBI and Hopkins are bit-identical for every molecule of all five models. +- SCI_unsup is bit-identical for E3FP, MolAE and MolSpectra. It differs for 1 GemNet molecule and + 2 UniMol molecules, which shifts the means by +2.4e-6 and +2.0e-6: on these near-tied silhouette + landscapes the KMeans local optimum depends on the OpenBLAS kernel. +- The clarity columns, which are not part of Table 2, differ by less than 2e-5 with numpy 2.4 and are + bit-identical under numpy 2.2 and 2.3. + +For FMG and MACE the run has no per-molecule output in this release; their reference values are +computed from the published embeddings. ## 8. Performance diff --git a/docs/metrics/energy.md b/docs/metrics/energy.md index 122acae..6b26187 100644 --- a/docs/metrics/energy.md +++ b/docs/metrics/energy.md @@ -4,9 +4,9 @@ This page defines the metrics behind the zero-shot energy tables of the paper (T text, Tables 6 and 7 in Appendix B.2). It covers the data, the sampling protocol, and each metric in its two implementations: -- `--metric-version paper` (the default) is the implementation that produced the published numbers. -- `--metric-version v2` follows the definitions written in Appendix C.3/C.6. Where the appendix - leaves a choice open, v2 makes that choice explicit. +- `--metric-version paper` (the default) is the implementation behind the published numbers. +- `--metric-version v2` implements the definitions of Appendix C.3/C.6, and makes the choices that + the appendix leaves open explicit. Both versions share the same windows and the same aggregation. `reproduce/energy_tables_3_6_7/` contains the end-to-end script and the expected values for both. @@ -33,12 +33,12 @@ Download them with `load_dataset("EscheWang/3dcs", name="traj_energies", split=" standalone repo `EscheWang/3dcs-traj-energies`. Either source can be converted from the original rMD17 npz files with `python -m three_dbench convert traj`. -**Earlier revision.** An earlier revision of `traj_energies` stored a float32 cast of these energies. -The schema was declared float64, but each molecule held only 1,096–3,334 distinct values, with a -resolution of 1/64–1/256 kcal/mol. Do not use that revision (see the dataset card). With those values: -- Pooled CKA means drop by up to about 40%, e.g. GemNet 0.0163 → 0.0113. -- Smoothness and KS shift by about 0.001. -- Table 7 no longer reproduces. +**Dataset revisions.** Revisions of `traj_energies` before 18 September 2026 stored the same energies +cast to float32 (1,096–3,334 distinct values per molecule, resolution 1/64–1/256 kcal/mol); the +current revision stores the original float64 values. The tables on this page are computed from the +float64 energies. Several metrics are sensitive to that precision: with the float32 values, pooled +CKA means are up to about 40 % lower (GemNet 0.0163 against 0.0113), Smoothness and KS shift by about +0.001, and the Table 7 jump counts change. The evaluator therefore checks the precision of the energies of every evaluated molecule (option `--energy-precision-check`, default `error`). A molecule is flagged as quantized when either: @@ -55,8 +55,8 @@ files: - The median |ΔE| between consecutive frames equals the median over random pairs, e.g. aspirin 5.847 vs 5.834 kcal/mol. -Metrics defined along consecutive frames (TS and Smoothness) are therefore computed on effectively -random pairs of frames. See §4. +Metrics defined along consecutive frames (TS and Smoothness in `v2`) therefore require +`--time-ordered`, which declares that the frames of the loaded dataset are in time order. See §4. **Embeddings.** `EscheWang/3dcs-embeddings` holds the backed-up files as `traj//rmd17_.{npz,pkl}`, one file per molecule, with rows in frame order: @@ -149,10 +149,10 @@ CKA = ⟨HK⁽ᴰ⁾H, HK⁽Δ⁾H⟩_F / (‖HK⁽ᴰ⁾H‖_F ‖HK⁽Δ⁾H | paper | **One shared** σ = √median{d² : d ∈ vec(dE) ∪ vec(Δ), d > 0} over the concatenation of both distance sets (`cka_rbf(..., share_sigma=True)`). | | v2 | **Separate** median heuristics σ_D = √median{dE² > 0} and σ_Δ = √median{Δ² > 0} (Appendix C.3). | -*Rationale for v2:* dE is in kcal/mol (median ≈ 6) and Δ ∈ [0, 1] (median ≈ 10⁻⁴–10⁻³ for the -continuous models). The shared median is therefore ≈ 0.003, far below almost every energy difference, and the -energy kernel degenerates to an indicator of near-tied energies. This is also why the published CKA -values depend strongly on the precision of the energies. +*Scales:* dE is in kcal/mol (median ≈ 6) and Δ ∈ [0, 1] (median ≈ 10⁻⁴–10⁻³ for the continuous +models). A shared bandwidth is therefore ≈ 0.003 and is set by Δ, so the energy kernel responds mainly +to near-tied energies; this is also why `paper` CKA values depend on the precision of the energies. +`v2` uses one bandwidth per distance set. ### Isotonic R² (key `iso_R2`) @@ -182,7 +182,7 @@ gives 94,275.340 ± 163.305 at λ = 2. The labels are yᵢⱼ = 1{dEᵢⱼ > 2σ̂_rms} with σ̂_rms = √(mean(dE²) + ε), and the scores are sᵢⱼ = Δᵢⱼ. The AUC is computed with `sklearn.metrics.roc_auc_score` (PR-AUC with `average_precision_score`). -Both versions use this definition. It already matched the appendix in `paper`. +Both versions use this definition. ### Thresholded smoothness TS (key `TS`) @@ -193,22 +193,22 @@ Both versions use this definition. It already matched the appendix in `paper`. ### Smoothness (key `Smoothness`; Table 6) -The final paper does not define this metric. Its two definitions here are: +Two definitions are available: | | Definition | |---|---| | paper | Consecutive frames in file order, in raw units: Smoothness = mean_k exp(−Δ_{k,k+1} / (\|E_{k+1} − E_k\| + 10⁻⁸)). Δ is dimensionless and \|ΔE\| is in kcal/mol. | | v2 | The scale-normalised TS expression over **all** consecutive segments (T_E = 0): mean_k exp(−(Δ_{k,k+1}/Q_Z) / (\|E_{k+1} − E_k\|/Q_E + ε)). **It is computed only with `--time-ordered`; otherwise it is NaN.** | -*Rationale for v2:* both quantities describe how the representation changes along a trajectory, which -requires time-ordered frames. rMD17 frames are not time-ordered (§1). The `paper` values for rMD17 -therefore average over pairs of frames that are effectively random, and v2 does not report them. +*Rationale for v2:* both quantities describe how the representation changes along a trajectory, so +`v2` computes them only over frames declared to be in time order (`--time-ordered`) and returns NaN +otherwise. The rMD17 frame order is described in §1. ### Distributional divergence KS / W1 (keys `KS`, `W1`) | | Definition | |---|---| -| paper | Two-sample KS statistic `scipy.stats.ks_2samp(vec Δ, vec dE)` and `wasserstein_distance(vec Δ, vec dE)`. Δ ∈ [0, 1] is compared directly with dE in kcal/mol, so KS is close to 1 for every continuous representation (≥ 0.976 in Table 6), and the value mostly reflects the unit mismatch. | +| paper | Two-sample KS statistic `scipy.stats.ks_2samp(vec Δ, vec dE)` and `wasserstein_distance(vec Δ, vec dE)`, computed on the raw quantities: Δ ∈ [0, 1] against dE in kcal/mol. The two ranges differ, so KS is close to 1 for every continuous representation (≥ 0.976 in Table 6). | | v2 | The same statistics between the dimensionless quantities vec(Δ)/Q_Z and vec(dE)/Q_E, with Q_Z and Q_E the 0.9-quantiles above. **Smaller is better** (more similar distribution shapes). | ### Distance correlation (key `dCor`, not reported in the paper) @@ -224,8 +224,8 @@ Both versions return the double-centred distance correlation of the two n × n m | Table 7: jumps at 0.1σ … 3σ | `EJS_num_jumps_lam0p1` … `EJS_num_jumps_lam3` | `reproduce/energy_tables_3_6_7/collect.py` performs this mapping. `expected.csv` in the same folder -lists, for every cell, the printed value, the value recomputed with this code, and notes on cells -where the two differ. +lists, for every cell, the value as printed in the paper, the reference value computed with this code, +and a note describing the metric and the variant. ## 6. Usage @@ -240,7 +240,7 @@ python -m three_dbench evaluate traj \ python -m three_dbench evaluate traj --dataset-dir data/hf/traj/energies \ --embeddings embeddings/traj/e3fp --model-name E3FP --n-jobs 16 -# Corrected definitions, subset of molecules +# v2 definitions, subset of molecules python -m three_dbench evaluate traj --dataset-dir data/hf/traj/energies \ --embeddings embeddings/traj/unimol --metric-version v2 --molecules aspirin ethanol --n-jobs 8 ``` diff --git a/docs/metrics/geometry.md b/docs/metrics/geometry.md index c9df3d1..fe9ac49 100644 --- a/docs/metrics/geometry.md +++ b/docs/metrics/geometry.md @@ -2,7 +2,7 @@ This page documents how `python -m three_dbench evaluate rotation` computes the geometry metrics, which definitions were used for the published Table 1, and what `--metric-version v2` -changes. Chirality and energy metrics are described in [chirality.md](chirality.md) and +computes. Chirality and energy metrics are described in [chirality.md](chirality.md) and [energy.md](energy.md); [../METRICS.md](../METRICS.md) is the index. ## Data and alignment @@ -11,9 +11,8 @@ changes. Chirality and energy metrics are described in [chirality.md](chirality. conformers, 16 shards. Each row has `key`, `shard`, `n_conformers`, `offset`, `mol_blocks` and `torsion_deg`. - **`offset` is per shard.** It restarts at 0 in every shard, and the rows are ordered by the - shard id as a string (`0, 1, 10, 11, …, 15, 2, …, 9`). The initial release (0.1.0) sliced a single - flat array with this offset, which picks the wrong conformers for every shard other than 0. - The evaluator now supports three layouts: + shard id as a string (`0, 1, 10, 11, …, 15, 2, …, 9`), so it is an index into that shard's + embeddings, not into a single concatenated array. The evaluator supports three layouts: | `--layout` | Embeddings | Slicing | |---|---|---| @@ -33,8 +32,8 @@ changes. Chirality and energy metrics are described in [chirality.md](chirality. runs (1,464,495 of 1,559,779 molecules qualify). - Molecules whose conformers cannot be merged for RMSD (different heavy-atom counts after `RemoveHs`) are skipped and listed in `config.json` (`failed_keys`). On the published data this - happens for `1-R5B2G8_6-R7B1G14_0_22` (shard 1) and, according to the backup outputs, - `6-R8B2G15_35-R2B1G6_0_36` (shard 2); the published full run also has no result for these two. + happens for `1-R5B2G8_6-R7B1G14_0_22` (shard 1) and `6-R8B2G15_35-R2B1G6_0_36` (shard 2); the + original full run has no result for these two either. - Aggregation: one value per molecule; `summary.csv` reports the mean and median over molecules with a finite value. @@ -46,7 +45,7 @@ Outputs: `_per_key.parquet` (one row per molecule and distance space), `s `--metric-version paper` is the default. The Python API also exposes `legacy` (the behaviour of `compute_all_geometry_metrics` in release 0.1.0) for comparison. -| Metric (output key) | `paper` (published runs) | `v2` (paper text) | `legacy` (release 0.1.0) | +| Metric (output key) | `paper` (published runs) | `v2` | `legacy` (release 0.1.0) | |---|---|---|---| | Spearman (`A1_spearman`) | Spearman of upper-triangular `D` vs `Δ` | same | same | | Kendall (`A2_kendall`) | as Spearman, **only for molecules with ≥ 11 conformers** (NaN otherwise) | all molecules with ≥ 2 pairs | all molecules | @@ -63,38 +62,37 @@ preservation (not in Table 1; they dominate the runtime). Notes on `v2`: -- **LIE**: the neighbour set of the paper definition does not contain the conformer itself. - With the conformer included, one of the k "neighbours" has distance 0 in both spaces, which - dilutes the score, and k = 10 exceeds the conformer count of most molecules (median 6). -- **AS**: the appendix defines AS as the median representation change per angular increment along - the scan. `‖z_{i+1} − z_i‖` is in the units of the embedding, so AS values are comparable +- **LIE**: the neighbour set follows the Table 1 caption and Appendix C.4, i.e. the k = 3 nearest + *other* conformers. In `paper` mode the set has k = 10 entries and contains the conformer itself, + whose distance is 0 in both spaces, and k = 10 exceeds the conformer count of most molecules + (median 6). +- **AS**: Appendix C.4 defines AS as the median representation change per angular increment along + the scan. `‖z_{i+1} − z_i‖` is in the units of the embedding, so `v2` AS values are comparable between conformers of one model but not across models with different embedding scales. The wrap-around step (largest torsion back to the smallest) is excluded because conformers do not necessarily cover the full circle after redundancy removal. -- **Isotonic R²**: the appendix fits `D` as a monotone function of `Δ`. The reverse fit rewards a - constant representation with R² = 1. +- **Isotonic R²**: `v2` fits `D` as a monotone function of `Δ`, as in the appendix. Under the reverse + fit a constant representation scores R² = 1. - **Kendall**: no conformer-count threshold. -- The `paper` column is kept as the default so that the published numbers can be regenerated. +- `paper` is the default, so that the published numbers can be regenerated. ## Provenance of Table 1 -The scripts that produced Table 1 (`eval_geo_single.py`, `eval_geo_single_sup.py`) are not -available. The `paper` definitions above were recovered by recomputing GemNet metrics from the -published GemNet rotation embeddings and matching them, molecule by molecule, against the backed-up -per-molecule outputs of the two original runs (the per-molecule JSON files are published in -`EscheWang/3dcs-embeddings` under `results/rotation/`): +Table 1 comes from two original runs. The `paper` definitions above reproduce their per-molecule +outputs: GemNet metrics recomputed from the published GemNet rotation embeddings were matched, +molecule by molecule, against those outputs, which are published in `EscheWang/3dcs-embeddings` under +`results/rotation/`: | Table 1 rows | Original run | Molecules | Backup file | |---|---|---|---| | Spearman, Kendall, CKA, Isotonic R², Torsion-SP | `eval_geo_single.py --sample-ratio 0.1 --sample-seed 2027 --pairs-cap 100` | 146,389 (10 % sample) | `metrics_all_0.1_1.json.gz` (= `all_metric.csv`) | | LIE@k, AS | `eval_geo_single_sup.py --sample-ratio 1 --sample-seed 2027 --pairs-cap 100` | 1,464,493 | `metrics_sup_100.json.gz` | -The 10 % sample cannot be regenerated from the seed because the sampling code is not available. -Its molecule keys, read from the backup file, are distributed as +The molecule keys of the 10 % sample, read from that output file, are distributed as [`reproduce/table1_geometry/sampled_molecules_seed2027.txt`](../../reproduce/table1_geometry/sampled_molecules_seed2027.txt) -(146,389 keys in dataset order; use it with `--molecule-list`). `--sample-ratio/--sample-seed` -implement a documented per-shard sampler (`numpy.random.default_rng(seed + shard)`), which does not -reproduce that list. +(146,389 keys in dataset order); pass them with `--molecule-list` to evaluate the same molecules. +`--sample-ratio/--sample-seed` implement a per-shard sampler +(`numpy.random.default_rng(seed + shard)`) that draws its own sample. ### Per-molecule agreement (GemNet, cosine) @@ -115,42 +113,29 @@ coordinates with 4 decimals, so RMSD values differ from the original coordinates which can reorder near-tied pairs. LIE is sensitive for GemNet because some GemNet cosine distances are of the order of the 1e-12 stabiliser. -Two further properties of the published runs were found this way: +Two further properties of the original runs follow from the same comparison: - **Kendall** is finite for every sampled molecule with ≥ 11 conformers and NaN for every molecule - with ≤ 10 conformers (24,198 of 146,389 molecules have a value). The exact rule of the original - script is not recoverable; any pair-count threshold between 46 and 55 selects the same molecules. -- **Embedding offsets in the full run.** In `metrics_sup_100.json.gz`, the molecules of shard 1 after - `1-R5B2G8_6-R7B1G14_0_22` (local offset 69,515, 3 conformers) and of shard 2 after - `6-R8B2G15_35-R2B1G6_0_36` (local offset 560,074, 7 conformers) were evaluated with embeddings - shifted back by 3 and 7 rows: the failing molecule was skipped without advancing the embedding - cursor. For GemNet this reproduces 530/530 and 67/67 sampled molecules after the failure in - shards 1 and 2 (LIE and AS), while the correctly aligned embeddings do not. 91,093 of the - 1,464,493 molecules (6.2 %) are affected. The 10 % run is not affected (neither molecule is in the - sample), and E3FP is not affected (its fingerprints are stored per molecule). - `--replicate-offset-drift` (by-shard layout) recomputes all metrics with this shift and stores - them as `__offset_drift` columns; the regular columns always use aligned embeddings. - -For the four learned models, the effect on the published LIE@k and AS means can be estimated from -the backup outputs by averaging over the unaffected molecules only (an estimate; exact aligned -values require the embeddings, which are available for GemNet only): - -| Model | LIE@k published | LIE@k, unaffected molecules | AS published | AS, unaffected molecules | -|---|---|---|---|---| -| E3FP | 0.3238 | 0.3239 | 2.7576 | 2.7850 | -| GemNet | 0.3901 | 0.3662 | 0.001825 | 0.001166 | -| MolAE | 0.3489 | 0.3321 | 0.004755 | 0.004022 | -| MolSpectra | 0.2379 | 0.2254 | 0.021443 | 0.020741 | -| UniMol | 0.3059 | 0.2863 | 0.005987 | 0.005291 | - -(For E3FP the small differences reflect the subset only.) - -### Not reproduced - -- The Euclidean-space LIE@k values of the full run (not used in Table 1). In that run AS is the same - in both spaces (computed from the cosine distance). -- Rotation embeddings for E3FP, UniMol, MolAE and MolSpectra are not available, so their Table 1 - values can only be compared with the backed-up per-molecule outputs. + with ≤ 10 conformers (24,198 of 146,389 molecules have a value). Any pair-count threshold between + 46 and 55 selects the same molecules. +- **Embedding indexing in the full run.** In `metrics_sup_100.json.gz`, the values of the molecules + after `1-R5B2G8_6-R7B1G14_0_22` in shard 1 (local offset 69,515, 3 conformers) and after + `6-R8B2G15_35-R2B1G6_0_36` in shard 2 (local offset 560,074, 7 conformers) correspond to embedding + rows shifted back by 3 and 7 positions, i.e. by the conformer count of the molecule that the run + skipped. For GemNet, recomputing with that shift reproduces 530 of 530 and 67 of 67 sampled LIE and + AS values after those points, and indexing by `offset` does not. This covers 91,093 of the + 1,464,493 molecules (6.2 %); it does not apply to the 10 % run (neither molecule is in its sample) + or to E3FP, whose fingerprints are stored per molecule. + `--replicate-offset-drift` (by-shard layout) recomputes all metrics with the same shift and writes + them as `__offset_drift` columns; the regular columns index the embeddings by the dataset + `offset`. + +### Scope + +- The Euclidean-space LIE@k values of the full run are not part of Table 1 and are not recomputed + here. In that run AS is the same in both spaces (computed from the cosine distance). +- This release publishes rotation embeddings for GemNet; for E3FP, UniMol, MolAE and MolSpectra the + Table 1 rows are covered by the per-molecule outputs of the original runs. ## Runtime diff --git a/reproduce/README.md b/reproduce/README.md index 0340dad..7e502c2 100644 --- a/reproduce/README.md +++ b/reproduce/README.md @@ -5,7 +5,7 @@ Each directory recomputes one or more tables of the ICLR 2026 paper from the rel | Directory | Tables | Command | Notes | |---|---|---|---| -| [`table1_geometry/`](table1_geometry/README.md) | 1 | `bash reproduce/table1_geometry/run.sh` | GemNet only (the other rotation embeddings are not available); `QUICK=1` runs shard 1 | +| [`table1_geometry/`](table1_geometry/README.md) | 1 | `bash reproduce/table1_geometry/run.sh` | GemNet (the rotation embeddings published in this release); `QUICK=1` runs shard 1 | | [`table2_chirality/`](table2_chirality/README.md) | 2 | `bash reproduce/table2_chirality/run.sh` | 7 models; Euclidean (published) and cosine / v2 variants | | [`energy_tables_3_6_7/`](energy_tables_3_6_7/README.md) | 3, 6, 7 | `bash reproduce/energy_tables_3_6_7/run.sh` | 7 models; float64 energies, legacy windows | @@ -27,9 +27,9 @@ Every directory contains: | `metric` | metric id used by the table's `collect` script | | `variant` | e.g. `paper`, `euclidean`, `cosine`, `v2` (see the table's README) | | `paper_value` | value as printed in the paper (empty if the variant is not in the paper) | -| `expected_value` | value recomputed with this code or taken from the original outputs, 6 decimals; empty if not available yet | +| `expected_value` | reference value for the cell, at 6 decimals: computed with this code, or, where the embeddings are not part of the release, the mean of the original per-molecule outputs; empty when this file holds no reference value | | `tolerance` | allowed absolute difference (default 0.001) | -| `notes` | provenance and known differences from the printed value (rounding vs truncation, cells that cannot be recomputed, …) | +| `notes` | what the row computes: metric version and variant, molecule population, and the provenance of the reference value | `results.csv`: `table,model,metric,variant,value`. @@ -44,11 +44,11 @@ Status per expected row: - `PASS`: `|value − expected_value| ≤ tolerance`; - `FAIL`: larger difference or a non-finite value; - `MISSING`: no result row, e.g. a model whose embeddings are not published (a failure with `--strict`); -- `NO_EXPECTED`: the reference value is not available yet (never a failure). +- `NO_EXPECTED`: this file holds no reference value for the row (never a failure). The exit code is 1 if any row fails and 2 if no row could be compared. `--tables` and `--variants` restrict the comparison. -The `expected_value` is compared, not the printed `paper_value`: printed values are rounded or, in -some cells, truncated, and a few printed cells differ from the recomputed values for reasons listed in -`notes`. +`compare.py` compares `expected_value`, not `paper_value`. `paper_value` is carried at the precision +printed in the paper (three decimals in most tables); `expected_value` is the full-precision reference +value of this release. diff --git a/reproduce/compare.py b/reproduce/compare.py index 5a2bb4d..0fac77e 100755 --- a/reproduce/compare.py +++ b/reproduce/compare.py @@ -10,7 +10,7 @@ PASS |value - expected_value| <= tolerance (tolerance defaults to 0.001) FAIL the difference exceeds the tolerance, or the value is not finite MISSING no result row (e.g. a model whose embeddings are not available); a failure with --strict - NO_EXPECTED expected_value is empty (reference value not available yet); never a failure + NO_EXPECTED expected_value is empty (this file holds no reference value for the row); never a failure Exit code: 0 if no FAIL, 1 if any FAIL, 2 if no row could be compared. """ diff --git a/reproduce/energy_tables_3_6_7/README.md b/reproduce/energy_tables_3_6_7/README.md index 80560a7..b69040c 100644 --- a/reproduce/energy_tables_3_6_7/README.md +++ b/reproduce/energy_tables_3_6_7/README.md @@ -2,7 +2,7 @@ ```bash bash reproduce/energy_tables_3_6_7/run.sh # published definitions -N_JOBS=24 METRIC_VERSIONS="paper v2" bash reproduce/energy_tables_3_6_7/run.sh # plus corrected definitions +N_JOBS=24 METRIC_VERSIONS="paper v2" bash reproduce/energy_tables_3_6_7/run.sh # plus the v2 definitions ``` `run.sh` does four things: @@ -25,19 +25,15 @@ for the 7 models (1,000 windows per model). See `timings.tsv` in the output fold | File | Content | |---|---| -| `expected.csv` | For every published cell (variant `paper`): the printed value, the value recomputed with this code (6 decimals), a tolerance of 0.001 and notes. Variant `v2` rows hold the corrected definitions, which have no printed counterpart. | +| `expected.csv` | One row per table cell and metric version: the value as printed in the paper, the reference value computed with this code (6 decimals), a tolerance of 0.001 and a note describing the row. Variant `v2` rows are not printed in the paper. | | `paper_values.csv` | Values as printed in the final paper. Table 7 is written without thousands separators. | -| `make_expected.py` | Regenerates `expected.csv` from a reference `results.csv` and `paper_values.csv`, classifying each cell as rounding to the printed value, matching only by truncation, or differing. | +| `make_expected.py` | Regenerates `expected.csv` from a reference `results.csv` and `paper_values.csv`. | | `collect.py` | Maps `summary.csv` keys to table cells (see `docs/metrics/energy.md` §5). | -**Where the recomputed values differ from the printed tables.** The `notes` column of -`expected.csv` records each case: - -- Table 3 KS for E3FP (0.916) and MolSpectra (0.977) differ from the Table 6 values for the same - cells (0.913, 0.976). The recomputation reproduces the Table 6 values. -- Some Table 3 cells are printed truncated rather than rounded, e.g. UniMol EJS 0.30155 → 0.301. -- MACE and FMG columns are only partially reproducible from the backed-up embeddings. Most of their - published 95% CIs are 4–5 times wider than a 1,000-window run gives, and the published FMG - Smoothness (0.972 ± 0.002) is not attainable with these embeddings. -- The v2 `TS` and `Smoothness` are not reported, because rMD17 frames are not time-ordered - (`docs/metrics/energy.md` §1). +**Reference values.** `expected_value` is what this code computes from the published trajectory +embeddings and the float64 rMD17 energies with the published protocol (legacy windows, 100 × 2,000 +frames, seed 2025); `compare.py` compares against it. `paper_value` carries the value as printed in +the table of that row. + +`v2` does not report `TS` and `Smoothness` here: both are defined along consecutive frames and are +computed only with `--time-ordered` (`docs/metrics/energy.md` §1). diff --git a/reproduce/energy_tables_3_6_7/expected.csv b/reproduce/energy_tables_3_6_7/expected.csv index 24daf32..3d02ce4 100644 --- a/reproduce/energy_tables_3_6_7/expected.csv +++ b/reproduce/energy_tables_3_6_7/expected.csv @@ -1,414 +1,414 @@ table,model,metric,variant,paper_value,expected_value,tolerance,notes -3,E3FP,Spearman,paper,0.026,0.026068,0.001,recomputed value rounds to the printed value -3,E3FP,Kendall,paper,0.018,0.018248,0.001,recomputed value rounds to the printed value -3,E3FP,CKA,paper,0.011,0.010683,0.001,recomputed value rounds to the printed value -3,E3FP,isoR2,paper,0.002,0.001507,0.001,recomputed value rounds to the printed value -3,E3FP,EJS,paper,0.184,0.183794,0.001,recomputed value rounds to the printed value -3,E3FP,EJS_ROCAUC,paper,0.531,0.531304,0.001,recomputed value rounds to the printed value -3,E3FP,TS,paper,0.385,0.385347,0.001,recomputed value rounds to the printed value -3,E3FP,KS,paper,0.916,0.913498,0.001,recomputed value does not match the printed value; Table 3 prints 0.916 while Table 6 prints 0.913 for the same mean; the Table 3 value is carried over from the submission draft and is not produced by any backed-up run -3,FMG,Spearman,paper,0.015,0.013975,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -3,FMG,Kendall,paper,0.009,0.009321,0.001,recomputed value rounds to the printed value; Table 3 prints 0.009 while Table 6 prints 0.010 for the same mean -3,FMG,CKA,paper,0.011,0.011691,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value; Table 3 prints 0.011 while Table 6 prints 0.012 for the same mean -3,FMG,isoR2,paper,0.001,0.000822,0.001,recomputed value rounds to the printed value -3,FMG,EJS,paper,0.269,0.266977,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -3,FMG,EJS_ROCAUC,paper,0.517,0.515996,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -3,FMG,TS,paper,0.582,0.582899,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value; Table 3 prints 0.582 while Table 6 prints 0.583 for the same mean -3,FMG,KS,paper,0.999,0.999827,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value; Table 3 prints 0.999 while Table 6 prints 1.000 for the same mean -3,GemNet,Spearman,paper,0.078,0.077526,0.001,recomputed value rounds to the printed value -3,GemNet,Kendall,paper,0.052,0.051807,0.001,recomputed value rounds to the printed value -3,GemNet,CKA,paper,0.016,0.016325,0.001,recomputed value rounds to the printed value -3,GemNet,isoR2,paper,0.013,0.013470,0.001,recomputed value rounds to the printed value -3,GemNet,EJS,paper,0.356,0.355988,0.001,recomputed value rounds to the printed value -3,GemNet,EJS_ROCAUC,paper,0.592,0.591777,0.001,recomputed value rounds to the printed value -3,GemNet,TS,paper,0.444,0.443715,0.001,recomputed value rounds to the printed value -3,GemNet,KS,paper,0.998,0.998367,0.001,recomputed value rounds to the printed value -3,MACE,Spearman,paper,0.236,0.236143,0.001,recomputed value rounds to the printed value -3,MACE,Kendall,paper,0.159,0.159662,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value; Table 3 prints 0.159 while Table 6 prints 0.160 for the same mean -3,MACE,CKA,paper,0.017,0.017391,0.001,recomputed value rounds to the printed value -3,MACE,isoR2,paper,0.080,0.080723,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value; Table 3 prints 0.080 while Table 6 prints 0.083 for the same mean -3,MACE,EJS,paper,0.578,0.572063,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -3,MACE,EJS_ROCAUC,paper,0.764,0.760630,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -3,MACE,TS,paper,0.424,0.424343,0.001,recomputed value rounds to the printed value -3,MACE,KS,paper,0.999,0.999448,0.001,recomputed value rounds to the printed value -3,MolAE,Spearman,paper,0.039,0.038868,0.001,recomputed value rounds to the printed value -3,MolAE,Kendall,paper,0.026,0.025930,0.001,recomputed value rounds to the printed value -3,MolAE,CKA,paper,0.024,0.024077,0.001,recomputed value rounds to the printed value -3,MolAE,isoR2,paper,0.003,0.002824,0.001,recomputed value rounds to the printed value -3,MolAE,EJS,paper,0.294,0.293714,0.001,recomputed value rounds to the printed value -3,MolAE,EJS_ROCAUC,paper,0.545,0.545296,0.001,recomputed value rounds to the printed value -3,MolAE,TS,paper,0.346,0.346272,0.001,recomputed value rounds to the printed value -3,MolAE,KS,paper,0.998,0.998199,0.001,recomputed value rounds to the printed value -3,MolSpectra,Spearman,paper,0.023,0.022909,0.001,recomputed value rounds to the printed value -3,MolSpectra,Kendall,paper,0.015,0.015282,0.001,recomputed value rounds to the printed value -3,MolSpectra,CKA,paper,0.006,0.006444,0.001,recomputed value rounds to the printed value -3,MolSpectra,isoR2,paper,0.001,0.001481,0.001,recomputed value rounds to the printed value -3,MolSpectra,EJS,paper,0.271,0.270563,0.001,recomputed value rounds to the printed value -3,MolSpectra,EJS_ROCAUC,paper,0.526,0.526507,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value; Table 3 prints 0.526 while Table 6 prints 0.527 for the same mean -3,MolSpectra,TS,paper,0.409,0.408949,0.001,recomputed value rounds to the printed value -3,MolSpectra,KS,paper,0.977,0.976430,0.001,recomputed value does not match the printed value; Table 3 prints 0.977 while Table 6 prints 0.976 for the same mean; the Table 3 value is carried over from the submission draft and is not produced by any backed-up run -3,UniMol,Spearman,paper,0.043,0.042610,0.001,recomputed value rounds to the printed value -3,UniMol,Kendall,paper,0.028,0.028426,0.001,recomputed value rounds to the printed value -3,UniMol,CKA,paper,0.019,0.018778,0.001,recomputed value rounds to the printed value -3,UniMol,isoR2,paper,0.003,0.003016,0.001,recomputed value rounds to the printed value -3,UniMol,EJS,paper,0.301,0.301547,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value -3,UniMol,EJS_ROCAUC,paper,0.549,0.549794,0.001,printed value is the 3-dp truncation (not rounding) of the recomputed value -3,UniMol,TS,paper,0.356,0.356314,0.001,recomputed value rounds to the printed value -3,UniMol,KS,paper,0.997,0.996900,0.001,recomputed value rounds to the printed value -6,E3FP,Spearman:mean,paper,0.026,0.026068,0.001,recomputed value rounds to the printed value -6,E3FP,Spearman:ci95,paper,0.001,0.001085,0.001,recomputed value rounds to the printed value -6,E3FP,Kendall:mean,paper,0.018,0.018248,0.001,recomputed value rounds to the printed value -6,E3FP,Kendall:ci95,paper,0.001,0.000751,0.001,recomputed value rounds to the printed value -6,E3FP,CKA:mean,paper,0.011,0.010683,0.001,recomputed value rounds to the printed value -6,E3FP,CKA:ci95,paper,0.000,0.000383,0.001,recomputed value rounds to the printed value -6,E3FP,isoR2:mean,paper,0.002,0.001507,0.001,recomputed value rounds to the printed value -6,E3FP,isoR2:ci95,paper,0.000,0.000084,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(0.1sigma):mean,paper,0.151,0.150918,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(0.1sigma):ci95,paper,0.006,0.005522,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(0.5sigma):mean,paper,0.155,0.154587,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(0.5sigma):ci95,paper,0.006,0.005622,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(1sigma):mean,paper,0.162,0.161624,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(1sigma):ci95,paper,0.006,0.005834,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(2sigma):mean,paper,0.184,0.183794,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(2sigma):ci95,paper,0.007,0.006687,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(3sigma):mean,paper,0.217,0.217036,0.001,recomputed value rounds to the printed value -6,E3FP,EJS(3sigma):ci95,paper,0.009,0.008698,0.001,recomputed value rounds to the printed value -6,E3FP,ROC_AUC:mean,paper,0.531,0.531304,0.001,recomputed value rounds to the printed value -6,E3FP,ROC_AUC:ci95,paper,0.001,0.001400,0.001,recomputed value rounds to the printed value -6,E3FP,Smoothness:mean,paper,0.847,0.846836,0.001,recomputed value rounds to the printed value -6,E3FP,Smoothness:ci95,paper,0.002,0.002126,0.001,recomputed value rounds to the printed value -6,E3FP,TS:mean,paper,0.385,0.385347,0.001,recomputed value rounds to the printed value -6,E3FP,TS:ci95,paper,0.010,0.010232,0.001,recomputed value rounds to the printed value -6,E3FP,KS:mean,paper,0.913,0.913498,0.001,recomputed value rounds to the printed value -6,E3FP,KS:ci95,paper,0.001,0.000972,0.001,recomputed value rounds to the printed value -6,FMG,Spearman:mean,paper,0.015,0.013975,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,FMG,Spearman:ci95,paper,0.004,0.000781,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,Kendall:mean,paper,0.010,0.009321,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,FMG,Kendall:ci95,paper,0.003,0.000521,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,CKA:mean,paper,0.012,0.011691,0.001,recomputed value rounds to the printed value -6,FMG,CKA:ci95,paper,0.001,0.000167,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,isoR2:mean,paper,0.001,0.000822,0.001,recomputed value rounds to the printed value -6,FMG,isoR2:ci95,paper,0.000,0.000064,0.001,recomputed value rounds to the printed value -6,FMG,EJS(0.1sigma):mean,paper,0.250,0.250204,0.001,recomputed value rounds to the printed value -6,FMG,EJS(0.1sigma):ci95,paper,0.000,0.000031,0.001,recomputed value rounds to the printed value -6,FMG,EJS(0.5sigma):mean,paper,0.253,0.252309,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,FMG,EJS(0.5sigma):ci95,paper,0.001,0.000168,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,EJS(1sigma):mean,paper,0.257,0.256173,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,FMG,EJS(1sigma):ci95,paper,0.002,0.000420,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,EJS(2sigma):mean,paper,0.269,0.266977,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,FMG,EJS(2sigma):ci95,paper,0.006,0.001189,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,EJS(3sigma):mean,paper,0.283,0.281762,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,FMG,EJS(3sigma):ci95,paper,0.016,0.002975,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,ROC_AUC:mean,paper,0.517,0.515996,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,FMG,ROC_AUC:ci95,paper,0.005,0.000930,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,Smoothness:mean,paper,0.972,0.999889,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value; published FMG Smoothness 0.972 +/- 0.002 is not attainable with the backed-up FMG embeddings: every one of the 1000 per-window values is >= 0.999 -6,FMG,Smoothness:ci95,paper,0.002,0.000007,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs; published FMG Smoothness 0.972 +/- 0.002 is not attainable with the backed-up FMG embeddings: every one of the 1000 per-window values is >= 0.999 -6,FMG,TS:mean,paper,0.583,0.582899,0.001,recomputed value rounds to the printed value -6,FMG,TS:ci95,paper,0.007,0.001634,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,FMG,KS:mean,paper,1.000,0.999827,0.001,recomputed value rounds to the printed value -6,FMG,KS:ci95,paper,0.000,0.000015,0.001,recomputed value rounds to the printed value -6,GemNet,Spearman:mean,paper,0.078,0.077526,0.001,recomputed value rounds to the printed value -6,GemNet,Spearman:ci95,paper,0.002,0.002243,0.001,recomputed value rounds to the printed value -6,GemNet,Kendall:mean,paper,0.052,0.051807,0.001,recomputed value rounds to the printed value -6,GemNet,Kendall:ci95,paper,0.002,0.001504,0.001,recomputed value rounds to the printed value -6,GemNet,CKA:mean,paper,0.016,0.016325,0.001,recomputed value rounds to the printed value -6,GemNet,CKA:ci95,paper,0.000,0.000417,0.001,recomputed value rounds to the printed value -6,GemNet,isoR2:mean,paper,0.013,0.013470,0.001,recomputed value rounds to the printed value -6,GemNet,isoR2:ci95,paper,0.001,0.000981,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(0.1sigma):mean,paper,0.252,0.251834,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(0.1sigma):ci95,paper,0.000,0.000067,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(0.5sigma):mean,paper,0.263,0.263241,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(0.5sigma):ci95,paper,0.000,0.000446,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(1sigma):mean,paper,0.286,0.285514,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(1sigma):ci95,paper,0.001,0.001190,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(2sigma):mean,paper,0.356,0.355988,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(2sigma):ci95,paper,0.004,0.003577,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(3sigma):mean,paper,0.451,0.450997,0.001,recomputed value rounds to the printed value -6,GemNet,EJS(3sigma):ci95,paper,0.007,0.007127,0.001,recomputed value rounds to the printed value -6,GemNet,ROC_AUC:mean,paper,0.592,0.591777,0.001,recomputed value rounds to the printed value -6,GemNet,ROC_AUC:ci95,paper,0.003,0.002668,0.001,recomputed value rounds to the printed value -6,GemNet,Smoothness:mean,paper,0.998,0.998238,0.001,recomputed value rounds to the printed value -6,GemNet,Smoothness:ci95,paper,0.000,0.000188,0.001,recomputed value rounds to the printed value -6,GemNet,TS:mean,paper,0.444,0.443715,0.001,recomputed value rounds to the printed value -6,GemNet,TS:ci95,paper,0.004,0.003603,0.001,recomputed value rounds to the printed value -6,GemNet,KS:mean,paper,0.998,0.998367,0.001,recomputed value rounds to the printed value -6,GemNet,KS:ci95,paper,0.000,0.000172,0.001,recomputed value rounds to the printed value -6,MACE,Spearman:mean,paper,0.236,0.236143,0.001,recomputed value rounds to the printed value -6,MACE,Spearman:ci95,paper,0.011,0.002361,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,Kendall:mean,paper,0.160,0.159662,0.001,recomputed value rounds to the printed value -6,MACE,Kendall:ci95,paper,0.008,0.001626,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,CKA:mean,paper,0.017,0.017391,0.001,recomputed value rounds to the printed value -6,MACE,CKA:ci95,paper,0.001,0.000285,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,isoR2:mean,paper,0.083,0.080723,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,MACE,isoR2:ci95,paper,0.010,0.002210,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,EJS(0.1sigma):mean,paper,0.255,0.255266,0.001,recomputed value rounds to the printed value -6,MACE,EJS(0.1sigma):ci95,paper,0.000,0.000079,0.001,recomputed value rounds to the printed value -6,MACE,EJS(0.5sigma):mean,paper,0.286,0.286281,0.001,recomputed value rounds to the printed value -6,MACE,EJS(0.5sigma):ci95,paper,0.002,0.000527,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,EJS(1sigma):mean,paper,0.350,0.349974,0.001,recomputed value rounds to the printed value -6,MACE,EJS(1sigma):ci95,paper,0.007,0.001425,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,EJS(2sigma):mean,paper,0.578,0.572063,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,MACE,EJS(2sigma):ci95,paper,0.019,0.004105,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,EJS(3sigma):mean,paper,0.845,0.837292,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,MACE,EJS(3sigma):ci95,paper,0.025,0.005381,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,ROC_AUC:mean,paper,0.764,0.760630,0.001,recomputed value does not match the printed value; MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no backed-up run reproduces this published value -6,MACE,ROC_AUC:ci95,paper,0.011,0.002222,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,Smoothness:mean,paper,0.999,0.999394,0.001,recomputed value rounds to the printed value -6,MACE,Smoothness:ci95,paper,0.000,0.000031,0.001,recomputed value rounds to the printed value -6,MACE,TS:mean,paper,0.424,0.424343,0.001,recomputed value rounds to the printed value -6,MACE,TS:ci95,paper,0.014,0.003126,0.001,recomputed value does not match the printed value; published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly 40-47 windows); no backed-up run reproduces the published MACE/FMG CIs -6,MACE,KS:mean,paper,0.999,0.999448,0.001,recomputed value rounds to the printed value -6,MACE,KS:ci95,paper,0.000,0.000022,0.001,recomputed value rounds to the printed value -6,MolAE,Spearman:mean,paper,0.039,0.038868,0.001,recomputed value rounds to the printed value -6,MolAE,Spearman:ci95,paper,0.001,0.001260,0.001,recomputed value rounds to the printed value -6,MolAE,Kendall:mean,paper,0.026,0.025930,0.001,recomputed value rounds to the printed value -6,MolAE,Kendall:ci95,paper,0.001,0.000841,0.001,recomputed value rounds to the printed value -6,MolAE,CKA:mean,paper,0.024,0.024077,0.001,recomputed value rounds to the printed value -6,MolAE,CKA:ci95,paper,0.000,0.000342,0.001,recomputed value rounds to the printed value -6,MolAE,isoR2:mean,paper,0.003,0.002824,0.001,recomputed value rounds to the printed value -6,MolAE,isoR2:ci95,paper,0.000,0.000180,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(0.1sigma):mean,paper,0.251,0.250616,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(0.1sigma):ci95,paper,0.000,0.000043,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(0.5sigma):mean,paper,0.256,0.255715,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(0.5sigma):ci95,paper,0.000,0.000266,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(1sigma):mean,paper,0.265,0.265368,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(1sigma):ci95,paper,0.001,0.000683,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(2sigma):mean,paper,0.294,0.293714,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(2sigma):ci95,paper,0.002,0.001914,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(3sigma):mean,paper,0.328,0.327981,0.001,recomputed value rounds to the printed value -6,MolAE,EJS(3sigma):ci95,paper,0.004,0.004323,0.001,recomputed value rounds to the printed value -6,MolAE,ROC_AUC:mean,paper,0.545,0.545296,0.001,recomputed value rounds to the printed value -6,MolAE,ROC_AUC:ci95,paper,0.001,0.001443,0.001,recomputed value rounds to the printed value -6,MolAE,Smoothness:mean,paper,0.996,0.996316,0.001,recomputed value rounds to the printed value -6,MolAE,Smoothness:ci95,paper,0.000,0.000125,0.001,recomputed value rounds to the printed value -6,MolAE,TS:mean,paper,0.346,0.346272,0.001,recomputed value rounds to the printed value -6,MolAE,TS:ci95,paper,0.001,0.001449,0.001,recomputed value rounds to the printed value -6,MolAE,KS:mean,paper,0.998,0.998199,0.001,recomputed value rounds to the printed value -6,MolAE,KS:ci95,paper,0.000,0.000061,0.001,recomputed value rounds to the printed value -6,MolSpectra,Spearman:mean,paper,0.023,0.022909,0.001,recomputed value rounds to the printed value -6,MolSpectra,Spearman:ci95,paper,0.001,0.001254,0.001,recomputed value rounds to the printed value -6,MolSpectra,Kendall:mean,paper,0.015,0.015282,0.001,recomputed value rounds to the printed value -6,MolSpectra,Kendall:ci95,paper,0.001,0.000836,0.001,recomputed value rounds to the printed value -6,MolSpectra,CKA:mean,paper,0.006,0.006444,0.001,recomputed value rounds to the printed value -6,MolSpectra,CKA:ci95,paper,0.000,0.000169,0.001,recomputed value rounds to the printed value -6,MolSpectra,isoR2:mean,paper,0.001,0.001481,0.001,recomputed value rounds to the printed value -6,MolSpectra,isoR2:ci95,paper,0.000,0.000174,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(0.1sigma):mean,paper,0.250,0.250276,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(0.1sigma):ci95,paper,0.000,0.000041,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(0.5sigma):mean,paper,0.253,0.252824,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(0.5sigma):ci95,paper,0.000,0.000258,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(1sigma):mean,paper,0.258,0.257560,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(1sigma):ci95,paper,0.001,0.000658,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(2sigma):mean,paper,0.271,0.270563,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(2sigma):ci95,paper,0.002,0.001780,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(3sigma):mean,paper,0.289,0.289048,0.001,recomputed value rounds to the printed value -6,MolSpectra,EJS(3sigma):ci95,paper,0.004,0.003845,0.001,recomputed value rounds to the printed value -6,MolSpectra,ROC_AUC:mean,paper,0.527,0.526507,0.001,recomputed value rounds to the printed value -6,MolSpectra,ROC_AUC:ci95,paper,0.001,0.001402,0.001,recomputed value rounds to the printed value -6,MolSpectra,Smoothness:mean,paper,0.974,0.973790,0.001,recomputed value rounds to the printed value -6,MolSpectra,Smoothness:ci95,paper,0.001,0.000667,0.001,recomputed value rounds to the printed value -6,MolSpectra,TS:mean,paper,0.409,0.408949,0.001,recomputed value rounds to the printed value -6,MolSpectra,TS:ci95,paper,0.004,0.004161,0.001,recomputed value rounds to the printed value -6,MolSpectra,KS:mean,paper,0.976,0.976430,0.001,recomputed value rounds to the printed value -6,MolSpectra,KS:ci95,paper,0.001,0.000974,0.001,recomputed value rounds to the printed value -7,all,jumps(0.1sigma):mean,paper,1839169.111,1839169.111000,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(0.1sigma):ci95,paper,34.666,34.665787,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(0.5sigma):mean,paper,1232341.449,1232341.449000,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(0.5sigma):ci95,paper,61.925,61.924894,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(1sigma):mean,paper,638505.706,638505.706000,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(1sigma):ci95,paper,197.971,197.970570,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(2sigma):mean,paper,102542.184,102542.184000,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(2sigma):ci95,paper,492.257,492.257172,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(3sigma):mean,paper,8929.385,8929.385000,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -7,all,jumps(3sigma):ci95,paper,167.468,167.468492,0.001,recomputed value rounds to the printed value; model-independent; printed with thousands separators in the paper -3,E3FP,Spearman,v2,,0.026068,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,E3FP,Kendall,v2,,0.018248,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,E3FP,CKA,v2,,0.016007,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,E3FP,isoR2,v2,,0.002812,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,E3FP,EJS,v2,,0.184661,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,E3FP,EJS_ROCAUC,v2,,0.531304,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,E3FP,KS,v2,,0.530219,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,FMG,Spearman,v2,,0.013975,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,FMG,Kendall,v2,,0.009319,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,FMG,CKA,v2,,0.009316,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,FMG,isoR2,v2,,0.000924,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,FMG,EJS,v2,,0.267582,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,FMG,EJS_ROCAUC,v2,,0.515996,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,FMG,KS,v2,,0.186257,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,GemNet,Spearman,v2,,0.077526,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,GemNet,Kendall,v2,,0.051798,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,GemNet,CKA,v2,,0.041768,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,GemNet,isoR2,v2,,0.012737,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,GemNet,EJS,v2,,0.359672,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,GemNet,EJS_ROCAUC,v2,,0.591777,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,GemNet,KS,v2,,0.159024,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MACE,Spearman,v2,,0.236143,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MACE,Kendall,v2,,0.159639,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MACE,CKA,v2,,0.175913,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MACE,isoR2,v2,,0.083043,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MACE,EJS,v2,,0.583026,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MACE,EJS_ROCAUC,v2,,0.760630,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MACE,KS,v2,,0.173223,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolAE,Spearman,v2,,0.038868,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolAE,Kendall,v2,,0.025923,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolAE,CKA,v2,,0.031395,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolAE,isoR2,v2,,0.003183,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolAE,EJS,v2,,0.295529,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolAE,EJS_ROCAUC,v2,,0.545296,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolAE,KS,v2,,0.325364,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolSpectra,Spearman,v2,,0.022909,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolSpectra,Kendall,v2,,0.015279,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolSpectra,CKA,v2,,0.013705,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolSpectra,isoR2,v2,,0.001810,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolSpectra,EJS,v2,,0.271520,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolSpectra,EJS_ROCAUC,v2,,0.526507,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,MolSpectra,KS,v2,,0.261547,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,UniMol,Spearman,v2,,0.042610,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,UniMol,Kendall,v2,,0.028420,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,UniMol,CKA,v2,,0.032049,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,UniMol,isoR2,v2,,0.003165,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,UniMol,EJS,v2,,0.303481,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,UniMol,EJS_ROCAUC,v2,,0.549794,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -3,UniMol,KS,v2,,0.301884,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,Spearman:mean,v2,,0.026068,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,Spearman:ci95,v2,,0.001085,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,Kendall:mean,v2,,0.018248,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,Kendall:ci95,v2,,0.000751,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,CKA:mean,v2,,0.016007,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,CKA:ci95,v2,,0.000732,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,isoR2:mean,v2,,0.002812,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,isoR2:ci95,v2,,0.000145,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(0.1sigma):mean,v2,,0.150929,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(0.1sigma):ci95,v2,,0.005522,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(0.5sigma):mean,v2,,0.154680,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(0.5sigma):ci95,v2,,0.005623,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(1sigma):mean,v2,,0.161898,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(1sigma):ci95,v2,,0.005840,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(2sigma):mean,v2,,0.184661,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(2sigma):ci95,v2,,0.006718,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(3sigma):mean,v2,,0.219369,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,EJS(3sigma):ci95,v2,,0.008872,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,ROC_AUC:mean,v2,,0.531304,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,ROC_AUC:ci95,v2,,0.001400,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,KS:mean,v2,,0.530219,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,E3FP,KS:ci95,v2,,0.008080,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,Spearman:mean,v2,,0.013975,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,Spearman:ci95,v2,,0.000781,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,Kendall:mean,v2,,0.009319,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,Kendall:ci95,v2,,0.000520,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,CKA:mean,v2,,0.009316,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,CKA:ci95,v2,,0.000441,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,isoR2:mean,v2,,0.000924,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,isoR2:ci95,v2,,0.000065,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(0.1sigma):mean,v2,,0.250330,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(0.1sigma):ci95,v2,,0.000028,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(0.5sigma):mean,v2,,0.252497,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(0.5sigma):ci95,v2,,0.000171,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(1sigma):mean,v2,,0.256485,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(1sigma):ci95,v2,,0.000432,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(2sigma):mean,v2,,0.267582,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(2sigma):ci95,v2,,0.001228,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(3sigma):mean,v2,,0.282903,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,EJS(3sigma):ci95,v2,,0.003169,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,ROC_AUC:mean,v2,,0.515996,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,ROC_AUC:ci95,v2,,0.000930,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,KS:mean,v2,,0.186257,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,FMG,KS:ci95,v2,,0.002900,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,Spearman:mean,v2,,0.077526,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,Spearman:ci95,v2,,0.002243,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,Kendall:mean,v2,,0.051798,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,Kendall:ci95,v2,,0.001504,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,CKA:mean,v2,,0.041768,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,CKA:ci95,v2,,0.001230,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,isoR2:mean,v2,,0.012737,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,isoR2:ci95,v2,,0.000859,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(0.1sigma):mean,v2,,0.252056,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(0.1sigma):ci95,v2,,0.000070,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(0.5sigma):mean,v2,,0.263785,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(0.5sigma):ci95,v2,,0.000468,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(1sigma):mean,v2,,0.286809,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(1sigma):ci95,v2,,0.001253,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(2sigma):mean,v2,,0.359672,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(2sigma):ci95,v2,,0.003771,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(3sigma):mean,v2,,0.456857,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,EJS(3sigma):ci95,v2,,0.007447,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,ROC_AUC:mean,v2,,0.591777,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,ROC_AUC:ci95,v2,,0.002668,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,KS:mean,v2,,0.159024,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,GemNet,KS:ci95,v2,,0.003895,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,Spearman:mean,v2,,0.236143,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,Spearman:ci95,v2,,0.002361,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,Kendall:mean,v2,,0.159639,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,Kendall:ci95,v2,,0.001625,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,CKA:mean,v2,,0.175913,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,CKA:ci95,v2,,0.001936,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,isoR2:mean,v2,,0.083043,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,isoR2:ci95,v2,,0.001556,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(0.1sigma):mean,v2,,0.255564,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(0.1sigma):ci95,v2,,0.000084,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(0.5sigma):mean,v2,,0.287487,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(0.5sigma):ci95,v2,,0.000562,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(1sigma):mean,v2,,0.353460,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(1sigma):ci95,v2,,0.001528,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(2sigma):mean,v2,,0.583026,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(2sigma):ci95,v2,,0.004338,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(3sigma):mean,v2,,0.849128,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,EJS(3sigma):ci95,v2,,0.005353,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,ROC_AUC:mean,v2,,0.760630,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,ROC_AUC:ci95,v2,,0.002222,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,KS:mean,v2,,0.173223,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MACE,KS:ci95,v2,,0.003402,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,Spearman:mean,v2,,0.038868,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,Spearman:ci95,v2,,0.001260,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,Kendall:mean,v2,,0.025923,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,Kendall:ci95,v2,,0.000840,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,CKA:mean,v2,,0.031395,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,CKA:ci95,v2,,0.001029,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,isoR2:mean,v2,,0.003183,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,isoR2:ci95,v2,,0.000162,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(0.1sigma):mean,v2,,0.250940,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(0.1sigma):ci95,v2,,0.000043,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(0.5sigma):mean,v2,,0.256201,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(0.5sigma):ci95,v2,,0.000277,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(1sigma):mean,v2,,0.266207,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(1sigma):ci95,v2,,0.000713,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(2sigma):mean,v2,,0.295529,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(2sigma):ci95,v2,,0.002003,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(3sigma):mean,v2,,0.331006,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,EJS(3sigma):ci95,v2,,0.004616,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,ROC_AUC:mean,v2,,0.545296,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,ROC_AUC:ci95,v2,,0.001443,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,KS:mean,v2,,0.325364,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolAE,KS:ci95,v2,,0.003224,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,Spearman:mean,v2,,0.022909,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,Spearman:ci95,v2,,0.001254,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,Kendall:mean,v2,,0.015279,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,Kendall:ci95,v2,,0.000836,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,CKA:mean,v2,,0.013705,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,CKA:ci95,v2,,0.000523,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,isoR2:mean,v2,,0.001810,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,isoR2:ci95,v2,,0.000164,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(0.1sigma):mean,v2,,0.250465,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(0.1sigma):ci95,v2,,0.000041,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(0.5sigma):mean,v2,,0.253109,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(0.5sigma):ci95,v2,,0.000268,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(1sigma):mean,v2,,0.258028,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(1sigma):ci95,v2,,0.000686,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(2sigma):mean,v2,,0.271520,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(2sigma):ci95,v2,,0.001857,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(3sigma):mean,v2,,0.290743,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,EJS(3sigma):ci95,v2,,0.004092,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,ROC_AUC:mean,v2,,0.526507,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,ROC_AUC:ci95,v2,,0.001402,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,KS:mean,v2,,0.261547,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -6,MolSpectra,KS:ci95,v2,,0.006749,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(0.1sigma):mean,v2,,1836048.939000,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(0.1sigma):ci95,v2,,156.743292,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(0.5sigma):mean,v2,,1218701.243000,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(0.5sigma):ci95,v2,,625.179967,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(1sigma):mean,v2,,620312.428000,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(1sigma):ci95,v2,,591.198447,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(2sigma):mean,v2,,94275.340000,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(2sigma):ci95,v2,,163.304862,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(3sigma):mean,v2,,7494.103000,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart -7,all,jumps(3sigma):ci95,v2,,105.603263,0.001,v2 metric definitions (docs/metrics/energy.md); no published counterpart +3,E3FP,Spearman,paper,0.026,0.026068,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,E3FP,Kendall,paper,0.018,0.018248,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,E3FP,CKA,paper,0.011,0.010683,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,E3FP,isoR2,paper,0.002,0.001507,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,E3FP,EJS,paper,0.184,0.183794,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,E3FP,EJS_ROCAUC,paper,0.531,0.531304,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,E3FP,TS,paper,0.385,0.385347,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,E3FP,KS,paper,0.916,0.913498,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,Spearman,paper,0.015,0.013975,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,Kendall,paper,0.009,0.009321,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,CKA,paper,0.011,0.011691,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,isoR2,paper,0.001,0.000822,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,EJS,paper,0.269,0.266977,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,EJS_ROCAUC,paper,0.517,0.515996,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,TS,paper,0.582,0.582899,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,FMG,KS,paper,0.999,0.999827,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,Spearman,paper,0.078,0.077526,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,Kendall,paper,0.052,0.051807,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,CKA,paper,0.016,0.016325,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,isoR2,paper,0.013,0.013470,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,EJS,paper,0.356,0.355988,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,EJS_ROCAUC,paper,0.592,0.591777,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,TS,paper,0.444,0.443715,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,GemNet,KS,paper,0.998,0.998367,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,Spearman,paper,0.236,0.236143,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,Kendall,paper,0.159,0.159662,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,CKA,paper,0.017,0.017391,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,isoR2,paper,0.080,0.080723,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,EJS,paper,0.578,0.572063,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,EJS_ROCAUC,paper,0.764,0.760630,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,TS,paper,0.424,0.424343,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MACE,KS,paper,0.999,0.999448,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,Spearman,paper,0.039,0.038868,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,Kendall,paper,0.026,0.025930,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,CKA,paper,0.024,0.024077,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,isoR2,paper,0.003,0.002824,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,EJS,paper,0.294,0.293714,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,EJS_ROCAUC,paper,0.545,0.545296,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,TS,paper,0.346,0.346272,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolAE,KS,paper,0.998,0.998199,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,Spearman,paper,0.023,0.022909,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,Kendall,paper,0.015,0.015282,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,CKA,paper,0.006,0.006444,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,isoR2,paper,0.001,0.001481,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,EJS,paper,0.271,0.270563,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,EJS_ROCAUC,paper,0.526,0.526507,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,TS,paper,0.409,0.408949,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,MolSpectra,KS,paper,0.977,0.976430,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,Spearman,paper,0.043,0.042610,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,Kendall,paper,0.028,0.028426,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,CKA,paper,0.019,0.018778,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,isoR2,paper,0.003,0.003016,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,EJS,paper,0.301,0.301547,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,EJS_ROCAUC,paper,0.549,0.549794,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,TS,paper,0.356,0.356314,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +3,UniMol,KS,paper,0.997,0.996900,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,Spearman:mean,paper,0.026,0.026068,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,Spearman:ci95,paper,0.001,0.001085,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,Kendall:mean,paper,0.018,0.018248,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,Kendall:ci95,paper,0.001,0.000751,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,CKA:mean,paper,0.011,0.010683,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,CKA:ci95,paper,0.000,0.000383,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,isoR2:mean,paper,0.002,0.001507,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,isoR2:ci95,paper,0.000,0.000084,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(0.1sigma):mean,paper,0.151,0.150918,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(0.1sigma):ci95,paper,0.006,0.005522,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(0.5sigma):mean,paper,0.155,0.154587,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(0.5sigma):ci95,paper,0.006,0.005622,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(1sigma):mean,paper,0.162,0.161624,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(1sigma):ci95,paper,0.006,0.005834,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(2sigma):mean,paper,0.184,0.183794,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(2sigma):ci95,paper,0.007,0.006687,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(3sigma):mean,paper,0.217,0.217036,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,EJS(3sigma):ci95,paper,0.009,0.008698,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,ROC_AUC:mean,paper,0.531,0.531304,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,ROC_AUC:ci95,paper,0.001,0.001400,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,Smoothness:mean,paper,0.847,0.846836,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,Smoothness:ci95,paper,0.002,0.002126,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,TS:mean,paper,0.385,0.385347,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,TS:ci95,paper,0.010,0.010232,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,KS:mean,paper,0.913,0.913498,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,E3FP,KS:ci95,paper,0.001,0.000972,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,Spearman:mean,paper,0.015,0.013975,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,Spearman:ci95,paper,0.004,0.000781,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,Kendall:mean,paper,0.010,0.009321,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,Kendall:ci95,paper,0.003,0.000521,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,CKA:mean,paper,0.012,0.011691,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,CKA:ci95,paper,0.001,0.000167,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,isoR2:mean,paper,0.001,0.000822,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,isoR2:ci95,paper,0.000,0.000064,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(0.1sigma):mean,paper,0.250,0.250204,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(0.1sigma):ci95,paper,0.000,0.000031,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(0.5sigma):mean,paper,0.253,0.252309,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(0.5sigma):ci95,paper,0.001,0.000168,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(1sigma):mean,paper,0.257,0.256173,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(1sigma):ci95,paper,0.002,0.000420,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(2sigma):mean,paper,0.269,0.266977,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(2sigma):ci95,paper,0.006,0.001189,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(3sigma):mean,paper,0.283,0.281762,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,EJS(3sigma):ci95,paper,0.016,0.002975,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,ROC_AUC:mean,paper,0.517,0.515996,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,ROC_AUC:ci95,paper,0.005,0.000930,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,Smoothness:mean,paper,0.972,0.999889,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,Smoothness:ci95,paper,0.002,0.000007,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,TS:mean,paper,0.583,0.582899,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,TS:ci95,paper,0.007,0.001634,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,KS:mean,paper,1.000,0.999827,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,FMG,KS:ci95,paper,0.000,0.000015,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,Spearman:mean,paper,0.078,0.077526,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,Spearman:ci95,paper,0.002,0.002243,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,Kendall:mean,paper,0.052,0.051807,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,Kendall:ci95,paper,0.002,0.001504,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,CKA:mean,paper,0.016,0.016325,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,CKA:ci95,paper,0.000,0.000417,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,isoR2:mean,paper,0.013,0.013470,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,isoR2:ci95,paper,0.001,0.000981,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(0.1sigma):mean,paper,0.252,0.251834,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(0.1sigma):ci95,paper,0.000,0.000067,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(0.5sigma):mean,paper,0.263,0.263241,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(0.5sigma):ci95,paper,0.000,0.000446,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(1sigma):mean,paper,0.286,0.285514,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(1sigma):ci95,paper,0.001,0.001190,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(2sigma):mean,paper,0.356,0.355988,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(2sigma):ci95,paper,0.004,0.003577,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(3sigma):mean,paper,0.451,0.450997,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,EJS(3sigma):ci95,paper,0.007,0.007127,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,ROC_AUC:mean,paper,0.592,0.591777,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,ROC_AUC:ci95,paper,0.003,0.002668,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,Smoothness:mean,paper,0.998,0.998238,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,Smoothness:ci95,paper,0.000,0.000188,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,TS:mean,paper,0.444,0.443715,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,TS:ci95,paper,0.004,0.003603,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,KS:mean,paper,0.998,0.998367,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,GemNet,KS:ci95,paper,0.000,0.000172,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,Spearman:mean,paper,0.236,0.236143,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,Spearman:ci95,paper,0.011,0.002361,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,Kendall:mean,paper,0.160,0.159662,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,Kendall:ci95,paper,0.008,0.001626,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,CKA:mean,paper,0.017,0.017391,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,CKA:ci95,paper,0.001,0.000285,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,isoR2:mean,paper,0.083,0.080723,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,isoR2:ci95,paper,0.010,0.002210,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(0.1sigma):mean,paper,0.255,0.255266,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(0.1sigma):ci95,paper,0.000,0.000079,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(0.5sigma):mean,paper,0.286,0.286281,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(0.5sigma):ci95,paper,0.002,0.000527,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(1sigma):mean,paper,0.350,0.349974,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(1sigma):ci95,paper,0.007,0.001425,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(2sigma):mean,paper,0.578,0.572063,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(2sigma):ci95,paper,0.019,0.004105,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(3sigma):mean,paper,0.845,0.837292,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,EJS(3sigma):ci95,paper,0.025,0.005381,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,ROC_AUC:mean,paper,0.764,0.760630,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,ROC_AUC:ci95,paper,0.011,0.002222,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,Smoothness:mean,paper,0.999,0.999394,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,Smoothness:ci95,paper,0.000,0.000031,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,TS:mean,paper,0.424,0.424343,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,TS:ci95,paper,0.014,0.003126,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,KS:mean,paper,0.999,0.999448,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MACE,KS:ci95,paper,0.000,0.000022,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,Spearman:mean,paper,0.039,0.038868,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,Spearman:ci95,paper,0.001,0.001260,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,Kendall:mean,paper,0.026,0.025930,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,Kendall:ci95,paper,0.001,0.000841,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,CKA:mean,paper,0.024,0.024077,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,CKA:ci95,paper,0.000,0.000342,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,isoR2:mean,paper,0.003,0.002824,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,isoR2:ci95,paper,0.000,0.000180,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(0.1sigma):mean,paper,0.251,0.250616,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(0.1sigma):ci95,paper,0.000,0.000043,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(0.5sigma):mean,paper,0.256,0.255715,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(0.5sigma):ci95,paper,0.000,0.000266,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(1sigma):mean,paper,0.265,0.265368,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(1sigma):ci95,paper,0.001,0.000683,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(2sigma):mean,paper,0.294,0.293714,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(2sigma):ci95,paper,0.002,0.001914,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(3sigma):mean,paper,0.328,0.327981,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,EJS(3sigma):ci95,paper,0.004,0.004323,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,ROC_AUC:mean,paper,0.545,0.545296,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,ROC_AUC:ci95,paper,0.001,0.001443,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,Smoothness:mean,paper,0.996,0.996316,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,Smoothness:ci95,paper,0.000,0.000125,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,TS:mean,paper,0.346,0.346272,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,TS:ci95,paper,0.001,0.001449,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,KS:mean,paper,0.998,0.998199,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolAE,KS:ci95,paper,0.000,0.000061,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,Spearman:mean,paper,0.023,0.022909,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,Spearman:ci95,paper,0.001,0.001254,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,Kendall:mean,paper,0.015,0.015282,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,Kendall:ci95,paper,0.001,0.000836,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,CKA:mean,paper,0.006,0.006444,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,CKA:ci95,paper,0.000,0.000169,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,isoR2:mean,paper,0.001,0.001481,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,isoR2:ci95,paper,0.000,0.000174,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(0.1sigma):mean,paper,0.250,0.250276,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(0.1sigma):ci95,paper,0.000,0.000041,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(0.5sigma):mean,paper,0.253,0.252824,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(0.5sigma):ci95,paper,0.000,0.000258,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(1sigma):mean,paper,0.258,0.257560,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(1sigma):ci95,paper,0.001,0.000658,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(2sigma):mean,paper,0.271,0.270563,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(2sigma):ci95,paper,0.002,0.001780,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(3sigma):mean,paper,0.289,0.289048,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,EJS(3sigma):ci95,paper,0.004,0.003845,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,ROC_AUC:mean,paper,0.527,0.526507,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,ROC_AUC:ci95,paper,0.001,0.001402,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,Smoothness:mean,paper,0.974,0.973790,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,Smoothness:ci95,paper,0.001,0.000667,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,TS:mean,paper,0.409,0.408949,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,TS:ci95,paper,0.004,0.004161,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,KS:mean,paper,0.976,0.976430,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +6,MolSpectra,KS:ci95,paper,0.001,0.000974,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025)" +7,all,jumps(0.1sigma):mean,paper,1839169.111,1839169.111000,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(0.1sigma):ci95,paper,34.666,34.665787,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(0.5sigma):mean,paper,1232341.449,1232341.449000,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(0.5sigma):ci95,paper,61.925,61.924894,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(1sigma):mean,paper,638505.706,638505.706000,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(1sigma):ci95,paper,197.971,197.970570,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(2sigma):mean,paper,102542.184,102542.184000,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(2sigma):ci95,paper,492.257,492.257172,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(3sigma):mean,paper,8929.385,8929.385000,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +7,all,jumps(3sigma):ci95,paper,167.468,167.468492,0.001,"paper metric definitions (docs/metrics/energy.md); reference value computed with this code from the published trajectory embeddings and the float64 rMD17 energies (legacy windows, 100 x 2000 frames, seed 2025); jump counts depend only on the energies and the windows, not on the representation; the paper prints them with thousands separators" +3,E3FP,Spearman,v2,,0.026068,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,E3FP,Kendall,v2,,0.018248,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,E3FP,CKA,v2,,0.016007,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,E3FP,isoR2,v2,,0.002812,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,E3FP,EJS,v2,,0.184661,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,E3FP,EJS_ROCAUC,v2,,0.531304,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,E3FP,KS,v2,,0.530219,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,FMG,Spearman,v2,,0.013975,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,FMG,Kendall,v2,,0.009319,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,FMG,CKA,v2,,0.009316,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,FMG,isoR2,v2,,0.000924,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,FMG,EJS,v2,,0.267582,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,FMG,EJS_ROCAUC,v2,,0.515996,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,FMG,KS,v2,,0.186257,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,GemNet,Spearman,v2,,0.077526,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,GemNet,Kendall,v2,,0.051798,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,GemNet,CKA,v2,,0.041768,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,GemNet,isoR2,v2,,0.012737,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,GemNet,EJS,v2,,0.359672,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,GemNet,EJS_ROCAUC,v2,,0.591777,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,GemNet,KS,v2,,0.159024,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MACE,Spearman,v2,,0.236143,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MACE,Kendall,v2,,0.159639,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MACE,CKA,v2,,0.175913,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MACE,isoR2,v2,,0.083043,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MACE,EJS,v2,,0.583026,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MACE,EJS_ROCAUC,v2,,0.760630,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MACE,KS,v2,,0.173223,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolAE,Spearman,v2,,0.038868,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolAE,Kendall,v2,,0.025923,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolAE,CKA,v2,,0.031395,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolAE,isoR2,v2,,0.003183,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolAE,EJS,v2,,0.295529,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolAE,EJS_ROCAUC,v2,,0.545296,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolAE,KS,v2,,0.325364,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolSpectra,Spearman,v2,,0.022909,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolSpectra,Kendall,v2,,0.015279,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolSpectra,CKA,v2,,0.013705,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolSpectra,isoR2,v2,,0.001810,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolSpectra,EJS,v2,,0.271520,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolSpectra,EJS_ROCAUC,v2,,0.526507,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,MolSpectra,KS,v2,,0.261547,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,UniMol,Spearman,v2,,0.042610,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,UniMol,Kendall,v2,,0.028420,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,UniMol,CKA,v2,,0.032049,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,UniMol,isoR2,v2,,0.003165,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,UniMol,EJS,v2,,0.303481,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,UniMol,EJS_ROCAUC,v2,,0.549794,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +3,UniMol,KS,v2,,0.301884,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,Spearman:mean,v2,,0.026068,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,Spearman:ci95,v2,,0.001085,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,Kendall:mean,v2,,0.018248,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,Kendall:ci95,v2,,0.000751,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,CKA:mean,v2,,0.016007,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,CKA:ci95,v2,,0.000732,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,isoR2:mean,v2,,0.002812,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,isoR2:ci95,v2,,0.000145,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(0.1sigma):mean,v2,,0.150929,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(0.1sigma):ci95,v2,,0.005522,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(0.5sigma):mean,v2,,0.154680,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(0.5sigma):ci95,v2,,0.005623,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(1sigma):mean,v2,,0.161898,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(1sigma):ci95,v2,,0.005840,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(2sigma):mean,v2,,0.184661,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(2sigma):ci95,v2,,0.006718,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(3sigma):mean,v2,,0.219369,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,EJS(3sigma):ci95,v2,,0.008872,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,ROC_AUC:mean,v2,,0.531304,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,ROC_AUC:ci95,v2,,0.001400,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,KS:mean,v2,,0.530219,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,E3FP,KS:ci95,v2,,0.008080,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,Spearman:mean,v2,,0.013975,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,Spearman:ci95,v2,,0.000781,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,Kendall:mean,v2,,0.009319,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,Kendall:ci95,v2,,0.000520,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,CKA:mean,v2,,0.009316,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,CKA:ci95,v2,,0.000441,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,isoR2:mean,v2,,0.000924,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,isoR2:ci95,v2,,0.000065,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(0.1sigma):mean,v2,,0.250330,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(0.1sigma):ci95,v2,,0.000028,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(0.5sigma):mean,v2,,0.252497,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(0.5sigma):ci95,v2,,0.000171,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(1sigma):mean,v2,,0.256485,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(1sigma):ci95,v2,,0.000432,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(2sigma):mean,v2,,0.267582,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(2sigma):ci95,v2,,0.001228,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(3sigma):mean,v2,,0.282903,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,EJS(3sigma):ci95,v2,,0.003169,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,ROC_AUC:mean,v2,,0.515996,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,ROC_AUC:ci95,v2,,0.000930,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,KS:mean,v2,,0.186257,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,FMG,KS:ci95,v2,,0.002900,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,Spearman:mean,v2,,0.077526,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,Spearman:ci95,v2,,0.002243,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,Kendall:mean,v2,,0.051798,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,Kendall:ci95,v2,,0.001504,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,CKA:mean,v2,,0.041768,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,CKA:ci95,v2,,0.001230,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,isoR2:mean,v2,,0.012737,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,isoR2:ci95,v2,,0.000859,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(0.1sigma):mean,v2,,0.252056,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(0.1sigma):ci95,v2,,0.000070,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(0.5sigma):mean,v2,,0.263785,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(0.5sigma):ci95,v2,,0.000468,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(1sigma):mean,v2,,0.286809,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(1sigma):ci95,v2,,0.001253,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(2sigma):mean,v2,,0.359672,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(2sigma):ci95,v2,,0.003771,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(3sigma):mean,v2,,0.456857,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,EJS(3sigma):ci95,v2,,0.007447,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,ROC_AUC:mean,v2,,0.591777,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,ROC_AUC:ci95,v2,,0.002668,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,KS:mean,v2,,0.159024,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,GemNet,KS:ci95,v2,,0.003895,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,Spearman:mean,v2,,0.236143,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,Spearman:ci95,v2,,0.002361,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,Kendall:mean,v2,,0.159639,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,Kendall:ci95,v2,,0.001625,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,CKA:mean,v2,,0.175913,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,CKA:ci95,v2,,0.001936,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,isoR2:mean,v2,,0.083043,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,isoR2:ci95,v2,,0.001556,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(0.1sigma):mean,v2,,0.255564,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(0.1sigma):ci95,v2,,0.000084,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(0.5sigma):mean,v2,,0.287487,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(0.5sigma):ci95,v2,,0.000562,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(1sigma):mean,v2,,0.353460,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(1sigma):ci95,v2,,0.001528,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(2sigma):mean,v2,,0.583026,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(2sigma):ci95,v2,,0.004338,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(3sigma):mean,v2,,0.849128,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,EJS(3sigma):ci95,v2,,0.005353,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,ROC_AUC:mean,v2,,0.760630,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,ROC_AUC:ci95,v2,,0.002222,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,KS:mean,v2,,0.173223,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MACE,KS:ci95,v2,,0.003402,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,Spearman:mean,v2,,0.038868,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,Spearman:ci95,v2,,0.001260,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,Kendall:mean,v2,,0.025923,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,Kendall:ci95,v2,,0.000840,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,CKA:mean,v2,,0.031395,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,CKA:ci95,v2,,0.001029,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,isoR2:mean,v2,,0.003183,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,isoR2:ci95,v2,,0.000162,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(0.1sigma):mean,v2,,0.250940,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(0.1sigma):ci95,v2,,0.000043,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(0.5sigma):mean,v2,,0.256201,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(0.5sigma):ci95,v2,,0.000277,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(1sigma):mean,v2,,0.266207,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(1sigma):ci95,v2,,0.000713,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(2sigma):mean,v2,,0.295529,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(2sigma):ci95,v2,,0.002003,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(3sigma):mean,v2,,0.331006,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,EJS(3sigma):ci95,v2,,0.004616,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,ROC_AUC:mean,v2,,0.545296,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,ROC_AUC:ci95,v2,,0.001443,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,KS:mean,v2,,0.325364,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolAE,KS:ci95,v2,,0.003224,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,Spearman:mean,v2,,0.022909,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,Spearman:ci95,v2,,0.001254,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,Kendall:mean,v2,,0.015279,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,Kendall:ci95,v2,,0.000836,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,CKA:mean,v2,,0.013705,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,CKA:ci95,v2,,0.000523,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,isoR2:mean,v2,,0.001810,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,isoR2:ci95,v2,,0.000164,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(0.1sigma):mean,v2,,0.250465,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(0.1sigma):ci95,v2,,0.000041,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(0.5sigma):mean,v2,,0.253109,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(0.5sigma):ci95,v2,,0.000268,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(1sigma):mean,v2,,0.258028,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(1sigma):ci95,v2,,0.000686,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(2sigma):mean,v2,,0.271520,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(2sigma):ci95,v2,,0.001857,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(3sigma):mean,v2,,0.290743,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,EJS(3sigma):ci95,v2,,0.004092,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,ROC_AUC:mean,v2,,0.526507,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,ROC_AUC:ci95,v2,,0.001402,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,KS:mean,v2,,0.261547,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +6,MolSpectra,KS:ci95,v2,,0.006749,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(0.1sigma):mean,v2,,1836048.939000,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(0.1sigma):ci95,v2,,156.743292,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(0.5sigma):mean,v2,,1218701.243000,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(0.5sigma):ci95,v2,,625.179967,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(1sigma):mean,v2,,620312.428000,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(1sigma):ci95,v2,,591.198447,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(2sigma):mean,v2,,94275.340000,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(2sigma):ci95,v2,,163.304862,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(3sigma):mean,v2,,7494.103000,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper +7,all,jumps(3sigma):ci95,v2,,105.603263,0.001,v2 metric definitions (docs/metrics/energy.md); not printed in the paper diff --git a/reproduce/energy_tables_3_6_7/make_expected.py b/reproduce/energy_tables_3_6_7/make_expected.py index 39b8559..8b10e0b 100644 --- a/reproduce/energy_tables_3_6_7/make_expected.py +++ b/reproduce/energy_tables_3_6_7/make_expected.py @@ -2,9 +2,9 @@ python make_expected.py --results out/results.csv --paper paper_values.csv --out expected.csv -expected_value is the recomputed value rounded to 6 decimals. For variant ``paper`` each cell is -classified against the printed value (3 decimals): it rounds to the printed value, matches it only by -truncation, or differs from it. Known issues are added to ``notes``. +``expected_value`` is the reference value of this release, rounded to 6 decimals; ``paper_value`` is +the value as printed in the paper; ``notes`` describes the metric version and the row. The console +summary reports, for the ``paper`` rows, how each reference value relates to the printed precision. """ from __future__ import annotations @@ -18,17 +18,16 @@ TOLERANCE = 0.001 -T3_VS_T6 = { # Table 3 cells printed differently from the same Table 6 mean - ("MACE", "Kendall"): ("0.159", "0.160"), - ("MACE", "isoR2"): ("0.080", "0.083"), - ("FMG", "Kendall"): ("0.009", "0.010"), - ("FMG", "CKA"): ("0.011", "0.012"), - ("FMG", "TS"): ("0.582", "0.583"), - ("FMG", "KS"): ("0.999", "1.000"), - ("MolSpectra", "EJS_ROCAUC"): ("0.526", "0.527"), - ("MolSpectra", "KS"): ("0.977", "0.976"), - ("E3FP", "KS"): ("0.916", "0.913"), -} +PAPER_NOTE = ( + "paper metric definitions (docs/metrics/energy.md); reference value computed with this code from " + "the published trajectory embeddings and the float64 rMD17 energies (legacy windows, " + "100 x 2000 frames, seed 2025)" +) +V2_NOTE = "v2 metric definitions (docs/metrics/energy.md); not printed in the paper" +TABLE7_NOTE = ( + "jump counts depend only on the energies and the windows, not on the representation; the paper " + "prints them with thousands separators" +) def _decimals(printed: str) -> int: @@ -46,44 +45,12 @@ def classify(value: float, printed: str) -> str: def notes_for(table: str, model: str, metric: str, value: float, printed: str | None) -> str: + """Describe what the row computes (metric version, variant, provenance of the reference value).""" if printed is None: return "" - status = classify(value, printed) - notes = [] - if status == "round": - notes.append("recomputed value rounds to the printed value") - elif status == "trunc": - notes.append("printed value is the 3-dp truncation (not rounding) of the recomputed value") - else: - notes.append("recomputed value does not match the printed value") - - base = metric.split(":")[0] - stat = metric.split(":")[1] if ":" in metric else "mean" - if table == "3" and (model, base) in T3_VS_T6: - t3, t6 = T3_VS_T6[(model, base)] - notes.append(f"Table 3 prints {t3} while Table 6 prints {t6} for the same mean") - if table == "3" and base == "KS" and model in ("E3FP", "MolSpectra"): - notes.append( - "the Table 3 value is carried over from the submission draft and is not produced by any backed-up run" - ) - if model in ("MACE", "FMG") and status == "mismatch": - if stat == "ci95": - notes.append( - "published MACE/FMG CIs are about 4-5x wider than this 1000-window run (consistent with roughly " - "40-47 windows); no backed-up run reproduces the published MACE/FMG CIs" - ) - else: - notes.append( - "MACE/FMG energy columns are only partially reproducible from the backed-up embeddings; no " - "backed-up run reproduces this published value" - ) - if model == "FMG" and base == "Smoothness": - notes.append( - "published FMG Smoothness 0.972 +/- 0.002 is not attainable with the backed-up FMG embeddings: " - "every one of the 1000 per-window values is >= 0.999" - ) + notes = [PAPER_NOTE] if table == "7": - notes.append("model-independent; printed with thousands separators in the paper") + notes.append(TABLE7_NOTE) return "; ".join(notes) @@ -105,7 +72,7 @@ def main() -> None: pv = printed.get((r.table, r.model, r.metric)) if r.variant == "paper" else None note = notes_for(r.table, r.model, r.metric, float(r.value), pv) if r.variant != "paper": - note = f"{r.variant} metric definitions (docs/metrics/energy.md); no published counterpart" + note = V2_NOTE if r.variant == "v2" else f"{r.variant} metric definitions (docs/metrics/energy.md)" rows.append( { "table": r.table, diff --git a/reproduce/energy_tables_3_6_7/run.sh b/reproduce/energy_tables_3_6_7/run.sh index eb885d5..bcf7c3e 100755 --- a/reproduce/energy_tables_3_6_7/run.sh +++ b/reproduce/energy_tables_3_6_7/run.sh @@ -9,7 +9,7 @@ # # Environment variables (all optional): # PYTHON=python OUT=/out N_JOBS=16 MODELS="e3fp gemnet molae molspectra unimol fmg mace" -# METRIC_VERSIONS="paper" (add v2 for the corrected definitions: METRIC_VERSIONS="paper v2") +# METRIC_VERSIONS="paper" (add v2 for the v2 definitions: METRIC_VERSIONS="paper v2") # ENERGY_DATASET_DIR=... use an existing save_to_disk copy of traj_energies instead of downloading # EMB_ROOT=... use existing embeddings: EMB_ROOT//rmd17_.{npz,pkl} # EMB_LAYOUT=hf|nyubox nyubox = original backup directory names (molspec, FMG) diff --git a/reproduce/table1_geometry/README.md b/reproduce/table1_geometry/README.md index d99a47d..99d4b11 100644 --- a/reproduce/table1_geometry/README.md +++ b/reproduce/table1_geometry/README.md @@ -19,25 +19,25 @@ QUICK=1 bash reproduce/table1_geometry/run.sh # shard 1 only | Variant | Rows | Definition | |---|---|---| -| `paper` | all 7 | Spearman, Kendall, CKA, isotonic R², Torsion-SP: mean over the 146,389 molecules in `sampled_molecules_seed2027.txt`; LIE@k and AS: mean over all molecules, with the embedding-offset shift of the original full run (`__offset_drift` columns). This is Table 1. | -| `paper_aligned` | LIE@k, AS | same definitions, correctly aligned embeddings | +| `paper` | all 7 | Spearman, Kendall, CKA, isotonic R², Torsion-SP: mean over the 146,389 molecules in `sampled_molecules_seed2027.txt`; LIE@k and AS: mean over all molecules, with the embedding indexing of the original full run (`__offset_drift` columns). This is Table 1. | +| `paper_aligned` | LIE@k, AS | the same definitions, with the embeddings indexed by the dataset `offset` | | `v2` | all 7 | `--metric-version v2`, mean over all evaluated molecules | -Definitions, provenance and the offset shift are described in -[docs/metrics/geometry.md](../../docs/metrics/geometry.md). +The definitions, the provenance of each row and the embedding indexing of the full run are described +in [docs/metrics/geometry.md](../../docs/metrics/geometry.md). ## Files | File | Content | |---|---| -| `expected.csv` | Table 1 cells for the five models. `expected_value` for `paper` rows = mean of the original per-molecule outputs (`metrics_all_0.1_1.json.gz` for the sampled rows, `metrics_sup_100.json.gz` for LIE@k and AS). Only GemNet can be recomputed; rows of the other models report `MISSING`. The GemNet `paper_aligned` and `v2` reference values need a full 16-shard run and are left empty (`NO_EXPECTED`). | +| `expected.csv` | Table 1 cells for the five models. `expected_value` for `paper` rows = mean of the original per-molecule outputs (`metrics_all_0.1_1.json.gz` for the sampled rows, `metrics_sup_100.json.gz` for LIE@k and AS). GemNet is the model whose rotation embeddings are published, so rows of the other models report `MISSING` unless results are supplied. The GemNet `paper_aligned` and `v2` rows carry no reference value (`NO_EXPECTED`). | | `expected_quick.csv` | shard 1 only: `paper` rows from the original outputs restricted to shard 1; `paper_aligned` and `v2` rows are regression values computed with this code. | -| `sampled_molecules_seed2027.txt` | the 146,389 molecule keys of the original 10 % sample run, in dataset order (read from the backup output; the sampling code is not available). SHA-256 `35da6a9ea84ee1112bd99c43028269cba9035ffa913f0eaafbfba7fab83163ac`. | +| `sampled_molecules_seed2027.txt` | the 146,389 molecule keys of the original 10 % sample run, in dataset order, read from that run's output. SHA-256 `35da6a9ea84ee1112bd99c43028269cba9035ffa913f0eaafbfba7fab83163ac`. | | `collect.py` | per-key parquet → `results.csv` | ## Run time Measured on the A100 box with 24 worker processes (shared with another 24-process job), cosine space only: shard 1 took 129 s with `--metric-version paper --replicate-offset-drift` (including the second -pass over the 80,901 shifted molecules) and 75 s with `--metric-version v2`. A full run is expected to +pass over the 80,901 molecules it covers) and 75 s with `--metric-version v2`. A full run is expected to take about 20–25 min per metric version, plus downloads (dataset ~7.5 GB, embeddings ~5.2 GB). diff --git a/reproduce/table1_geometry/collect.py b/reproduce/table1_geometry/collect.py index 5c886b1..ede5bb4 100755 --- a/reproduce/table1_geometry/collect.py +++ b/reproduce/table1_geometry/collect.py @@ -3,13 +3,13 @@ Table 1 of the paper combines two original runs (see docs/metrics/geometry.md): - Spearman, Kendall, CKA, isotonic R^2 and Torsion-SP are means over the 10 % molecule sample - (``sampled_molecules_seed2027.txt``); that run used correctly aligned embeddings; -- LIE@k and AS are means over all evaluated molecules of the full run, in which the embeddings of - shards 1 and 2 were misaligned after a molecule that failed (``__offset_drift`` columns, - written by ``evaluate rotation --replicate-offset-drift``). + (``sampled_molecules_seed2027.txt``), indexed by the dataset offset; +- LIE@k and AS are means over all evaluated molecules of the full run, whose embedding indexing in + parts of shards 1 and 2 is reproduced by the ``__offset_drift`` columns that + ``evaluate rotation --replicate-offset-drift`` writes. Variant ``paper`` reproduces the published values; variant ``paper_aligned`` gives LIE@k and AS with -correctly aligned embeddings; variant ``v2`` gives all rows (means over all evaluated molecules) -with the corrected definitions. Means exclude non-finite per-molecule values. +the embeddings indexed by the dataset offset; variant ``v2`` gives all rows (means over all evaluated +molecules) with the v2 definitions. Means exclude non-finite per-molecule values. """ from __future__ import annotations diff --git a/reproduce/table1_geometry/expected.csv b/reproduce/table1_geometry/expected.csv index 3794117..9fe13ce 100644 --- a/reproduce/table1_geometry/expected.csv +++ b/reproduce/table1_geometry/expected.csv @@ -1,45 +1,45 @@ -table,model,metric,variant,paper_value,expected_value,tolerance,notes -1,e3fp,spearman,paper,0.406,0.405619,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,gemnet,spearman,paper,0.560,0.559754,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recomputed from the published GemNet rotation embeddings" -1,molae,spearman,paper,0.640,0.639965,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,molspectra,spearman,paper,0.682,0.682009,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,unimol,spearman,paper,0.697,0.697176,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,e3fp,kendall,paper,0.272,0.272296,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,gemnet,kendall,paper,0.336,0.335809,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recomputed from the published GemNet rotation embeddings" -1,molae,kendall,paper,0.483,0.483252,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,molspectra,kendall,paper,0.506,0.505864,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,unimol,kendall,paper,0.563,0.562824,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,e3fp,cka,paper,0.757,0.757292,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,gemnet,cka,paper,0.813,0.813476,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recomputed from the published GemNet rotation embeddings" -1,molae,cka,paper,0.862,0.862460,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,molspectra,cka,paper,0.904,0.903560,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,unimol,cka,paper,0.889,0.889347,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,e3fp,isotonic_r2,paper,0.521,0.521339,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,gemnet,isotonic_r2,paper,0.667,0.666678,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recomputed from the published GemNet rotation embeddings" -1,molae,isotonic_r2,paper,0.734,0.734246,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,molspectra,isotonic_r2,paper,0.770,0.770137,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,unimol,isotonic_r2,paper,0.782,0.782243,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,e3fp,lie_k,paper,0.324,0.323791,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,gemnet,lie_k,paper,0.390,0.390109,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); recomputed from the published GemNet rotation embeddings; requires --replicate-offset-drift (shards 1 and 2 of the original run used shifted embeddings after a failed molecule; see docs/metrics/geometry.md)" -1,molae,lie_k,paper,0.349,0.348912,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied; published value includes the embedding-offset shift of shards 1-2 (see docs/metrics/geometry.md)" -1,molspectra,lie_k,paper,0.238,0.237903,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied; published value includes the embedding-offset shift of shards 1-2 (see docs/metrics/geometry.md)" -1,unimol,lie_k,paper,0.306,0.305915,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied; published value includes the embedding-offset shift of shards 1-2 (see docs/metrics/geometry.md)" -1,e3fp,torsion_sp,paper,0.313,0.313169,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,gemnet,torsion_sp,paper,0.420,0.420297,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recomputed from the published GemNet rotation embeddings" -1,molae,torsion_sp,paper,0.437,0.436622,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,molspectra,torsion_sp,paper,0.508,0.508405,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,unimol,torsion_sp,paper,0.504,0.504171,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied" -1,e3fp,as,paper,2.757,2.757644,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied; paper value truncated (2.75764)" -1,gemnet,as,paper,0.0018,0.001825,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); recomputed from the published GemNet rotation embeddings; requires --replicate-offset-drift (shards 1 and 2 of the original run used shifted embeddings after a failed molecule; see docs/metrics/geometry.md)" -1,molae,as,paper,0.0048,0.004755,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied; published value includes the embedding-offset shift of shards 1-2 (see docs/metrics/geometry.md)" -1,molspectra,as,paper,0.0214,0.021443,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied; published value includes the embedding-offset shift of shards 1-2 (see docs/metrics/geometry.md)" -1,unimol,as,paper,0.006,0.005987,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); rotation embeddings for this model are not available; row compares only if results are supplied; published value includes the embedding-offset shift of shards 1-2 (see docs/metrics/geometry.md)" -1,gemnet,lie_k,paper_aligned,,,0.001,same definitions as paper with correctly aligned embeddings; reference value pending a full 16-shard run (mean over the molecules not affected by the offset shift: 0.366238) -1,gemnet,as,paper_aligned,,,0.00001,same definitions as paper with correctly aligned embeddings; reference value pending a full 16-shard run (mean over the molecules not affected by the offset shift: 0.001166) -1,gemnet,spearman,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; reference value pending a full 16-shard run" -1,gemnet,kendall,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; reference value pending a full 16-shard run" -1,gemnet,cka,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; reference value pending a full 16-shard run" -1,gemnet,isotonic_r2,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; reference value pending a full 16-shard run" -1,gemnet,lie_k,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; reference value pending a full 16-shard run" -1,gemnet,torsion_sp,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; reference value pending a full 16-shard run" -1,gemnet,as,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; reference value pending a full 16-shard run" +table,model,metric,variant,paper_value,expected_value,tolerance,notes +1,e3fp,spearman,paper,0.406,0.405619,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,gemnet,spearman,paper,0.560,0.559754,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recompute it with the published GemNet rotation embeddings" +1,molae,spearman,paper,0.640,0.639965,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,molspectra,spearman,paper,0.682,0.682009,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,unimol,spearman,paper,0.697,0.697176,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,e3fp,kendall,paper,0.272,0.272296,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,gemnet,kendall,paper,0.336,0.335809,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recompute it with the published GemNet rotation embeddings" +1,molae,kendall,paper,0.483,0.483252,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,molspectra,kendall,paper,0.506,0.505864,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,unimol,kendall,paper,0.563,0.562824,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,e3fp,cka,paper,0.757,0.757292,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,gemnet,cka,paper,0.813,0.813476,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recompute it with the published GemNet rotation embeddings" +1,molae,cka,paper,0.862,0.862460,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,molspectra,cka,paper,0.904,0.903560,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,unimol,cka,paper,0.889,0.889347,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,e3fp,isotonic_r2,paper,0.521,0.521339,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,gemnet,isotonic_r2,paper,0.667,0.666678,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recompute it with the published GemNet rotation embeddings" +1,molae,isotonic_r2,paper,0.734,0.734246,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,molspectra,isotonic_r2,paper,0.770,0.770137,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,unimol,isotonic_r2,paper,0.782,0.782243,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,e3fp,lie_k,paper,0.324,0.323791,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,gemnet,lie_k,paper,0.390,0.390109,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); recompute it with the published GemNet rotation embeddings; run with --replicate-offset-drift: the LIE@k and AS values of the full run use the embedding indexing described in docs/metrics/geometry.md" +1,molae,lie_k,paper,0.349,0.348912,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,molspectra,lie_k,paper,0.238,0.237903,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,unimol,lie_k,paper,0.306,0.305915,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,e3fp,torsion_sp,paper,0.313,0.313169,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,gemnet,torsion_sp,paper,0.420,0.420297,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); recompute it with the published GemNet rotation embeddings" +1,molae,torsion_sp,paper,0.437,0.436622,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,molspectra,torsion_sp,paper,0.508,0.508405,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,unimol,torsion_sp,paper,0.504,0.504171,0.001,"mean over the 146,389 molecules of the 10% sample run (metrics_all_0.1_1.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied" +1,e3fp,as,paper,2.757,2.757644,0.001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,gemnet,as,paper,0.0018,0.001825,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); recompute it with the published GemNet rotation embeddings; run with --replicate-offset-drift: the LIE@k and AS values of the full run use the embedding indexing described in docs/metrics/geometry.md" +1,molae,as,paper,0.0048,0.004755,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,molspectra,as,paper,0.0214,0.021443,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,unimol,as,paper,0.006,0.005987,0.00001,"mean over the 1,464,493 molecules of the full run (metrics_sup_100.json.gz; cosine; E3FP Tanimoto); this release does not publish rotation embeddings for this model, so the row compares only when results are supplied; the full-run values use the embedding indexing described in docs/metrics/geometry.md" +1,gemnet,lie_k,paper_aligned,,,0.001,"same definitions as the paper rows, with the embeddings indexed by the dataset offset; this file holds no reference value for the row" +1,gemnet,as,paper_aligned,,,0.00001,"same definitions as the paper rows, with the embeddings indexed by the dataset offset; this file holds no reference value for the row" +1,gemnet,spearman,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; this file holds no reference value for the row" +1,gemnet,kendall,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; this file holds no reference value for the row" +1,gemnet,cka,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; this file holds no reference value for the row" +1,gemnet,isotonic_r2,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; this file holds no reference value for the row" +1,gemnet,lie_k,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; this file holds no reference value for the row" +1,gemnet,torsion_sp,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; this file holds no reference value for the row" +1,gemnet,as,v2,,,0.001,"metric-version v2, mean over all evaluated molecules; this file holds no reference value for the row" diff --git a/reproduce/table1_geometry/expected_quick.csv b/reproduce/table1_geometry/expected_quick.csv index cd051d0..12c18e5 100644 --- a/reproduce/table1_geometry/expected_quick.csv +++ b/reproduce/table1_geometry/expected_quick.csv @@ -1,17 +1,17 @@ -table,model,metric,variant,paper_value,expected_value,tolerance,notes -1-shard1,gemnet,spearman,paper,,0.569728,0.001,"backup 10% run, mean over its 9,116 shard-1 molecules" -1-shard1,gemnet,kendall,paper,,0.342239,0.001,"backup 10% run, mean over its 9,116 shard-1 molecules" -1-shard1,gemnet,cka,paper,,0.818822,0.001,"backup 10% run, mean over its 9,116 shard-1 molecules" -1-shard1,gemnet,isotonic_r2,paper,,0.674854,0.001,"backup 10% run, mean over its 9,116 shard-1 molecules" -1-shard1,gemnet,lie_k,paper,,0.708220,0.001,"backup full run (with its embedding-offset shift), mean over 91,481 shard-1 molecules; use --replicate-offset-drift" -1-shard1,gemnet,torsion_sp,paper,,0.427353,0.001,"backup 10% run, mean over its 9,116 shard-1 molecules" -1-shard1,gemnet,as,paper,,0.010496,0.00001,"backup full run (with its embedding-offset shift), mean over 91,481 shard-1 molecules; use --replicate-offset-drift" -1-shard1,gemnet,lie_k,paper_aligned,,0.365596,0.001,"regression value computed with this code (A100 box, 24 workers); aligned embeddings, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,as,paper_aligned,,0.001037,0.00001,"regression value computed with this code (A100 box, 24 workers); aligned embeddings, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,spearman,v2,,0.561983,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,kendall,v2,,0.447778,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,cka,v2,,0.763413,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,isotonic_r2,v2,,0.637885,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,lie_k,v2,,0.325691,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,torsion_sp,v2,,0.421115,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" -1-shard1,gemnet,as,v2,,1.814237,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" +table,model,metric,variant,paper_value,expected_value,tolerance,notes +1-shard1,gemnet,spearman,paper,,0.569728,0.001,"original 10 % sample run, mean over its 9,116 shard-1 molecules" +1-shard1,gemnet,kendall,paper,,0.342239,0.001,"original 10 % sample run, mean over its 9,116 shard-1 molecules" +1-shard1,gemnet,cka,paper,,0.818822,0.001,"original 10 % sample run, mean over its 9,116 shard-1 molecules" +1-shard1,gemnet,isotonic_r2,paper,,0.674854,0.001,"original 10 % sample run, mean over its 9,116 shard-1 molecules" +1-shard1,gemnet,lie_k,paper,,0.708220,0.001,"original full run, mean over 91,481 shard-1 molecules; run with --replicate-offset-drift (docs/metrics/geometry.md)" +1-shard1,gemnet,torsion_sp,paper,,0.427353,0.001,"original 10 % sample run, mean over its 9,116 shard-1 molecules" +1-shard1,gemnet,as,paper,,0.010496,0.00001,"original full run, mean over 91,481 shard-1 molecules; run with --replicate-offset-drift (docs/metrics/geometry.md)" +1-shard1,gemnet,lie_k,paper_aligned,,0.365596,0.001,"regression value computed with this code (A100 box, 24 workers); embeddings indexed by the dataset offset, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,as,paper_aligned,,0.001037,0.00001,"regression value computed with this code (A100 box, 24 workers); embeddings indexed by the dataset offset, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,spearman,v2,,0.561983,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,kendall,v2,,0.447778,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,cka,v2,,0.763413,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,isotonic_r2,v2,,0.637885,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,lie_k,v2,,0.325691,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,torsion_sp,v2,,0.421115,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" +1-shard1,gemnet,as,v2,,1.814237,0.001,"regression value computed with this code (A100 box, 24 workers); metric-version v2, all 91,481 evaluated shard-1 molecules" diff --git a/reproduce/table2_chirality/README.md b/reproduce/table2_chirality/README.md index 2435e7b..7ef2fcf 100644 --- a/reproduce/table2_chirality/README.md +++ b/reproduce/table2_chirality/README.md @@ -31,22 +31,19 @@ All variants scan the best-k silhouette up to `k = n - 1` (`--unsup-kmax n-1`, t ## What to expect -- **`euclidean` against the paper.** - - All published cells are within 0.001 of `expected_value`, apart from GemNet SCI. The paper - prints +0.015; the computed value is −0.015151. - - The printed table mixes rounding and truncation. UniMol ES-AUC is 0.622998 and is printed as 0.622. - - E3FP Hopkins is "–" in the paper, because Hopkins is not defined for fingerprints. That cell is not - listed in `expected.csv`. -- **`euclidean` against the Sept-2025 outputs** (`en_sep_results`, in the embeddings repo under - `results/chirality/`). These outputs exist for E3FP, GemNet, MolAE, MolSpectra and UniMol. +- **`euclidean`** is the published protocol. `compare.py` reports PASS for all 34 cells of + `expected.csv` when the run uses the published embeddings. E3FP Hopkins is not defined for + fingerprints (NaN) and has no row. +- **`euclidean` against the original per-molecule outputs** (`en_sep_results`, in the embeddings repo + under `results/chirality/`), which cover E3FP, GemNet, MolAE, MolSpectra and UniMol: - ES-AUC, NN@1-Acc, SCI, Hopkins and DBI are bit-identical for every molecule. - - SCI_unsup is bit-identical for E3FP, MolAE and MolSpectra. - - GemNet and UniMol differ by about 2e-6, because KMeans local optima depend on the BLAS kernel. -- **FMG and MACE** have no original metric output. Their values are recomputed from the published - embeddings. -- **Other variants.** The `cosine` and `v2_*` rows are not in the paper. They are given so that the - alternative definitions can be checked. See `docs/metrics/chirality.md` for the definitions and the - reasons behind them. + - SCI_unsup is bit-identical for E3FP, MolAE and MolSpectra; GemNet and UniMol differ by about + 2e-6, because KMeans local optima depend on the BLAS kernel. + - For FMG and MACE the release has no per-molecule output; their reference values are computed from + the published embeddings. +- **Other variants.** The `cosine` and `v2_*` rows are not printed in the paper. They are given so + that the alternative definitions can be checked. See `docs/metrics/chirality.md` for what each one + computes. ## Run time diff --git a/reproduce/table2_chirality/expected.csv b/reproduce/table2_chirality/expected.csv index 79dabcd..85e30e0 100644 --- a/reproduce/table2_chirality/expected.csv +++ b/reproduce/table2_chirality/expected.csv @@ -1,137 +1,137 @@ -table,model,metric,variant,paper_value,expected_value,tolerance,notes -table2,e3fp,ES-AUC,euclidean,0.486,0.485935,0.001,"paper value = round(x, 3); |expected-paper|=0.000065; bit-identical to backup en_sep_results/summary.csv (Sep 2025)" -table2,e3fp,NN@1-Acc,euclidean,0.178,0.177939,0.001,"paper value = round(x, 3); |expected-paper|=0.000061; bit-identical to backup en_sep_results/summary.csv (Sep 2025)" -table2,e3fp,SCI,euclidean,-0.012,-0.012543,0.001,paper value = truncation of x; |expected-paper|=0.000543; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,e3fp,SCI_unsup,euclidean,0.033,0.033825,0.001,paper value = truncation of x; |expected-paper|=0.000825; bit-identical to backup en_sep_results/summary.csv (Sep 2025); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) -table2,gemnet,ES-AUC,euclidean,0.577,0.577410,0.001,round and truncation agree; |expected-paper|=0.000410; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,gemnet,NN@1-Acc,euclidean,0.292,0.291641,0.001,"paper value = round(x, 3); |expected-paper|=0.000359; bit-identical to backup en_sep_results/summary.csv (Sep 2025)" -table2,gemnet,Hopkins,euclidean,0.593,0.592812,0.001,"paper value = round(x, 3); |expected-paper|=0.000188; bit-identical to backup en_sep_results/summary.csv (Sep 2025)" -table2,gemnet,SCI,euclidean,0.015,-0.015151,0.001,paper prints +0.015; the computed value and the Sep-2025 output are -0.0152 (sign typo in the paper); bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,gemnet,SCI_unsup,euclidean,0.272,0.271961,0.001,"paper value = round(x, 3); |expected-paper|=0.000039; backup en_sep_results/summary.csv = 0.271958887 (diff +2.41e-06; KMeans local optimum is BLAS-kernel dependent); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10)" -table2,molae,ES-AUC,euclidean,0.782,0.782544,0.001,paper value = truncation of x; |expected-paper|=0.000544; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,molae,NN@1-Acc,euclidean,0.497,0.497546,0.001,paper value = truncation of x; |expected-paper|=0.000546; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,molae,Hopkins,euclidean,0.602,0.602659,0.001,paper value = truncation of x; |expected-paper|=0.000659; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,molae,SCI,euclidean,0.115,0.115274,0.001,round and truncation agree; |expected-paper|=0.000274; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,molae,SCI_unsup,euclidean,0.247,0.247099,0.001,round and truncation agree; |expected-paper|=0.000099; bit-identical to backup en_sep_results/summary.csv (Sep 2025); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) -table2,molspectra,ES-AUC,euclidean,0.545,0.544744,0.001,"paper value = round(x, 3); |expected-paper|=0.000256; bit-identical to backup en_sep_results/summary.csv (Sep 2025)" -table2,molspectra,NN@1-Acc,euclidean,0.235,0.235250,0.001,round and truncation agree; |expected-paper|=0.000250; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,molspectra,Hopkins,euclidean,0.533,0.532642,0.001,"paper value = round(x, 3); |expected-paper|=0.000358; bit-identical to backup en_sep_results/summary.csv (Sep 2025)" -table2,molspectra,SCI,euclidean,-0.020,-0.020326,0.001,round and truncation agree; |expected-paper|=0.000326; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,molspectra,SCI_unsup,euclidean,0.127,0.127115,0.001,round and truncation agree; |expected-paper|=0.000115; bit-identical to backup en_sep_results/summary.csv (Sep 2025); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) -table2,unimol,ES-AUC,euclidean,0.622,0.622998,0.001,paper value = truncation of x; |expected-paper|=0.000998; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,unimol,NN@1-Acc,euclidean,0.339,0.339582,0.001,paper value = truncation of x; |expected-paper|=0.000582; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,unimol,Hopkins,euclidean,0.559,0.559572,0.001,paper value = truncation of x; |expected-paper|=0.000572; bit-identical to backup en_sep_results/summary.csv (Sep 2025) -table2,unimol,SCI,euclidean,0.012,0.011623,0.001,"paper value = round(x, 3); |expected-paper|=0.000377; bit-identical to backup en_sep_results/summary.csv (Sep 2025)" -table2,unimol,SCI_unsup,euclidean,0.152,0.152108,0.001,round and truncation agree; |expected-paper|=0.000108; backup en_sep_results/summary.csv = 0.152106160 (diff +1.98e-06; KMeans local optimum is BLAS-kernel dependent); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) -table2,fmg,ES-AUC,euclidean,0.706,0.705652,0.001,"paper value = round(x, 3); |expected-paper|=0.000348; no original metric output in the backup; recomputed from the published embedding" -table2,fmg,NN@1-Acc,euclidean,0.412,0.411765,0.001,"paper value = round(x, 3); |expected-paper|=0.000235; no original metric output in the backup; recomputed from the published embedding" -table2,fmg,Hopkins,euclidean,0.752,0.751741,0.001,"paper value = round(x, 3); |expected-paper|=0.000259; no original metric output in the backup; recomputed from the published embedding" -table2,fmg,SCI,euclidean,0.117,0.117174,0.001,round and truncation agree; |expected-paper|=0.000174; no original metric output in the backup; recomputed from the published embedding -table2,fmg,SCI_unsup,euclidean,0.509,0.509364,0.001,round and truncation agree; |expected-paper|=0.000364; no original metric output in the backup; recomputed from the published embedding; best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) -table2,mace,ES-AUC,euclidean,0.485,0.485611,0.001,paper value = truncation of x; |expected-paper|=0.000611; no original metric output in the backup; recomputed from the published embedding -table2,mace,NN@1-Acc,euclidean,0.199,0.199113,0.001,round and truncation agree; |expected-paper|=0.000113; no original metric output in the backup; recomputed from the published embedding -table2,mace,Hopkins,euclidean,0.637,0.637379,0.001,round and truncation agree; |expected-paper|=0.000379; no original metric output in the backup; recomputed from the published embedding -table2,mace,SCI,euclidean,-0.094,-0.093877,0.001,"paper value = round(x, 3); |expected-paper|=0.000123; no original metric output in the backup; recomputed from the published embedding" -table2,mace,SCI_unsup,euclidean,0.369,0.369336,0.001,round and truncation agree; |expected-paper|=0.000336; no original metric output in the backup; recomputed from the published embedding; best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) -table2,e3fp,ES-AUC,cosine,,0.485935,0.001,"not in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,e3fp,NN@1-Acc,cosine,,0.177939,0.001,"not in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,e3fp,SCI,cosine,,-0.012543,0.001,"not in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,e3fp,SCI_unsup,cosine,,0.033825,0.001,"not in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,gemnet,ES-AUC,cosine,,0.601837,0.001,"not in the paper; paper definitions, --distance cosine" -table2,gemnet,NN@1-Acc,cosine,,0.312251,0.001,"not in the paper; paper definitions, --distance cosine" -table2,gemnet,Hopkins,cosine,,0.592812,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,gemnet,SCI,cosine,,0.004663,0.001,"not in the paper; paper definitions, --distance cosine" -table2,gemnet,SCI_unsup,cosine,,0.271961,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,molae,ES-AUC,cosine,,0.781942,0.001,"not in the paper; paper definitions, --distance cosine" -table2,molae,NN@1-Acc,cosine,,0.497357,0.001,"not in the paper; paper definitions, --distance cosine" -table2,molae,Hopkins,cosine,,0.602659,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,molae,SCI,cosine,,0.189480,0.001,"not in the paper; paper definitions, --distance cosine" -table2,molae,SCI_unsup,cosine,,0.247099,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,molspectra,ES-AUC,cosine,,0.543114,0.001,"not in the paper; paper definitions, --distance cosine" -table2,molspectra,NN@1-Acc,cosine,,0.235108,0.001,"not in the paper; paper definitions, --distance cosine" -table2,molspectra,Hopkins,cosine,,0.532642,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,molspectra,SCI,cosine,,-0.037315,0.001,"not in the paper; paper definitions, --distance cosine" -table2,molspectra,SCI_unsup,cosine,,0.127115,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,unimol,ES-AUC,cosine,,0.622620,0.001,"not in the paper; paper definitions, --distance cosine" -table2,unimol,NN@1-Acc,cosine,,0.338899,0.001,"not in the paper; paper definitions, --distance cosine" -table2,unimol,Hopkins,cosine,,0.559572,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,unimol,SCI,cosine,,0.021010,0.001,"not in the paper; paper definitions, --distance cosine" -table2,unimol,SCI_unsup,cosine,,0.152108,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,fmg,ES-AUC,cosine,,0.712555,0.001,"not in the paper; paper definitions, --distance cosine" -table2,fmg,NN@1-Acc,cosine,,0.422004,0.001,"not in the paper; paper definitions, --distance cosine" -table2,fmg,Hopkins,cosine,,0.751741,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,fmg,SCI,cosine,,0.158030,0.001,"not in the paper; paper definitions, --distance cosine" -table2,fmg,SCI_unsup,cosine,,0.509364,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,mace,ES-AUC,cosine,,0.484971,0.001,"not in the paper; paper definitions, --distance cosine" -table2,mace,NN@1-Acc,cosine,,0.200708,0.001,"not in the paper; paper definitions, --distance cosine" -table2,mace,Hopkins,cosine,,0.637379,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,mace,SCI,cosine,,-0.151823,0.001,"not in the paper; paper definitions, --distance cosine" -table2,mace,SCI_unsup,cosine,,0.369336,0.001,"not in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" -table2,e3fp,ES-AUC,v2_euclidean,,0.485935,0.001,"not in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" -table2,e3fp,NN@1-Acc,v2_euclidean,,0.256400,0.001,"not in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" -table2,e3fp,SCI,v2_euclidean,,-0.012543,0.001,"not in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" -table2,e3fp,SCI_unsup,v2_euclidean,,0.033825,0.001,"not in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" -table2,gemnet,ES-AUC,v2_euclidean,,0.577410,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,gemnet,NN@1-Acc,v2_euclidean,,0.415357,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,gemnet,Hopkins,v2_euclidean,,0.592823,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,gemnet,SCI,v2_euclidean,,-0.015151,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,gemnet,SCI_unsup,v2_euclidean,,0.271961,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molae,ES-AUC,v2_euclidean,,0.782544,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molae,NN@1-Acc,v2_euclidean,,0.709640,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molae,Hopkins,v2_euclidean,,0.602748,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molae,SCI,v2_euclidean,,0.115274,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molae,SCI_unsup,v2_euclidean,,0.247099,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molspectra,ES-AUC,v2_euclidean,,0.544744,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molspectra,NN@1-Acc,v2_euclidean,,0.337786,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molspectra,Hopkins,v2_euclidean,,0.532736,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molspectra,SCI,v2_euclidean,,-0.020326,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,molspectra,SCI_unsup,v2_euclidean,,0.127115,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,unimol,ES-AUC,v2_euclidean,,0.622998,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,unimol,NN@1-Acc,v2_euclidean,,0.481977,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,unimol,Hopkins,v2_euclidean,,0.559674,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,unimol,SCI,v2_euclidean,,0.011623,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,unimol,SCI_unsup,v2_euclidean,,0.152108,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,fmg,ES-AUC,v2_euclidean,,0.705652,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,fmg,NN@1-Acc,v2_euclidean,,0.587976,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,fmg,Hopkins,v2_euclidean,,0.752106,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,fmg,SCI,v2_euclidean,,0.117174,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,fmg,SCI_unsup,v2_euclidean,,0.509364,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,mace,ES-AUC,v2_euclidean,,0.485611,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,mace,NN@1-Acc,v2_euclidean,,0.283468,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,mace,Hopkins,v2_euclidean,,0.637289,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,mace,SCI,v2_euclidean,,-0.093877,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,mace,SCI_unsup,v2_euclidean,,0.369336,0.001,"not in the paper; v2 definitions, --distance euclidean" -table2,e3fp,ES-AUC,v2_cosine,,0.485935,0.001,"not in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,e3fp,NN@1-Acc,v2_cosine,,0.256400,0.001,"not in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,e3fp,SCI,v2_cosine,,-0.012543,0.001,"not in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,e3fp,SCI_unsup,v2_cosine,,0.033825,0.001,"not in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" -table2,gemnet,ES-AUC,v2_cosine,,0.601837,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,gemnet,NN@1-Acc,v2_cosine,,0.444804,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,gemnet,Hopkins,v2_cosine,,0.577979,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,gemnet,SCI,v2_cosine,,0.004663,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,gemnet,SCI_unsup,v2_cosine,,0.380451,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molae,ES-AUC,v2_cosine,,0.781942,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molae,NN@1-Acc,v2_cosine,,0.709405,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molae,Hopkins,v2_cosine,,0.602690,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molae,SCI,v2_cosine,,0.189480,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molae,SCI_unsup,v2_cosine,,0.408498,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molspectra,ES-AUC,v2_cosine,,0.543114,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molspectra,NN@1-Acc,v2_cosine,,0.337668,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molspectra,Hopkins,v2_cosine,,0.533677,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molspectra,SCI,v2_cosine,,-0.037315,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,molspectra,SCI_unsup,v2_cosine,,0.232817,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,unimol,ES-AUC,v2_cosine,,0.622620,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,unimol,NN@1-Acc,v2_cosine,,0.480923,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,unimol,Hopkins,v2_cosine,,0.559664,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,unimol,SCI,v2_cosine,,0.021010,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,unimol,SCI_unsup,v2_cosine,,0.270063,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,fmg,ES-AUC,v2_cosine,,0.712555,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,fmg,NN@1-Acc,v2_cosine,,0.601655,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,fmg,Hopkins,v2_cosine,,0.751049,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,fmg,SCI,v2_cosine,,0.158030,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,fmg,SCI_unsup,v2_cosine,,0.682689,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,mace,ES-AUC,v2_cosine,,0.484971,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,mace,NN@1-Acc,v2_cosine,,0.285157,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,mace,Hopkins,v2_cosine,,0.637111,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,mace,SCI,v2_cosine,,-0.151823,0.001,"not in the paper; v2 definitions, --distance cosine" -table2,mace,SCI_unsup,v2_cosine,,0.566272,0.001,"not in the paper; v2 definitions, --distance cosine" +table,model,metric,variant,paper_value,expected_value,tolerance,notes +table2,e3fp,ES-AUC,euclidean,0.486,0.485935,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/e3fp embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,e3fp,NN@1-Acc,euclidean,0.178,0.177939,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/e3fp embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,e3fp,SCI,euclidean,-0.012,-0.012543,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/e3fp embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,e3fp,SCI_unsup,euclidean,0.033,0.033825,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/e3fp embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) +table2,gemnet,ES-AUC,euclidean,0.577,0.577410,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/gemnet embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,gemnet,NN@1-Acc,euclidean,0.292,0.291641,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/gemnet embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,gemnet,Hopkins,euclidean,0.593,0.592812,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/gemnet embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,gemnet,SCI,euclidean,0.015,-0.015151,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/gemnet embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,gemnet,SCI_unsup,euclidean,0.272,0.271961,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/gemnet embedding; original Sep-2025 per-molecule output gives 0.271958887 (KMeans local optimum depends on the BLAS kernel); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) +table2,molae,ES-AUC,euclidean,0.782,0.782544,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molae embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molae,NN@1-Acc,euclidean,0.497,0.497546,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molae embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molae,Hopkins,euclidean,0.602,0.602659,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molae embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molae,SCI,euclidean,0.115,0.115274,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molae embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molae,SCI_unsup,euclidean,0.247,0.247099,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molae embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) +table2,molspectra,ES-AUC,euclidean,0.545,0.544744,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molspectra embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molspectra,NN@1-Acc,euclidean,0.235,0.235250,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molspectra embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molspectra,Hopkins,euclidean,0.533,0.532642,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molspectra embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molspectra,SCI,euclidean,-0.020,-0.020326,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molspectra embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,molspectra,SCI_unsup,euclidean,0.127,0.127115,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/molspectra embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) +table2,unimol,ES-AUC,euclidean,0.622,0.622998,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/unimol embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,unimol,NN@1-Acc,euclidean,0.339,0.339582,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/unimol embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,unimol,Hopkins,euclidean,0.559,0.559572,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/unimol embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,unimol,SCI,euclidean,0.012,0.011623,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/unimol embedding; per-molecule values bit-identical to the original Sep-2025 output (results/chirality/en_sep_results/) +table2,unimol,SCI_unsup,euclidean,0.152,0.152108,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/unimol embedding; original Sep-2025 per-molecule output gives 0.152106160 (KMeans local optimum depends on the BLAS kernel); best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) +table2,fmg,ES-AUC,euclidean,0.706,0.705652,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/fmg embedding; the release has no per-molecule output for this model +table2,fmg,NN@1-Acc,euclidean,0.412,0.411765,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/fmg embedding; the release has no per-molecule output for this model +table2,fmg,Hopkins,euclidean,0.752,0.751741,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/fmg embedding; the release has no per-molecule output for this model +table2,fmg,SCI,euclidean,0.117,0.117174,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/fmg embedding; the release has no per-molecule output for this model +table2,fmg,SCI_unsup,euclidean,0.509,0.509364,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/fmg embedding; the release has no per-molecule output for this model; best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) +table2,mace,ES-AUC,euclidean,0.485,0.485611,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/mace embedding; the release has no per-molecule output for this model +table2,mace,NN@1-Acc,euclidean,0.199,0.199113,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/mace embedding; the release has no per-molecule output for this model +table2,mace,Hopkins,euclidean,0.637,0.637379,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/mace embedding; the release has no per-molecule output for this model +table2,mace,SCI,euclidean,-0.094,-0.093877,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/mace embedding; the release has no per-molecule output for this model +table2,mace,SCI_unsup,euclidean,0.369,0.369336,0.001,published protocol (--distance euclidean --metric-version paper); expected_value computed with this code from the published chirality/mace embedding; the release has no per-molecule output for this model; best-k silhouette scans k = 2..n-1 (release 0.1.0 capped k at 10) +table2,e3fp,ES-AUC,cosine,,0.485935,0.001,"not printed in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,e3fp,NN@1-Acc,cosine,,0.177939,0.001,"not printed in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,e3fp,SCI,cosine,,-0.012543,0.001,"not printed in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,e3fp,SCI_unsup,cosine,,0.033825,0.001,"not printed in the paper; paper definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,gemnet,ES-AUC,cosine,,0.601837,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,gemnet,NN@1-Acc,cosine,,0.312251,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,gemnet,Hopkins,cosine,,0.592812,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,gemnet,SCI,cosine,,0.004663,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,gemnet,SCI_unsup,cosine,,0.271961,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,molae,ES-AUC,cosine,,0.781942,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,molae,NN@1-Acc,cosine,,0.497357,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,molae,Hopkins,cosine,,0.602659,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,molae,SCI,cosine,,0.189480,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,molae,SCI_unsup,cosine,,0.247099,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,molspectra,ES-AUC,cosine,,0.543114,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,molspectra,NN@1-Acc,cosine,,0.235108,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,molspectra,Hopkins,cosine,,0.532642,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,molspectra,SCI,cosine,,-0.037315,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,molspectra,SCI_unsup,cosine,,0.127115,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,unimol,ES-AUC,cosine,,0.622620,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,unimol,NN@1-Acc,cosine,,0.338899,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,unimol,Hopkins,cosine,,0.559572,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,unimol,SCI,cosine,,0.021010,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,unimol,SCI_unsup,cosine,,0.152108,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,fmg,ES-AUC,cosine,,0.712555,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,fmg,NN@1-Acc,cosine,,0.422004,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,fmg,Hopkins,cosine,,0.751741,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,fmg,SCI,cosine,,0.158030,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,fmg,SCI_unsup,cosine,,0.509364,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,mace,ES-AUC,cosine,,0.484971,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,mace,NN@1-Acc,cosine,,0.200708,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,mace,Hopkins,cosine,,0.637379,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,mace,SCI,cosine,,-0.151823,0.001,"not printed in the paper; paper definitions, --distance cosine" +table2,mace,SCI_unsup,cosine,,0.369336,0.001,"not printed in the paper; paper definitions, --distance cosine; paper definitions compute this on raw vectors (Euclidean), independent of --distance" +table2,e3fp,ES-AUC,v2_euclidean,,0.485935,0.001,"not printed in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" +table2,e3fp,NN@1-Acc,v2_euclidean,,0.256400,0.001,"not printed in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" +table2,e3fp,SCI,v2_euclidean,,-0.012543,0.001,"not printed in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" +table2,e3fp,SCI_unsup,v2_euclidean,,0.033825,0.001,"not printed in the paper; v2 definitions, --distance euclidean; fingerprints always use Tanimoto distance" +table2,gemnet,ES-AUC,v2_euclidean,,0.577410,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,gemnet,NN@1-Acc,v2_euclidean,,0.415357,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,gemnet,Hopkins,v2_euclidean,,0.592823,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,gemnet,SCI,v2_euclidean,,-0.015151,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,gemnet,SCI_unsup,v2_euclidean,,0.271961,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molae,ES-AUC,v2_euclidean,,0.782544,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molae,NN@1-Acc,v2_euclidean,,0.709640,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molae,Hopkins,v2_euclidean,,0.602748,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molae,SCI,v2_euclidean,,0.115274,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molae,SCI_unsup,v2_euclidean,,0.247099,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molspectra,ES-AUC,v2_euclidean,,0.544744,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molspectra,NN@1-Acc,v2_euclidean,,0.337786,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molspectra,Hopkins,v2_euclidean,,0.532736,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molspectra,SCI,v2_euclidean,,-0.020326,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,molspectra,SCI_unsup,v2_euclidean,,0.127115,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,unimol,ES-AUC,v2_euclidean,,0.622998,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,unimol,NN@1-Acc,v2_euclidean,,0.481977,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,unimol,Hopkins,v2_euclidean,,0.559674,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,unimol,SCI,v2_euclidean,,0.011623,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,unimol,SCI_unsup,v2_euclidean,,0.152108,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,fmg,ES-AUC,v2_euclidean,,0.705652,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,fmg,NN@1-Acc,v2_euclidean,,0.587976,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,fmg,Hopkins,v2_euclidean,,0.752106,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,fmg,SCI,v2_euclidean,,0.117174,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,fmg,SCI_unsup,v2_euclidean,,0.509364,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,mace,ES-AUC,v2_euclidean,,0.485611,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,mace,NN@1-Acc,v2_euclidean,,0.283468,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,mace,Hopkins,v2_euclidean,,0.637289,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,mace,SCI,v2_euclidean,,-0.093877,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,mace,SCI_unsup,v2_euclidean,,0.369336,0.001,"not printed in the paper; v2 definitions, --distance euclidean" +table2,e3fp,ES-AUC,v2_cosine,,0.485935,0.001,"not printed in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,e3fp,NN@1-Acc,v2_cosine,,0.256400,0.001,"not printed in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,e3fp,SCI,v2_cosine,,-0.012543,0.001,"not printed in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,e3fp,SCI_unsup,v2_cosine,,0.033825,0.001,"not printed in the paper; v2 definitions, --distance cosine; fingerprints always use Tanimoto distance" +table2,gemnet,ES-AUC,v2_cosine,,0.601837,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,gemnet,NN@1-Acc,v2_cosine,,0.444804,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,gemnet,Hopkins,v2_cosine,,0.577979,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,gemnet,SCI,v2_cosine,,0.004663,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,gemnet,SCI_unsup,v2_cosine,,0.380451,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molae,ES-AUC,v2_cosine,,0.781942,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molae,NN@1-Acc,v2_cosine,,0.709405,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molae,Hopkins,v2_cosine,,0.602690,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molae,SCI,v2_cosine,,0.189480,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molae,SCI_unsup,v2_cosine,,0.408498,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molspectra,ES-AUC,v2_cosine,,0.543114,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molspectra,NN@1-Acc,v2_cosine,,0.337668,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molspectra,Hopkins,v2_cosine,,0.533677,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molspectra,SCI,v2_cosine,,-0.037315,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,molspectra,SCI_unsup,v2_cosine,,0.232817,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,unimol,ES-AUC,v2_cosine,,0.622620,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,unimol,NN@1-Acc,v2_cosine,,0.480923,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,unimol,Hopkins,v2_cosine,,0.559664,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,unimol,SCI,v2_cosine,,0.021010,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,unimol,SCI_unsup,v2_cosine,,0.270063,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,fmg,ES-AUC,v2_cosine,,0.712555,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,fmg,NN@1-Acc,v2_cosine,,0.601655,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,fmg,Hopkins,v2_cosine,,0.751049,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,fmg,SCI,v2_cosine,,0.158030,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,fmg,SCI_unsup,v2_cosine,,0.682689,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,mace,ES-AUC,v2_cosine,,0.484971,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,mace,NN@1-Acc,v2_cosine,,0.285157,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,mace,Hopkins,v2_cosine,,0.637111,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,mace,SCI,v2_cosine,,-0.151823,0.001,"not printed in the paper; v2 definitions, --distance cosine" +table2,mace,SCI_unsup,v2_cosine,,0.566272,0.001,"not printed in the paper; v2 definitions, --distance cosine" diff --git a/splits/rmd17/README.md b/splits/rmd17/README.md index 12e47fb..129342c 100644 --- a/splits/rmd17/README.md +++ b/splits/rmd17/README.md @@ -18,9 +18,9 @@ of the rMD17 `.npz`, so these indices can be used directly on the HF data. - The zero-shot energy benchmark (Tables 3, 6 and 7) does **not** use these splits; it samples windows over all frames. -- The fine-tuning inputs for Tables 8 and 9 that we found in our backups are exactly split **01** - (`index_train_01.csv` / `index_test_01.csv`, 1000/1000 frames, no validation split) for all ten - molecules. Fine-tuning code and checkpoints for rMD17 are not part of this repository. +- The fine-tuning inputs for Tables 8 and 9 are split **01** (`index_train_01.csv` / + `index_test_01.csv`, 1000/1000 frames, no validation split) for all ten molecules. Fine-tuning code + and checkpoints for rMD17 are not part of this repository. ## SHA-256 diff --git a/src/three_dbench/__main__.py b/src/three_dbench/__main__.py index 03a8846..a7ddf68 100644 --- a/src/three_dbench/__main__.py +++ b/src/three_dbench/__main__.py @@ -57,7 +57,7 @@ def _parse_args() -> argparse.Namespace: type=str, choices=["paper", "v2"], default="paper", - help="All tasks: 'paper' reproduces the published numbers (default), 'v2' uses the corrected " + help="All tasks: 'paper' reproduces the published numbers (default), 'v2' uses the alternative " "definitions documented in docs/METRICS.md", ) evaluate.add_argument("--lie-k", type=int, default=None, help="Rotation: override k of LIE@k") diff --git a/src/three_dbench/benchmarks/rotation.py b/src/three_dbench/benchmarks/rotation.py index 7c3c019..c112437 100644 --- a/src/three_dbench/benchmarks/rotation.py +++ b/src/three_dbench/benchmarks/rotation.py @@ -422,7 +422,7 @@ def _run_rows(records: list[dict], *, n_jobs: int, chunk_size: int, progress: bo def _offset_drift_records(selected: pd.DataFrame, failed_keys: set[str]) -> tuple[list[dict], dict]: - """Rows after a failed molecule of the same shard, with the shifted per-shard offset.""" + """Rows after a skipped molecule of the same shard, with the shifted per-shard offset.""" records: list[dict] = [] info: dict[str, Any] = {} for shard, grp in selected.groupby("shard", sort=False): @@ -526,13 +526,12 @@ def evaluate_rotation_embeddings( Molecules whose RMSD matrix cannot be computed (e.g. conformers with different heavy-atom counts after ``RemoveHs``) are skipped and listed in ``config["selection"]["failed_keys"]``. - ``replicate_offset_drift`` (by-shard layout only) additionally recomputes every metric as the - published full geometry run did (Table 1 LIE@k / AS rows): in that run a molecule that failed - was skipped without advancing the per-shard embedding cursor, so every later molecule of the - same shard was evaluated with embeddings shifted back by the failed molecule's conformer count. - These values are stored in extra columns ``__offset_drift``; the regular columns always - use correctly aligned embeddings. Use it with complete shards, since only failures among the - evaluated molecules are detected. + ``replicate_offset_drift`` (by-shard layout only) additionally recomputes every metric with the + embedding indexing of the original full geometry run (Table 1 LIE@k / AS rows): there, the rows + of every molecule after a skipped one of the same shard are shifted back by the skipped + molecule's conformer count. These values are stored in extra columns ``__offset_drift``; + the regular columns index the embeddings by the dataset ``offset``. Use it with complete shards, + since only skipped molecules among the evaluated ones are detected. Returns a dict with ``per_key`` (DataFrame, one row per key and distance space), ``summary`` (DataFrame) and ``config`` (dict). When ``output_dir`` is given, writes diff --git a/src/three_dbench/chirality/evaluation.py b/src/three_dbench/chirality/evaluation.py index 4853322..a84a115 100644 --- a/src/three_dbench/chirality/evaluation.py +++ b/src/three_dbench/chirality/evaluation.py @@ -10,7 +10,7 @@ ``metric_version`` ``"paper"`` (default) reproduces the code path that produced the published numbers. - ``"v2"`` applies the corrected definitions listed in ``docs/metrics/chirality.md`` + ``"v2"`` applies the alternative definitions listed in ``docs/metrics/chirality.md`` (tie-aware NN1 restricted to points with a same-class partner, centroid DBI, explicit Hopkins population, best-k silhouette on the selected distance). @@ -987,7 +987,7 @@ def evaluate_en_separation_from_counts( unsup_kmax: largest k for the best-k silhouette; ``None`` = ``n - 1`` (published setting). max_molecules: evaluate only the first N molecules (quick tests). distance: ``"euclidean"`` (published Table 2) or ``"cosine"``; ignored for fingerprints. - metric_version: ``"paper"`` (published definitions) or ``"v2"`` (corrected definitions). + metric_version: ``"paper"`` (published definitions) or ``"v2"`` (alternative definitions). n_jobs: worker processes (``-1`` = all CPUs). Results do not depend on ``n_jobs``. progress: show a tqdm progress bar. diff --git a/src/three_dbench/traj/metrics_v2.py b/src/three_dbench/traj/metrics_v2.py index 29fc1ac..0f174f2 100644 --- a/src/three_dbench/traj/metrics_v2.py +++ b/src/three_dbench/traj/metrics_v2.py @@ -1,12 +1,11 @@ -"""Corrected ("v2") energy metrics for the trajectory benchmark. +"""Alternative ("v2") energy metrics for the trajectory benchmark. The published tables (paper Tables 3, 6, 7) were computed with the definitions in :func:`three_dbench.traj.evaluation.compute_energy_metrics_from_condensed`, :func:`~three_dbench.traj.evaluation.thresholded_smoothness` and :func:`~three_dbench.traj.evaluation.ks_wasserstein_against_energy_diff` (``--metric-version paper``). -Those implementations differ from the definitions written in the paper appendix (C.3, C.6) in -several places. ``--metric-version v2`` implements the appendix definitions, with the choices the -appendix leaves open made explicit. See ``docs/metrics/energy.md`` for formulas and rationale. +``--metric-version v2`` implements the definitions of the paper appendix (C.3, C.6), with the choices +the appendix leaves open made explicit. See ``docs/metrics/energy.md`` for formulas and rationale. Differences from ``paper`` (per window of ``n`` frames, ``dE_ij = |E_i - E_j|``, ``dZ_ij = Delta_ij``): diff --git a/tests/test_baselines.py b/tests/test_baselines.py new file mode 100644 index 0000000..c5b88af --- /dev/null +++ b/tests/test_baselines.py @@ -0,0 +1,652 @@ +"""Offline tests for the extraction scripts in ``baselines/``. + +Only the parts that do not need a model are exercised: that every script has a working +``--help`` with the shared flags, and that the helpers in ``baselines/common.py`` behave. +Running a model is out of scope here; each one needs its own environment and a GPU. +""" + +from __future__ import annotations + +import importlib.util +import pickle +import subprocess +import sys +from pathlib import Path + +import numpy as np +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +BASELINES_DIR = REPO_ROOT / "baselines" +MODELS = ("e3fp", "fmg", "gemnet", "mace", "molae", "molspectra", "unimol") + + +def _load_common(): + """Import ``baselines/common.py`` the way the extraction scripts do.""" + spec = importlib.util.spec_from_file_location("baselines_common", BASELINES_DIR / "common.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +common = _load_common() + + +# --------------------------------------------------------------------------- # +# the scripts +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("model", MODELS) +def test_extract_script_help(model): + """``--help`` works without any model dependency installed.""" + script = BASELINES_DIR / model / "extract_chirality.py" + assert script.exists() + result = subprocess.run( + [sys.executable, str(script), "--help"], + capture_output=True, + text=True, + timeout=180, + ) + assert result.returncode == 0, result.stderr + for flag in ("--dataset", "--out", "--verify"): + assert flag in result.stdout, f"{model}: {flag} missing from --help" + + +@pytest.mark.parametrize("model", MODELS) +def test_model_directory_is_documented(model): + assert (BASELINES_DIR / model / "ENVIRONMENT.md").is_file() + assert f"]({model}/)" in (BASELINES_DIR / "README.md").read_text(encoding="utf-8") + + +def test_published_table_matches_the_package(): + """``common.PUBLISHED_CHIRALITY`` mirrors ``three_dbench.embeddings``.""" + from three_dbench.embeddings import published_embedding + + assert set(common.PUBLISHED_CHIRALITY) == set(MODELS) + for model, (path, key) in common.PUBLISHED_CHIRALITY.items(): + entry = published_embedding("chirality", model) + assert entry.path == path + assert entry.key == key + + +# --------------------------------------------------------------------------- # +# files +# --------------------------------------------------------------------------- # +def test_sha256_file(tmp_path): + import hashlib + + path = tmp_path / "blob.bin" + payload = b"3dcs" * 1000 + path.write_bytes(payload) + assert common.sha256_file(path) == hashlib.sha256(payload).hexdigest() + + +def test_write_npz_roundtrip(tmp_path): + arr = np.arange(12, dtype=np.float32).reshape(3, 4) + for key, compress in (("arr_0", False), ("gemnet", False), ("embeddings", True)): + path = tmp_path / f"{key}.npz" + digest = common.write_npz(path, arr, key=key, compress=compress) + assert digest == common.sha256_file(path) + with np.load(path) as data: + assert data.files == [key] + np.testing.assert_array_equal(common.load_embedding(path, key), arr) + # a single numeric array is found without naming its key + np.testing.assert_array_equal(common.load_embedding(path), arr) + + +def test_load_embedding_key_errors(tmp_path): + path = tmp_path / "two.npz" + np.savez(path, a=np.zeros(3), b=np.ones(3)) + with pytest.raises(KeyError): + common.load_embedding(path) + with pytest.raises(KeyError): + common.load_embedding(path, "c") + np.testing.assert_array_equal(common.load_embedding(path, "b"), np.ones(3)) + + +def test_load_embedding_pickle(tmp_path): + path = tmp_path / "fp.pkl" + with path.open("wb") as handle: + pickle.dump({"e3fp": [1, 2, 3], "morgan": [4]}, handle) + assert common.load_embedding(path, "e3fp") == [1, 2, 3] + with pytest.raises(KeyError): + common.load_embedding(path) + + +def test_load_embedding_unsupported(tmp_path): + path = tmp_path / "x.txt" + path.write_text("nope") + with pytest.raises(ValueError): + common.load_embedding(path) + + +# --------------------------------------------------------------------------- # +# comparison +# --------------------------------------------------------------------------- # +def test_compare_vectors_identical(rng): + arr = rng.standard_normal((16, 8)).astype(np.float32) + result = common.compare_vectors(arr, arr.copy()) + assert result["shape_match"] and result["array_equal"] + assert result["max_abs_diff"] == 0.0 + assert result["cosine_min"] == pytest.approx(1.0) + assert result["rows_cosine_below_1-1e-6"] == 0 + assert result["allclose_1e-5"] + + +def test_compare_vectors_perturbed(rng): + arr = rng.standard_normal((16, 8)).astype(np.float32) + other = arr + np.float32(1e-4) + result = common.compare_vectors(arr, other) + assert not result["array_equal"] + assert result["max_abs_diff"] == pytest.approx(1e-4, rel=0.05) + assert not result["allclose_1e-5"] + assert result["allclose_1e-3"] + assert result["cosine_min"] == pytest.approx(1.0, abs=1e-6) + + +def test_compare_vectors_shape_mismatch(): + result = common.compare_vectors(np.zeros((4, 3)), np.zeros((5, 3))) + assert result["shape_match"] is False + assert "cosine_min" not in result + + +def test_compare_vectors_zero_rows(): + arr = np.zeros((3, 4)) + arr[1] = 1.0 + result = common.compare_vectors(arr, arr.copy()) + assert result["zero_norm_rows"] == 2 + assert result["cosine_min"] == pytest.approx(1.0) + + +def test_compare_fingerprints(): + from rdkit.DataStructs import ExplicitBitVect + + def bits(indices): + vector = ExplicitBitVect(16) + for index in indices: + vector.SetBit(index) + return vector + + left = [bits([0, 1]), bits([2, 3]), bits([4])] + right = [bits([0, 1]), bits([2, 3]), bits([5])] + result = common.compare_fingerprints(left, right) + assert result["shape_match"] + assert result["identical"] == 2 + assert result["identical_fraction"] == pytest.approx(2 / 3) + assert result["tanimoto_min"] == pytest.approx(0.0) + assert result["first_mismatched_rows"] == [2] + + assert common.compare_fingerprints(left, right[:2])["shape_match"] is False + + +def test_format_report_covers_every_field(): + text = common.format_report({"a": 1.5, "b": "x", "c": True}) + assert "a" in text and "b" in text and "c" in text + assert len(text.splitlines()) == 3 + + +# --------------------------------------------------------------------------- # +# references and verification +# --------------------------------------------------------------------------- # +def test_resolve_reference_local_path(): + path, key = common.resolve_reference("some/where/file.npz") + assert path == Path("some/where/file.npz") + assert key is None + + +def test_resolve_reference_unknown_model(): + with pytest.raises(ValueError): + common.resolve_reference("published", model="not-a-model") + + +def test_verify_against_a_local_file(tmp_path, rng, capsys): + arr = rng.standard_normal((10, 6)).astype(np.float32) + produced = tmp_path / "produced.npz" + reference = tmp_path / "reference.npz" + common.write_npz(produced, arr, key="arr_0") + common.write_npz(reference, arr + np.float32(1e-6), key="arr_0") + + result = common.verify( + produced, + model="mace", + reference=str(reference), + produced_key="arr_0", + reference_key="arr_0", + ) + assert result["produced_sha256"] == common.sha256_file(produced) + assert result["reference_sha256"] == common.sha256_file(reference) + assert result["sha256_identical"] is False + assert result["shape_match"] and not result["array_equal"] + assert result["cosine_min"] > 0.999999 + assert "[verify]" in capsys.readouterr().out + + +def test_verify_detects_an_identical_file(tmp_path, rng): + arr = rng.standard_normal((5, 4)).astype(np.float32) + produced = tmp_path / "produced.npz" + common.write_npz(produced, arr, key="arr_0") + result = common.verify(produced, model="mace", reference=str(produced), reference_key="arr_0") + assert result["sha256_identical"] is True + assert result["array_equal"] is True + + +# --------------------------------------------------------------------------- # +# inputs +# --------------------------------------------------------------------------- # +def _ethanol(): + from rdkit import Chem + + return Chem.AddHs(Chem.MolFromSmiles("CCO")) + + +def test_load_conformers_from_a_list_pickle(tmp_path): + path = tmp_path / "mols.pkl" + with path.open("wb") as handle: + pickle.dump([_ethanol(), _ethanol(), _ethanol()], handle) + assert len(common.load_conformers(str(path))) == 3 + assert len(common.load_conformers(str(path), limit=2)) == 2 + + +def test_load_conformers_from_a_dict_pickle(tmp_path): + path = tmp_path / "mols.pkl" + with path.open("wb") as handle: + pickle.dump({"a": [_ethanol(), _ethanol()], "b": [_ethanol()]}, handle) + assert len(common.load_conformers(str(path))) == 3 + + +def test_load_conformers_rejects_other_payloads(tmp_path): + path = tmp_path / "mols.pkl" + with path.open("wb") as handle: + pickle.dump(42, handle) + with pytest.raises(TypeError): + common.load_conformers(str(path)) + + +# --------------------------------------------------------------------------- # +# reporting +# --------------------------------------------------------------------------- # +def test_print_versions(capsys): + found = common.print_versions(("numpy", "definitely_not_a_module")) + assert found["python"] == sys.version.split()[0] + assert found["numpy"] == np.__version__ + assert "not importable" in found["definitely_not_a_module"] + assert "[versions]" in capsys.readouterr().out + + +# --------------------------------------------------------------------------- # +# the one --dataset syntax (baselines/README.md "Common to all of them") +# --------------------------------------------------------------------------- # +def test_parse_dataset_spec_hub_forms(): + assert common.parse_dataset_spec("hf:EscheWang/3dcs:chirality") == ("hub", "EscheWang/3dcs", "chirality") + assert common.parse_dataset_spec("hf:EscheWang/3dcs") == ("hub", "EscheWang/3dcs", "chirality") + assert common.parse_dataset_spec("EscheWang/3dcs") == ("hub", "EscheWang/3dcs", "chirality") + assert common.parse_dataset_spec("hf:EscheWang/3dcs:rotation") == ("hub", "EscheWang/3dcs", "rotation") + assert common.parse_dataset_spec("hf:EscheWang/3dcs", hf_config="traj")[2] == "traj" + + +def test_parse_dataset_spec_local_forms(tmp_path): + directory = tmp_path / "chirality" + directory.mkdir() + pickle_path = tmp_path / "mols.pkl" + pickle_path.write_bytes(b"") + lmdb_path = tmp_path / "shard.lmdb" + lmdb_path.write_bytes(b"") + + assert common.parse_dataset_spec(f"hfdisk:{directory}") == ("disk", str(directory), None) + assert common.parse_dataset_spec(str(directory)) == ("disk", str(directory), None) + assert common.parse_dataset_spec(str(pickle_path)) == ("pickle", str(pickle_path), None) + assert common.parse_dataset_spec(f"lmdb:{lmdb_path}") == ("lmdb", str(lmdb_path), None) + assert common.parse_dataset_spec(str(lmdb_path)) == ("lmdb", str(lmdb_path), None) + + +def test_parse_dataset_spec_rejects_unusable_values(tmp_path): + with pytest.raises(FileNotFoundError): + common.parse_dataset_spec(str(tmp_path / "missing.pkl")) + with pytest.raises(FileNotFoundError): + common.parse_dataset_spec("./not/here") + with pytest.raises(ValueError): + common.parse_dataset_spec("") + with pytest.raises(ValueError): + common.parse_dataset_spec("hf:") + + +# --------------------------------------------------------------------------- # +# reading the published dataset layout +# --------------------------------------------------------------------------- # +def _mol_block(smiles: str, seed: int) -> str: + from rdkit import Chem + from rdkit.Chem import AllChem + + mol = Chem.AddHs(Chem.MolFromSmiles(smiles)) + AllChem.EmbedMolecule(mol, randomSeed=seed) + return Chem.MolToMolBlock(mol) + + +@pytest.fixture(scope="module") +def tiny_chirality_dir(tmp_path_factory): + """A `save_to_disk` directory shaped like the `chirality` config, rows out of order. + + Three rows holding 2, 1 and 3 conformers: 6 in total, whose published row order is the + order of ascending `offset`, not the order the rows happen to be stored in. + """ + datasets = pytest.importorskip("datasets") + pytest.importorskip("rdkit") + rows = [ + {"key": "c", "offset": 3, "mol_blocks": [_mol_block(s, i) for i, s in enumerate(("CCO", "CCC", "CCN"), 30)]}, + {"key": "a", "offset": 0, "mol_blocks": [_mol_block(s, i) for i, s in enumerate(("C[C@H](N)O", "CO"), 10)]}, + {"key": "b", "offset": 2, "mol_blocks": [_mol_block("CCCl", 20)]}, + ] + directory = tmp_path_factory.mktemp("hf") / "chirality" + datasets.Dataset.from_list(rows).save_to_disk(str(directory)) + return directory + + +@pytest.fixture(scope="module") +def tiny_chirality_pickle(tiny_chirality_dir, tmp_path_factory): + """The same six conformers as a pickle, in the same row order.""" + path = tmp_path_factory.mktemp("pkl") / "mols.pkl" + with path.open("wb") as handle: + pickle.dump(common.load_conformers(f"hfdisk:{tiny_chirality_dir}"), handle) + return path + + +def _atom_counts(mols): + return [mol.GetNumAtoms() for mol in mols] + + +#: Atom counts of the six conformers of ``tiny_chirality_dir``, in ascending ``offset``: +#: row "a" (offset 0) C[C@H](N)O, CO; row "b" (offset 2) CCCl; row "c" (offset 3) CCO, CCC, CCN. +TINY_ATOM_COUNTS = [11, 6, 8, 9, 11, 10] + + +def test_load_conformers_orders_rows_by_offset(tiny_chirality_dir): + """The rows are stored c, a, b; reading them gives the conformers in offset order.""" + mols = common.load_conformers(f"hfdisk:{tiny_chirality_dir}") + assert len(mols) == 6 + assert _atom_counts(mols) == TINY_ATOM_COUNTS + plain = common.load_conformers(str(tiny_chirality_dir)) + assert _atom_counts(plain) == TINY_ATOM_COUNTS + + +def test_load_conformers_limit_and_start(tiny_chirality_dir): + all_mols = common.load_conformers(f"hfdisk:{tiny_chirality_dir}") + assert _atom_counts(common.load_conformers(f"hfdisk:{tiny_chirality_dir}", limit=2)) == _atom_counts(all_mols[:2]) + assert _atom_counts(common.load_conformers(f"hfdisk:{tiny_chirality_dir}", start=4)) == _atom_counts(all_mols[4:]) + sliced = common.load_conformers(f"hfdisk:{tiny_chirality_dir}", start=2, limit=2) + assert _atom_counts(sliced) == _atom_counts(all_mols[2:4]) + + +def test_load_conformers_rejects_a_dataset_without_mol_blocks(tmp_path): + datasets = pytest.importorskip("datasets") + directory = tmp_path / "wrong" + datasets.Dataset.from_list([{"key": "a", "offset": 0, "smiles": "CCO"}]).save_to_disk(str(directory)) + with pytest.raises(KeyError, match="mol_blocks"): + common.load_conformers(f"hfdisk:{directory}") + + +def test_load_conformers_rejects_rows_that_are_not_contiguous(tmp_path): + datasets = pytest.importorskip("datasets") + directory = tmp_path / "gap" + rows = [ + {"key": "a", "offset": 0, "mol_blocks": [_mol_block("CCO", 1)]}, + {"key": "b", "offset": 7, "mol_blocks": [_mol_block("CCC", 2)]}, + ] + datasets.Dataset.from_list(rows).save_to_disk(str(directory)) + with pytest.raises(ValueError, match="contiguous"): + common.load_conformers(f"hfdisk:{directory}") + + +# --------------------------------------------------------------------------- # +# every script takes the same --dataset values (baselines/README.md) +# --------------------------------------------------------------------------- # +def _script_module(model): + """Import ``baselines//extract_chirality.py`` without its model dependencies.""" + name = f"baselines_{model}_extract" + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, BASELINES_DIR / model / "extract_chirality.py") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module # a module-level @dataclass looks itself up in sys.modules + try: + spec.loader.exec_module(module) + except Exception: + del sys.modules[name] + raise + return module + + +@pytest.mark.parametrize("model", MODELS) +def test_every_script_reads_every_input_form(model, tiny_chirality_dir, tiny_chirality_pickle): + """`hfdisk:`, a plain directory and a pickle give the same conformers in every script.""" + load = _script_module(model).load_molecules + reference = _atom_counts(common.load_conformers(f"hfdisk:{tiny_chirality_dir}")) + for spec in (f"hfdisk:{tiny_chirality_dir}", str(tiny_chirality_dir), str(tiny_chirality_pickle)): + assert _atom_counts(list(load(spec))) == reference, f"{model} disagrees on {spec}" + assert _atom_counts(list(load(f"hfdisk:{tiny_chirality_dir}", limit=3))) == reference[:3] + assert _atom_counts(list(load(f"hfdisk:{tiny_chirality_dir}", start=3))) == reference[3:] + + +@pytest.mark.parametrize("model", MODELS) +def test_every_script_resolves_the_hub_form_without_downloading(model, monkeypatch): + """`--dataset hf:EscheWang/3dcs:chirality` reaches ``datasets.load_dataset`` in every script.""" + pytest.importorskip("datasets") + seen = {} + + def fake_load_dataset(repo, name=None, split=None, revision=None): + seen.update(repo=repo, name=name, split=split) + raise RuntimeError("stop before the download") + + import datasets + + monkeypatch.setattr(datasets, "load_dataset", fake_load_dataset) + with pytest.raises(RuntimeError, match="stop before the download"): + list(_script_module(model).load_molecules("hf:EscheWang/3dcs:chirality")) + assert seen == {"repo": "EscheWang/3dcs", "name": "chirality", "split": "train"} + + +def test_molae_reads_one_row_per_conformer(tiny_chirality_dir): + """Regression: the Mol-AE HF path reads the `mol_blocks` list of every row, not one block.""" + molae = _script_module("molae") + source = (BASELINES_DIR / "molae" / "extract_chirality.py").read_text(encoding="utf-8") + assert "mol_block" not in source.replace("mol_blocks", "") + mols = molae.load_molecules(f"hfdisk:{tiny_chirality_dir}") + assert len(mols) == 6 # three rows, 2 + 1 + 3 conformers + assert molae.load_molecules(str(tiny_chirality_dir), "train", "chirality", limit=2) is not None + + +# --------------------------------------------------------------------------- # +# --verify over part of the conformers +# --------------------------------------------------------------------------- # +def test_parse_row_selection_defaults(): + assert common.parse_row_selection(None, produced_rows=10, reference_rows=10) == (None, "full") + indices, mode = common.parse_row_selection(None, produced_rows=3, reference_rows=10) + assert mode == "prefix" + np.testing.assert_array_equal(indices, [0, 1, 2]) + + +def test_parse_row_selection_slices_and_files(tmp_path): + indices, mode = common.parse_row_selection("2000:2003", produced_rows=3, reference_rows=52391) + assert mode == "slice" + np.testing.assert_array_equal(indices, [2000, 2001, 2002]) + indices, mode = common.parse_row_selection("5+", produced_rows=2, reference_rows=10) + np.testing.assert_array_equal(indices, [5, 6]) + + path = tmp_path / "rows.npy" + np.save(path, np.array([7, 1, 4])) + indices, mode = common.parse_row_selection(f"@{path}", produced_rows=3, reference_rows=10) + assert mode == "file" + np.testing.assert_array_equal(indices, [7, 1, 4]) + + text = tmp_path / "rows.txt" + text.write_text("0 2 4\n") + indices, _ = common.parse_row_selection(f"@{text}", produced_rows=3, reference_rows=10) + np.testing.assert_array_equal(indices, [0, 2, 4]) + + +def test_parse_row_selection_errors(tmp_path): + with pytest.raises(ValueError, match="full"): + common.parse_row_selection("full", produced_rows=3, reference_rows=10) + with pytest.raises(ValueError, match="more than"): + common.parse_row_selection("prefix", produced_rows=11, reference_rows=10) + with pytest.raises(ValueError, match="selects"): + common.parse_row_selection("0:5", produced_rows=3, reference_rows=10) + with pytest.raises(ValueError, match="outside"): + common.parse_row_selection("8:11", produced_rows=3, reference_rows=10) + with pytest.raises(ValueError, match="expected"): + common.parse_row_selection("every second row", produced_rows=3, reference_rows=10) + + +def test_compare_vectors_over_a_row_subset(rng): + reference = rng.standard_normal((20, 6)).astype(np.float32) + produced = reference[:5] + np.float32(1e-6) + result = common.compare_vectors(produced, reference, rows=np.arange(5), row_mode="prefix") + assert result["shape_match"] is False # the two files do not have the same number of rows + assert result["compared_shape_match"] is True + assert result["compared_rows"] == 5 + assert result["row_selection"] == "prefix" + assert result["cosine_min"] > 0.999999 + assert result["max_abs_diff"] == pytest.approx(1e-6, rel=0.05) + + scattered = common.compare_vectors(reference[[7, 1, 4]], reference, rows=np.array([7, 1, 4]), row_mode="file") + assert scattered["array_equal"] is True + assert scattered["compared_rows"] == 3 + + +def test_verify_compares_a_partial_output_with_the_prefix(tmp_path, rng, capsys): + reference = rng.standard_normal((50, 6)).astype(np.float32) + ref_path = tmp_path / "reference.npz" + common.write_npz(ref_path, reference, key="arr_0") + produced = tmp_path / "produced.npz" + common.write_npz(produced, reference[:8] + np.float32(1e-6), key="arr_0") + + result = common.verify(produced, model="mace", reference=str(ref_path), produced_key="arr_0", reference_key="arr_0") + assert result["compared_rows"] == 8 + assert result["row_selection"] == "prefix" + assert result["compared_shape_match"] is True + assert result["cosine_min"] > 0.999999 + assert "8 of the 50 rows" in capsys.readouterr().out + + +def test_verify_compares_a_slice_of_the_reference(tmp_path, rng): + reference = rng.standard_normal((50, 6)).astype(np.float32) + ref_path = tmp_path / "reference.npz" + common.write_npz(ref_path, reference, key="arr_0") + produced = tmp_path / "produced.npz" + common.write_npz(produced, reference[20:30], key="arr_0") + + result = common.verify( + produced, + model="mace", + reference=str(ref_path), + produced_key="arr_0", + reference_key="arr_0", + rows="20:30", + ) + assert result["row_selection"] == "slice" + assert result["array_equal"] is True + assert result["compared_rows"] == 10 + + wrong = common.verify( + produced, model="mace", reference=str(ref_path), produced_key="arr_0", reference_key="arr_0", rows="0:10" + ) + assert wrong["array_equal"] is False + + +def test_compare_fingerprints_over_a_subset(): + from rdkit.DataStructs import ExplicitBitVect + + def bits(indices): + vector = ExplicitBitVect(16) + for index in indices: + vector.SetBit(index) + return vector + + reference = [bits([0]), bits([1]), bits([2]), bits([3])] + result = common.compare_fingerprints(reference[:2], reference, rows=np.arange(2), row_mode="prefix") + assert result["compared_shape_match"] is True + assert result["identical"] == 2 + assert result["compared_rows"] == 2 + + +# --------------------------------------------------------------------------- # +# the documented commands and environments +# --------------------------------------------------------------------------- # +def _readme_commands(): + """The `baselines/README.md` "One command per model" block, one command per model.""" + text = (BASELINES_DIR / "README.md").read_text(encoding="utf-8") + block = text.split("## One command per model", 1)[1].split("```bash", 1)[1].split("```", 1)[0] + commands = {} + for raw in block.replace("\\\n", " ").splitlines(): + line = raw.strip() + if not line.startswith("python baselines/"): + continue + tokens = line.split() + model = tokens[1].split("/")[1] + commands[model] = tokens + return commands + + +@pytest.mark.parametrize("model", MODELS) +def test_readme_command_is_runnable(model): + """Every flag of the README command exists in that script, and the input spec is the shared one.""" + commands = _readme_commands() + assert model in commands, f"{model} has no command in baselines/README.md" + tokens = commands[model] + assert (REPO_ROOT / tokens[1]).is_file() + assert "--dataset" in tokens + assert tokens[tokens.index("--dataset") + 1] == "hf:EscheWang/3dcs:chirality" + + help_text = subprocess.run( + [sys.executable, str(REPO_ROOT / tokens[1]), "--help"], capture_output=True, text=True, timeout=180 + ).stdout + for token in tokens: + if token.startswith("--"): + assert token in help_text, f"{model}: {token} is not a flag of {tokens[1]}" + + +@pytest.mark.parametrize("model", MODELS) +def test_help_documents_the_shared_flags(model): + script = BASELINES_DIR / model / "extract_chirality.py" + result = subprocess.run([sys.executable, str(script), "--help"], capture_output=True, text=True, timeout=180) + assert result.returncode == 0, result.stderr + for flag in ("--dataset", "--out", "--verify", "--verify-rows", "--limit", "--start"): + assert flag in result.stdout, f"{model}: {flag} missing from --help" + assert "hf:EscheWang/3dcs:chirality" in result.stdout + + +def test_fmg_requirements_are_installable_together(): + """`pip install -r` needs a tqdm floor that `datasets==4.0.0` accepts.""" + packaging_specifiers = pytest.importorskip("packaging.specifiers") + text = (BASELINES_DIR / "fmg" / "requirements.txt").read_text(encoding="utf-8") + assert "reproduce/embeddings" not in text # the file lives at baselines/fmg/ + assert "baselines/fmg/extract_chirality.py" in text + pins = {} + for line in text.splitlines(): + line = line.split("#", 1)[0].strip() + if not line: + continue + name = line.split("==")[0].split(">=")[0].split("<")[0].strip() + pins[name] = line[len(name) :] + assert pins["datasets"] == "==4.0.0" + # datasets 4.0.0 requires tqdm>=4.66.3; the pin here has to allow it + assert packaging_specifiers.SpecifierSet(pins["tqdm"]).contains("4.66.3") + + +def test_weight_retrieval_is_documented(): + fmg = (BASELINES_DIR / "fmg" / "ENVIRONMENT.md").read_text(encoding="utf-8") + assert "drive.google.com/drive/folders/1XpOfCPRvPu22dSgbWgfGRF0Lul7ygdC7" in fmg + assert "gdown" in fmg + assert "f55ec38f2b6c20ad3a2e4e6287efb77af3901d46548449543bdbab33357d2afa" in fmg + + molae = (BASELINES_DIR / "molae" / "ENVIRONMENT.md").read_text(encoding="utf-8") + assert "ZIP" in molae and "unzip" in molae + assert "1NKObZCfE80GCLS9yJ7hqMGzjfGol4LLo" in molae + assert "b4ca21a63799976fbf435a1c7275d5ef6e93a854cc7d90955dbaec50ef89b8c0" in molae + + +@pytest.mark.parametrize("model", MODELS) +def test_environment_states_the_input_precision(model): + text = (BASELINES_DIR / model / "ENVIRONMENT.md").read_text(encoding="utf-8") + assert "## Input precision" in text, f"{model}: no Input precision section" + section = text.split("## Input precision", 1)[1].split("\n## ", 1)[0] + assert "full float precision" in section + assert "four decimals" in section