diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..7e9ce2e9 --- /dev/null +++ b/.flake8 @@ -0,0 +1,11 @@ +[flake8] +exclude = + .git, + .claude, + __pycache__, + build, + dist, + docs/build, + *.egg-info, + .venv, + venv diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..fac305a0 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,25 @@ +## What and why + + + +## How it was verified + + + +## Checklist + +See [CONTRIBUTING.md](https://github.com/harmslab/tfscreen/blob/main/CONTRIBUTING.md) for details on each item. + +- [ ] Branch is up to date with `upstream/main` +- [ ] New and changed code has full unit test coverage, including branches and error paths +- [ ] Bug fixes include a test that fails without the fix +- [ ] Slow tests are marked `@pytest.mark.slow` +- [ ] `pytest tests/tfscreen --runslow` passes locally +- [ ] `diff-cover coverage.xml --compare-branch=upstream/main` reports 100% +- [ ] `flake8 . --count --select=E9,F63,F7,F82` reports no errors +- [ ] Public functions, classes and methods have numpydoc docstrings +- [ ] New components, CLIs and YAML keys follow the conventions in `CLAUDE.md` +- [ ] `CLAUDE.md` is updated if the change alters how the code works +- [ ] `CHANGELOG.md` has an entry under `[Unreleased]` for any user-visible change +- [ ] No data, checkpoints or other large files are committed diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 360c936f..225f4a29 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,6 +12,11 @@ on: type: boolean default: false +# A newer push to the same branch or pull request cancels the older run. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: test: name: Python ${{ matrix.python-version }} / ${{ matrix.os }} @@ -20,7 +25,11 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.12", "3.13", "3.14"] + # The coverage job runs this one, plus the slow tests. + exclude: + - os: ubuntu-latest + python-version: "3.12" steps: - uses: actions/checkout@v4 @@ -29,6 +38,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: pip - name: Install package and test dependencies run: pip install -e ".[test]" @@ -40,5 +50,47 @@ jobs: env: NUMBA_DISABLE_JIT: "1" run: > - pytest tests/tfscreen + pytest -n auto tests/tfscreen ${{ (github.event_name == 'workflow_dispatch' && inputs.run_slow_tests) && '--runslow' || '' }} + + coverage: + name: Coverage + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + # diff-cover compares against main, so it needs the full history. + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install package and test dependencies + run: pip install -e ".[test]" + + # Slow tests are included so code tested only by them counts as covered. + # pytest-cov combines the workers' data; settings come from pyproject.toml + # (branch coverage, source = tfscreen only, so planning/, tests/ and + # docs/ are never measured). + - name: Run unit tests under coverage + env: + NUMBA_DISABLE_JIT: "1" + run: pytest -n auto --runslow --cov --cov-report= tests/tfscreen + + - name: Report total coverage + run: | + coverage xml + coverage report --fail-under=90 + + # Every line and branch a pull request adds or changes in src/ must be + # covered. Files outside coverage.xml are not judged. + - name: Check coverage of changed lines + if: github.event_name == 'pull_request' + run: > + diff-cover coverage.xml + --compare-branch=origin/${{ github.base_ref }} + --fail-under=100 diff --git a/.gitignore b/.gitignore index 6321bc4b..e350fc49 100644 --- a/.gitignore +++ b/.gitignore @@ -205,3 +205,45 @@ cython_debug/ marimo/_static/ marimo/_lsp/ __marimo__/ + +# Local test/lint/coverage output from run_all_tests.sh +reports/ +docs/badges/ + +# Lab data for development; never committed (C10) +planning/dev-data/ +planning/results/ +# Run outputs downloaded into study directories (lab-data fits, multi-GB +# simulations); the study's scripts and README stay tracked +planning/studies/recipe/ +planning/studies/full-size-sim/sim_*/ +planning/studies/full-size-sim/pilot_*/ +planning/studies/full-size-sim-*/ +planning/studies/m-bias-factorial/m_bias/ +planning/studies/m-bias-factorial/m_subsets/ +planning/studies/m-bias-factorial/m_subsets_real/ +planning/studies/m-bias-factorial/m_subsets_poisson/ +planning/studies/m-bias-factorial/held_s2/ +planning/studies/m-bias-factorial/held_kanR_*/ +planning/studies/m-bias-factorial/held_all_*/ +planning/studies/m-bias-factorial/held_mix_*/ +planning/studies/m-bias-factorial/kanR_only_*/ +planning/studies/m-bias-factorial/jk_seed*/ +planning/studies/m-bias-factorial/km_draws/ +planning/studies/m-bias-factorial/sim_realistic_seed2/ +planning/studies/m-bias-factorial/sim_realistic_s2/ +planning/studies/m-bias-factorial/real_depth/ +planning/studies/m-bias-factorial/sim_realistic_nocong/ +planning/studies/two-stage-validation/run_*/ +planning/studies/m-bias-factorial/stage2_*/ + +.DS_Store + +planning/studies/x-vs-binding/out/ +planning/studies/within-tube-m/out/ +planning/studies/within-tube-m/runs/ +planning/studies/relative-m/runs/ +planning/studies/joint-fit-sim/runs/ +planning/studies/joint-fit-sim/out/ +planning/studies/joint-fit-real/runs/ +planning/studies/joint-fit-real/out/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 396fa180..467f845d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,1469 @@ fall into two kinds: ## [Unreleased] -Nothing yet. +## [0.5.0] - 2026-10-24 + +The release for the first manuscript on the 2026 screen: someone in the lab +can take a screen from per-tube read counts to per-genotype *X* curves with +95% intervals through one command, `tfs-fit-two-stage`, documented in +"Analyzing a screen" (`docs/source/screen-analysis.rst`). On simulations of +that screen the deep genotypes' intervals cover the truth 0.88-0.91 of the +time up to 1e5 reads and 0.76-0.81 above. tfscreen now needs Python 3.12 or +newer. + +The entries come in two groups. The first covers 2026-10-03 to +2026-10-24: the two-stage fit, its validation and results table, tube checks +and corrections, dependency floors, and the experimental joint +growth/binding pieces, which the two-stage fit refuses. The second, from +"Changes up to 2026-10-03" on, is the release as summarized then. + +### Fixed + +- **A ΔΔG CSV column can supply two structures.** `load_ddG_prior_csv` + failed ("The truth value of a Series is ambiguous") whenever one file + column fed two structures: a `column_map` such as `{HD: H}` for a model, + or, with no model given, a column kept as its own structure and copied to + another. The numeric check now looks at each source column once + (`theta/thermo/io.py`). +- **`--shared_shift` works under pandas 3.** `shared_shift.row_shifts` + wrote wt's zero into the array from `Series.to_numpy()`, which pandas + 3's copy-on-write returns as a read-only view (`ValueError: assignment + destination is read-only`), so `tfs-predict-theta` and + `tfs-predict-epistasis` with `--shared_shift` failed, and so did the + two-stage fit's predict step. It now takes a copy. CI (pandas 3) had + failed these 7 tests on every job since the shift was added; locally + (pandas 2) they passed. `tests/conftest.py` now turns copy-on-write on + under pandas 2, so the local suite behaves as CI's does. + +- **`tfs-fit-two-stage`'s k-shift refits stay at stage 2's MAP.** They + restarted at Adam's default step size (1e-3), fell about 6.7e5 nats off + stage 2's MAP and re-descended along the dk_geno population against wt, + so on the real data X moved +0.32 even with k unchanged and the shift + measured where the optimizer stopped + (`planning/studies/two-stage-validation/`, run 1). The refits now start + at `two_stage.kshift_step_size` (default 1e-5) with the dk_geno + population hyperparameters held at stage 2's MAP + (`two_stage.dk_geno_hypers`, `pin_dk_geno_hypers`), and a third refit + with k unchanged (`two_stage.kshift_control`, on by default) checks the + procedure: `kshift/drift.csv` gives each refit's change in X from stage + 2's MAP, and the command warns when the control moves more than a + quarter of the shift. Existing `kshift/` directories are reused as + they are; delete them (and `predict/`) to rerun. + +- **`tfs-fit-two-stage` with a pinned k or m in stage 1.** A pinned k or + m is a deterministic site, which a MAP checkpoint does not hold, so + `tfs-extract-params` skipped both and stage 1 failed writing `km.csv`. + It now falls back to `two_stage.km_from_map`, which reads the MAP's + parameters and takes a pinned condition's prior location. + +- **`total_num_reads` is the total over all replicates.** `tfs-simulate` + passed every replicate the full total and each split it over its own + tubes, so a simulation had N times the configured reads, and with a + design a replicate with few tubes got far more reads per tube (2.5e9 for + the dev-data design's two-tube replicate). Each sequenced replicate now + gets its share by tube count (`simulate_cli.replicate_read_totals`). + Multi-replicate simulations now produce the reads their config states, + 1/N of what they produced before. +- **Simulated wt keeps its reference curve under `hill_geno`.** The + simulation sampler drew wt's phenotype category and perturbation like any + genotype's, so wt could come out flat ("never binds", about 5% of seeds) + or shifted; a relative fit gauges X on wt, and the full-size simulation's + flat wt stretched the X truth 3,600-fold. `thermo_to_growth` now sets wt + to the `wt_*` SimPriors curve unless a wt override is given + (`_pin_wt_reference`). Simulations with `hill_geno` change for wt only. +- **`tfs-configure-model --set_priors` works under pandas 3.** The priors + CSV's `value` column mixes numbers and text, which pandas 3 reads as a + string dtype that refused the new float (`TypeError: Invalid value ... for + dtype 'str'`). `priors_edit.apply_priors_updates` reads it as object. This + failed every CI job from 2026-10-04 on; pandas 2 was unaffected. +- `python -m tfscreen.tfmodel.scripts.sample_posterior_cli` ran the command + twice (a duplicated `__main__` block). The `tfs-sample-posterior` console + script was unaffected. + +### Changed + +- **Dependency floors and a `cuda12` extra.** `pyproject.toml` gives + every dependency a lower bound, the oldest versions the suite has passed + with (pandas 2.3, numpy 2.3, jax 0.11.2, numpyro 0.22, among others); + there are no upper bounds. `pip install -e ".[cuda12]"` installs GPU jax + on Linux, where pip's plain jax is CPU-only. A fresh install on the + newest releases (pandas 3.0.6, numpy 2.5.4, jax 0.12.0) passed the full + suite. The README, quickstart and "Analyzing a screen" page say how to + install. +- **tfscreen needs Python 3.12 or newer** (was 3.11): jax 0.11 and newer, + the new floor, have no 3.11 wheels. CI tests 3.12, 3.13 and 3.14. +- **`tfs-fit-two-stage` no longer adds k's uncertainty by default.** A new + `two_stage.kshift` setting (default false) skips the jackknife and the + k-shift refits, and predict runs without `--shared_shift`; `tfs_x.csv`'s + `k_shift` is then blank. Validation run 2 found the repaired shift a + third or less of the true one on simulations + (`planning/studies/two-stage-validation/`). Without it, the deep + genotypes' 95% intervals covered 0.88-0.91 up to 1e5 reads and + 0.76-0.81 above. The whole fit now takes about half a day. Set + `kshift: true` for the old behavior, which `--step jackknife` and + `--step kshift` still run. The example SLURM chain follows the setting. +- **`tfs-fit-two-stage` refuses the experimental joint-fit pieces and the + marker warp.** `read_two_stage_config` now fails on `binding_df`, a + theta growth or binding noise model other than `zero` (such as + `genotype_offset`), the titrant-shift, binding-offset and warp priors in + `set_priors`, and a `growth_priors` table whose `m_warp` column turns the + warp on. A warp was fitted in stage 1 and silently dropped in stage 2, + whose pins carry only k and m. Those components' docstrings now say they + are experimental. +- **`binding_weight` defaults to 1.** Each binding observation now counts + once, the likelihood of the data as measured. The old default, + `N_growth_rows / N_binding_rows`, made the binding data weigh as much as + the whole growth dataset, so any + mismatch between binding and growth distorted the growth fit; it was one + reason binding was split from growth on 2026-09-26. A config stores its + resolved weight, so one written before this change keeps it; pass the + ratio as `--binding_weight` to reproduce the old default in a new one. + The count-likelihood, congression-calibration, svi-overconfidence and + real-data-fit studies already set 1. Joint-fit plan step 3 + (`planning/joint-fit-return.md`). + +- CI runs the tests in parallel (`pytest-xdist`, `-n auto`) and the coverage + job measures through `pytest-cov`; `pytest-xdist` and `pytest-cov` joined + the `test` extra. The Ubuntu / Python 3.12 matrix job is gone, since the + coverage job runs the same suite there plus the slow tests. A newer push + to a branch cancels its older run. +- The fit's progress line counts plateau windows from 1: `Plateau: 1/3` in + the first window that could stall, `3/3` in the window that decides a + step-size cut or a stop. + +### Added + +- **"Analyzing a screen" docs page** (`docs/source/screen-analysis.rst`): + the real-screen route from per-tube counts to `tfs_x.csv`, with tube + checks and corrections, the two-stage YAML, what each step does, cost, + the results table's columns, what *X* is and what its intervals carry + and leave out. The index, quickstart, README and `fitting.rst` point to + it; `fitting.rst` marks the joint growth/binding model experimental. + `examples/tfmodel/` gains `growth_priors_loose.csv` (named by + `two_stage.yaml`, which until now pointed at a file that was not there) + and a SLURM chain, `two_stage_submit.sh` with `two_stage_step.srun`. +- **`tfs-fit-two-stage` writes one results table, `{out_dir}/tfs_x.csv`.** + A new last step, `summary`: one row per genotype and concentration with + X's quantiles (with the shared k shift), the shift itself (`k_shift`), + the genotype's number of mutations, its reads in total and per library + (counted once into `{out_dir}/genotype_reads.csv`), `in_training_data`, + and the marker-consistency `consistency_z` and `inconsistent` (NaN when + the check did not run). It is rewritten on every run, so it picks up a + consistency step run later (`two_stage.genotype_reads`, `num_mutations`, + `x_summary`). +- **`tfs-configure-model --congression_sets`.** The congression mixture's + fixed co-resident draws (entry n-1 = number of sets with n co-residents; + default 12 3 1) were an orchestrator setting with no flag; now settable + and written to the config, refused without `transformation_model + mixture`. Each set is a cell class grown through the whole model, so + cost grows with the total (140 sets took about twice the default's wall + time on a small stage 1). On the joint-fit simulation the mixture's + approximation is the largest remaining source of stage 1's m bias + (`planning/studies/joint-fit-sim/`, run 8). + +- **`hill_geno` holds any population SD.** `theta_logit_low_hyper_scale_fixed`, + `theta_logit_delta_hyper_scale_fixed` and `theta_log_hill_K_hyper_scale_fixed` + join `theta_log_hill_n_hyper_scale_fixed`: > 0 makes that SD a + deterministic site at the held value with no guide parameters, as in + `hill_relative`. Set with `tfs-configure-model --set_priors`. On the + joint-fit simulation's stage 1, the learned SDs (tight HalfNormal priors, + log K's at 0.1) and the population locations' priors compressed the + singles' theta range and steepened m by 14% + (`planning/studies/joint-fit-sim/`, run 5). + +- **Simulator pieces for the joint fit.** A linear growth condition takes + `warp_a` and `warp_x0` (`growth_linkage.warp_theta`, the fit's `m_warp` + in numpy; truth `warp_log_a`, `warp_x0` in + `tfs_sim_growth_parameters.csv`); the top-level `growth_titrant_scale` + (s) makes growth see each curve at s times the nominal titrant while + binding sees the nominal one (`library_prediction` scales the growth + concentrations of the simulator's data, so the prior draw, wt's + reference and the overrides all follow; truth log s in + `tfs_sim_growth_titrant_scale.csv`); `binding_data.genotype_offset_sd` + adds one Normal offset per binding genotype to its whole curve + (`binding_data.apply_genotype_offsets`; truth in + `tfs_sim_binding_offsets.csv`). All off by default, so existing configs + simulate as before. Study: `planning/studies/joint-fit-sim/`, joint-fit + plan step 5. + +- **`tfs-extract-params` writes linear growth's warp.** With `m_warp` on, + `{out_prefix}_warp_log_a.csv` and `{out_prefix}_warp_x0.csv` hold each + warped condition's steepness and threshold (the extraction context now + carries the condition_growth priors, so only warped conditions get rows). + The warp runs with `hill_geno`'s absolute theta and the congression + mixture's partition-function rules, acting on each cell class's theta; + tested with the titrant shift and the binding offsets together. Joint-fit + plan step 4(c) (`planning/joint-fit-return.md`). + +- **`theta_binding_noise: genotype_offset`.** Each binding genotype's + binding theta is its curve plus one offset of its own, the same at every + titrant concentration, delta_g = tau * z_g with z_g ~ Normal(0, 1) and + tau ~ HalfNormal(0.2), learned unless `offset_scale_fixed` > 0 holds it. + Growth never sees it, so the shared curve is the in vivo one; a genotype + whose in vitro binding differs from its in vivo occupancy (M42I), or + whose curve carries its day's normalization error, disagrees alone + instead of pulling the growth parameters. The offsets live in a plate + over the binding genotypes only, indexed by a new + `BindingData.binding_ordinal` (binding-tensor order; the batch's indices + in binding-only mode), so a full library carries no prior-only offsets. + Subset and chunked predictions remap the plate by binding genotype label + (`analysis/prediction.py`); `tfs-extract-params` writes + `{out_prefix}_offset.csv` (per binding genotype) and + `{out_prefix}_offset_scale.csv`, and now runs the binding noise + component's extraction specs. Joint-fit plan step 4(b) + (`planning/joint-fit-return.md`). + +- **A titrant scale between binding and growth (`hill_geno`).** With + `theta_growth_log_conc_shift_scale` > 0 (a priors row, e.g. + `tfs-configure-model --set_priors theta_growth_log_conc_shift_scale=1`), + one global site `theta_growth_log_conc_shift` (log s) ~ Normal(loc, + scale) is sampled and growth sees each genotype's curve at s times the + nominal titrant concentration, while binding sees the nominal one: the + in vivo IPTG need not equal the nominal, and the curve is in binding + units. Applied in `generative/model.py` (`growth_titrant`) to the growth + data the theta component sees, including the congression mixture's + population and `tfs-predict-growth`'s; extracted as + `{out_prefix}_growth_log_conc_shift.csv`. Off by default (scale 0, no + site), so existing models trace unchanged. `tfs-predict-theta` still + reports the curve at the concentrations asked for, in binding units. + Joint-fit plan step 4(a) (`planning/joint-fit-return.md`). + +- **`tfs-process-binding`: a lab binding table to a `binding_df`.** One + row per measurement (genotype, protein and titrant concentrations in + molar by default, theta, replicate and date) becomes one row per + genotype and IPTG concentration at a chosen protein concentration (2 uM + by default), with the titrant in mM. Concentrations are rounded to six + significant digits before grouping, since the lab's values carry + floating-point residue that differs between days and would otherwise + split replicates of one point. Measurements within a replicate are + averaged first; `theta_std` is the standard error of the replicate mean + from one replicate SD pooled over every replicated point at that protein + concentration (most points have one replicate), or from `--theta_std`. + Writes `{out_prefix}.csv` for `tfs-configure-model --binding_df` and a + summary with replicate counts (`process_raw/binding.py`; joint-fit plan + step 2, `planning/joint-fit-return.md`). + +- **`tfs-check-tubes`: flag tubes whose composition disagrees with their + titrant label.** A routine QC of the growth table from + `tfs-process-counts`, for any titrant. Within each library x condition + panel, every tube's centered log-ratio composition is fit robustly as a + row (replicate x time) effect plus a per-row-scaled concentration profile, + scored on the axis from the lowest concentration's profile (0) to the + highest's (1), and tested within its row by a leave-one-out isotonic + order test and a level test against its label's expected score. Tube + totals and read depth cancel, and rows missing concentrations are not + shifted, unlike the row centering of the study script + (`planning/studies/relative-m/tube_qc.py`). Writes `{out_prefix}.csv`, one + row per tube with `score`, `expected`, `order_z`, `deviation_z` and + `flagged` (`process_raw/tube_qc.py::score_tubes`). On the dev data it + flags four tubes of pheS+4CP replicate 2, t_sel 82, and nothing else. + +- **`tfs-correct-tubes`: documented corrections to a tube table.** A YAML + corrections file kept with the experiment's data names each correction + (`id`, `action` swap/relabel/exclude, `tubes`, `reason`, `by`, `date`, + optional `evidence`), and the command writes a new tube table for + `tfs-process-counts` with a `correction_id` column, a long-form + `{out_prefix}_changes.csv`, and a provenance file holding the + corrections and the SHA-256 of the inputs, evidence files and output. + A correction moves only the design columns + (`tube_corrections.DESIGN_COLUMNS`); the sample name, its count file, + OD600 and totals stay with the physical sample, which is right for a + bench swap and a sample-sheet swap alike. It refuses unknown tubes, a + tube in two corrections, a swap across libraries or one that changes + nothing, a relabel of a non-design column, a result where two tubes + share a library and design, an empty reason, by or date, and an output + that would overwrite an input. Prompted by the pheS+4CP replicate 2, + t_sel 82 row of the dev data (`planning/studies/relative-m/`, run 3). + Example: `examples/process_raw/corrections.yaml`. + +- **A per-condition warp of theta in `linear` growth.** `m_warp` (bool, or + per condition through a 0/1 `--growth_priors` column) makes a + condition's growth follow f(theta) = (s(theta) - s(0)) / (s(1) - s(0)), + s a logistic with steepness a and threshold x0 sampled per warped + condition (priors `warp_log_a_*`, `warp_x0_*`); f(0) = 0 and f(1) = 1, + so wt's gauge and m keep their meaning, and f is linear as a -> 0. For + selection markers that read occupancy through differently shaped maps + (dev data: kanR switch-like, pheS graded; + `planning/studies/relative-m/`). +- **A titrant-dependent k in `linear` growth.** The prior + `k_titrant_loglinear_scale`, set with `tfs-configure-model + --set_priors`, adds a growth shift shared by every genotype to the + selection phase of selection conditions, one slope in log + concentration per selection condition; 0 at no titrant, 0 (off) by + default. The dev data's flat genotypes grow faster at 1 mM IPTG than + at none under both selections (`planning/studies/within-tube-m/`). A + free shift at every concentration was tried and dropped: it is not + identified when every genotype has its own curve. +- **Hold wt's curve in `hill_relative`.** Priors `theta_wt_hill_K_fixed` + (a titrant concentration) and `theta_wt_hill_n_fixed`, set with + `tfs-configure-model --set_priors`, hold wt's Hill K and n when > 0 + (default 0, free). wt's curve sets the X gauge, and a one-library fit + of the dev data collapsed it below 1 uM IPTG. +- **`--shared_shift` for `tfs-predict-theta` and `tfs-predict-epistasis`.** + A table of X shifts (`genotype`, `titrant_name`, `titrant_conc`, + `shift`) shared by every non-wt genotype: each posterior draw gets one + standard-normal multiple of it before quantiles (or epistasis) are taken + (`tfmodel/analysis/shared_shift.py`), so each genotype's interval carries + the uncertainty while differences between genotypes keep only what does + not cancel; wt (fixed by the X gauge) does not move. Needs a posterior + with draws. Written for the two-stage fit's k uncertainty, which moves + every genotype's X nearly alike. +- **`tfs-fit-two-stage`**: the two-stage fit of a growth-only relative + model as one command (`planning/two-stage-fit.md`). In its output + directory it runs stage 1 (wt, spikes and singles with the congression + mixture, `max` rule, lambda fitted with the measured value as its prior), + a delete-d jackknife of stage 1 (30 refits, 20% of tubes dropped, + stratified by condition and IPTG), stage 2 (the full library with every + condition's k and m pinned at stage 1; MAP, Laplace, X), two stage 2 + refits at k plus and minus one jackknife SD (the X shift per SD), X (and + optionally epistasis) with that shift as one shared offset per draw, and + a marker-consistency check (stage 2 on one library alone; per-genotype + z-scores; on by default, `--no_consistency`). The YAML's top-level keys + are `tfs-configure-model`'s arguments; a `two_stage` block holds the rest + (`examples/tfmodel/two_stage.yaml`). Steps skip finished work, `--step` + runs one and `--jackknife_task i` one jackknife subsample (for a cluster + array). +- **`tfmodel/two_stage.py`**: the two-stage fit's pieces as functions + (stage-1 genotypes, stratified delete-d jackknife subsamples of tubes, + jackknife k/m standard errors and k's main direction, pinned growth-prior + tables, the X shift table), ported from `planning/studies/m-bias-factorial/` + and checked to reproduce its numbers; also chunked growth-table filters + and the marker-consistency statistic. `tfs-fit-two-stage` chains them. + +- **Per-condition k/m pins in `linear` growth.** `m_pinned` and `k_pinned` + take one flag per `condition_rep` as well as a single bool, so one + marker's k and m can be held at an independent estimate while the + other's are fitted. `tfs-configure-model --growth_priors` takes 0/1 + `k_pinned`/`m_pinned` columns (indexed rows in the priors CSV, read back + as a tuple; flags that all agree collapse to the bool). The free + conditions are sampled as `condition_growth_{k,m}_free` and + `condition_growth_{k,m}` is a deterministic site holding every condition, + so it is read from a posterior file, not a MAP checkpoint. Configs + without per-condition pins trace exactly as before. + +- **Merged transformants, per-origin skew.** The simulator draws + single-plasmid cells as clone counts per genotype, one weighted row each + (`_sim_transform_collapsed`), so `transform_sizes` can be realistic (the + dev-data calibration has 6.7e7 double transformants) at one row per + genotype; founder sampling and demographic growth add up exactly over + merged clones. `max_congressed_cells` caps the multi-plasmid cells kept + one by one. `lib_assembly_skew_sigma` takes one value per library origin. + The random stream changed, so a seed no longer reproduces a simulation + made before this. +- **`unassigned_read_fraction`** simulate key (one number or one per + library): that share of each tube's reads goes to `__unknown__`, as in + real count files, where the dev data's kanR library lost 66% of its + reads. The raw count files and the growth table carry it like real data. + +- **Tube-offset diagnostic in `tfs-summarize-fit`** (pipeline plan step 7). + `tfs-extract-params` now writes the per-tube sample offsets, + `{out_prefix}_sample_offset_offset.csv` (`level`) or `_delta_k.csv` + (`normal`), one row per tube labeled by its design, plus their SD + (`sample_offset/_tubes.py`). `tfs-summarize-fit` reports them in prior SDs + and, per condition, their Spearman trend with titrant and with time + (`tfmodel/analysis/tube_offsets.py`), writes `_tube_offsets.csv`, + `_tube_offset_trends.csv` and `_tube_offsets.pdf`, and flags a fit whose + offsets trend (BH q < 0.05) in the JSON's new `tube_offsets` block, so a + fit in the offset mode is not accepted silently. +- **`cfu0` per library** in the simulate config (`{library: cells}`), so + separately grown libraries can start at their own densities, and + **`tfs-simulate --no_write_growth`**, which skips the direct growth table + (several GB at a real library's size; `tfs-process-counts` rebuilds it from + the raw files). The full-size simulation study calibrates `cfu0` per + library from a pilot run. + +- **Simulation designs** (`design` key in the simulate config, + `simulate/build_sample_dataframes.read_design`): a tube table in the + format `tfs-process-counts` reads, in place of `condition_blocks`, so a + simulation follows a real experiment's layout exactly (irregular times, + replicates sampled differently). Each replicate gets its own tubes and the + raw output keeps the design's tube names. Used by the full-size simulation + study (`planning/studies/full-size-sim/`). + +- **`tfs-simulate` writes the raw experiment** (`simulate/raw_output.py`, + pipeline plan step 6): one counts file per sequenced tube in + `tfs-process-fastq`'s format (`{out_prefix}_counts/`), the tube table + (`{out_prefix}_tubes.csv`) and, with an `od600` block, the OD600 table and + a copy of the calibration, and prints the `tfs-process-counts` command + that processes them. A simulation now goes through exactly the commands + real data do; the output matches `tfs_sim_growth.csv` row for row. + `--no_write_raw` skips it. `examples/simulate-and-analyze/run.sh` and the + quickstart now fit the processed table. + +- **MAP convergence on the exact loss** (`RunInference.full_batch_loss`, + `run_optimization(exact_loss=None)`, `ConvergenceMonitor.end_window(..., + exact_loss=)`). A MAP fit (and the staged MAP's stages, the SVI pre-MAP + and `tfs-prefit-calibration`) now computes the exact full-batch negative + log joint at the end of each convergence window and judges the window by + a line through the recent exact losses. On the dev-data screen the + mini-batch loss's trend SE was about 5e4 nats per window, so a descent of + 1e5 nats per window read as a stall and every MAP was cut to its final + step size while still descending; the reference fit and the staged fit + stopped at points 6e3 nats apart that the final losses ranked the wrong + way round. The convergence CSV gains a `loss_exact` column. SVI keeps the + mini-batch test. + +- **Staged MAP for level tube offsets** (`tfmodel/inference/staged_map.py`, + `tfs-fit-model --stage_offsets auto|on|off --staged_step_size`). A cold + MAP with `sample_offset: level` could settle with the tube offsets near + ±2.8, carrying the population's growth; the real-data fit avoided that by + hand over six runs. A fresh level-offset MAP now runs three MAPs itself: + the offsets held at 0, the offsets alone with everything else held there, + and the joint MAP from that point at step size 1e-4. Each stage writes its + own checkpoint, convergence record and `_params.npz` + (`{out_prefix}_stage1_*`, `{out_prefix}_stage2_*`); a finished stage is + reused and an interrupted one resumes. `HeldModel` wraps a model with + sites held at fixed values (a `numpyro.handlers.condition`). Dev-data + validation (`planning/studies/staged-map/`): it avoids the offset mode, + which the cold MAP fell into (4.8e5 nats worse), but run to convergence + the offset mode scores 7.9e4 nats better than the staged MAP; it is the + model's preferred optimum, with IPTG-structured offsets and unphysical + k/m. Filed: `planning/offset-mode-growth-transition.md`. + +### Changes up to 2026-10-03 + +A modeling release and an interface release. The model changes (the +congression mixture, the count likelihood, the growth-only relative fit, tube +offsets, OD600 tube totals, the convergence monitor and the block and +arrowhead Laplace) are listed from "Added" onward below the interface +entries. The interface entries come first: the command-line tools were made +consistent, three hand-edited steps of the real-data fit became flags, and +the documentation was rewritten to match the code. Many commands changed +their arguments; old scripts and run templates need updating. + +### Added + +- **Provenance on every run** (`tfscreen.util.provenance`). Every `tfs-*` + command prints the tfscreen version, git commit (with a flag for + uncommitted changes) and command line when it starts, and writes them to + `{out_prefix}_provenance.json` when it takes `--out_prefix`. The + `tfs-configure-model` YAML has a `provenance:` block and fit checkpoints a + `provenance` entry. Study READMEs no longer need to infer commits from file + times. +- **`tfs-process-counts` reads OD600.** `--od600_file` (one reading per + tube), `--od600_calibration_file` (a `tfs-calibrate-od600` calibration) and + `--tube_volume_mL` give each tube's `sample_cfu` and SD + (`od600.tube_totals_from_od600`), with the calibration curve's shared error + (`sample_cfu_curve_std`) kept apart from the reading's own + (`sample_cfu_reading_std`). A reading below the detection threshold is an + error. Supplied `sample_cfu` still works. Several libraries go through in + one call, as before, now tested against separate runs. +- **`tfs-configure-model --set_priors name=value ...`** sets any scalar prior + by full row name or unique dotted suffix (`sigma_fixed=0.17`, + `theta_log_hill_n_hyper_scale_fixed=0.5`); unknown or ambiguous names are + errors. Replaces `sed` on the priors CSV. +- **`tfs-configure-model --growth_priors`** (per-condition linear k/m priors + from a table) and **`--growth_priors_wt_rates`** (k and m from wt + monoculture rates on the relative-X gauge, SD floored by + `--growth_priors_sd_floor`), for models the pre-fit does not calibrate. + Replaces `set_growth_priors.py`. Shared helpers in + `tfscreen.tfmodel.priors_edit`, which `tfs-prefit-calibration` now uses too. +- **`tfs-sample-posterior --laplace auto|full|arrowhead|blocks|point`** and + `--laplace_max_params` (default 20,000). `auto` uses the full Laplace up + to that many MAP parameters and the arrowhead Laplace above. The arrowhead + writes the Schur-complement directions it held at the MAP to + `{out_prefix}_held_directions.csv` (`RunInference.held_shared_directions`). +- **CLI reference page** `docs/source/cli.rst`, generated from the commands' + own parsers by `docs/scripts/make_cli_reference.py`; + `tests/tfscreen/util/cli/test_cli_reference.py` fails when it is stale. + `generalized_main.capture_parser` returns a command's parser without + running it. +- **Per-argument `--help`.** `generalized_main` takes each argument's help + from the function's numpydoc `Parameters` entry and shows defaults; the + program name is the command's own. +- `examples/process_raw/`: a synthetic tube table, OD600 table and per-tube + count files (`make_example_data.py`) in the formats `tfs-process-counts` + reads. +- Documentation: a pipeline page (reads to posterior, one command per step), + and the old analysis page split into fitting, model and downstream pages. + +### Changed + +- **A switch that defaults to on is turned off with `--no_`.** + Previously `--` silently inverted it: `tfs-sample-prior --noise` + turned noise off, `tfs-build-empirical --drop_railed` stopped dropping and + `tfs-process-counts --verbose` made it quiet. `--` now does nothing. +- **A flag whose default is None reads a string** unless the command types + it. `tfs-summarize-fit --out_prefix` and `tfs-process-fastq + --max_num_reads` crashed with "invalid NoneType value" whenever they were + passed. +- `tfs-process-fastq library_config f1_fastq f2_fastq [--out_dir counts]`: + the library YAML comes first and the output directory is a flag (was + `f1 f2 out_dir run_config`). `--num_workers` defaults to -1. +- `tfs-process-counts sample_file counts_dir [--out_prefix]` writes + `{out_prefix}.csv` (default `tfs_growth`, or `tfs_presplit` with + `--presplit`); the output path was a positional. A missing `library` + column defaults to `default`. The `obs_file` column is no longer written. +- `tfs-simulate config_file [--out_prefix tfs_sim]` writes + `{out_prefix}_{name}.csv` (was positional `output_dir` and + `--output_prefix tfs_sim_`). Default file names are unchanged. +- `tfs-configure-model` refuses a library whose spiked genotype has no growth + data (`check_spikes_in_data`); `--allow_missing_spikes` restores the old + report-only behavior. +- `tfs-sample-posterior`: `--laplace` replaces `--map_point`, + `--laplace_blocks` and `--laplace_shared`. A small MAP is unchanged (full + Laplace); a MAP above 20,000 parameters now gets the arrowhead Laplace by + default instead of a dense Hessian that does not fit in memory. +- `tfs-build-empirical growth_df [--binding_df] [--seed] + [--growth_calibration_file] [--base_growth_df]` (was `growth_file seed + --binding_file --calibration_file --base_growth_file`); `--seed` is + required only when the pre-fit runs. +- `tfs-fit-genotypes growth_df growth_calibration_file`, default prefix + `tfs_fit_genotypes` (was `growth_file calibration_file`, `tfs_mle`). +- `tfs-predict-growth --seed` (was `--subset_seed`). +- `ModelOrchestrator`'s `theta_growth_noise` defaults to `zero`, as + `tfs-configure-model` does (was `logit_normal`), so the Python API and the + CLI build the same model by default. Saved configs record the component, + so they read back unchanged. +- `tfs-setup-grid` and `tfs-setup-sim-grid` take `--out_dir` (was + `--out_prefix`, which named a directory); `grid_summary.json` records + `out_dir`. +- Docstrings corrected where they described old behavior: MAP predictions + give a `q0.5` column, not `point_est`; default quantiles are the bare + `q` ladder; `tfs-build-empirical` writes + `_phenotype_model.json`; `tfs-summarize-fit`'s default prefix is + `{run_dir}/summary/tfs_summarize`. + +### Removed + +- `tfs-process-presplit`: use `tfs-process-counts --presplit`. +- `tfs-diagnose-nan` (SVI component guide only, the one hand-written argparse + CLI), `tfs-subset-genotypes` (its output no longer fit the configure and + fit flow), `tfs-report-cfu0` and `tfs-summarize-sbc` (superseded by + `tfs-summarize-calibration`; `error_calibration.summarize_sbc` remains as a + library function). +- `coverage_prob` and the `param_std` argument of + `analysis.stats_test_suite`. Its only caller, `tfs-summarize-fit`, never + passed uncertainties, so the statistic was always null; coverage is in + the summary's calibration outputs. + +### Fixed + +- The first spiked sequence in the example library configs + (`examples/process_raw/library_config.yaml`, + `examples/simulate/simulate_config.yaml`, + `examples/simulate-empirical/simulate_config.yaml`) carried a stray H74A + codon, so it decoded to H74A/K84L instead of the K84L its comment names. + The `examples/process_raw` count files were regenerated. +- The presplit table now carries `library`. `tfs-process-presplit` wrote + `replicate, condition_pre, genotype, ln_cfu, ln_cfu_std`, and + `tfs-configure-model --presplit_df` refused it ("Missing columns: + library"), so processed presplit data could not enter the model. +- `tfs-summarize-fit` finds a posterior written as `{out_prefix}.h5`, the + name `tfs-sample-posterior` uses. It looked only for `*_posterior.h5`, so + any prefix other than the default `tfs_posterior` lost the trajectory + plots. `*_ground_truth.h5` prior draws are not taken for a posterior. + +### Added + +- **`tfs-sample-posterior --laplace_blocks`** (`RunInference.get_laplace_posteriors(block_genotypes=True)`, + `block_laplace_factors`): a per-genotype Laplace at a MAP. The shared + parameters (growth k and m, hyperparameters, tube offsets) stay at the + MAP. Given them, each genotype's parameters couple to no other genotype, + so the Hessian splits into one small block per genotype (9-11 + parameters). B Hessian-vector products per genotype chunk give every + block at once (`--genotype_chunk_size`), so it runs on a full library, + where the full Hessian (about 2M parameters) cannot. It checks that the + blocks really separate, and refuses a model where they do not. On the + relative-fit run-4 MAPs, X 95% coverage was 0.93 (Poisson) and 0.84 + (realistic), against 0.98 / 0.94 for the full Laplace and 0.81 / 0.76 for + the low-rank guide. It leaves out the k/m uncertainty (coverage 0). + +- **`tfs-sample-posterior --laplace_blocks --laplace_shared`** + (`get_laplace_posteriors(block_shared=True)`, `arrowhead_laplace_factors`): + the block Laplace with the shared parameters' uncertainty. The Hessian is + an arrowhead: genotype blocks, each genotype's coupling to the shared + parameters, and the shared block. One more Hessian-vector product per + shared parameter per chunk gives all of it. The shared parameters are drawn + from their marginal (the Schur complement) and each genotype from its + conditional given them. That is the full Laplace, and it matched the + dense inverse Hessian exactly on a toy model, chunked or not. Two + departures, both for MAPs short of an optimum: a floored direction of a + genotype block carries no coupling, and a negative direction of the Schur + complement is held at the MAP and named in the log. On the relative-fit + run-4 MAPs, X 95% coverage was 0.97 (Poisson) and 0.94 (realistic), with + 0.95-0.98 above 1000 reads. k/m coverage was 0.92 / 1.0 and 0.50 / 0.42, + close to the full Laplace's 0.92 / 0.92 and 0.58 / 0.50. + +- **`theta: hill_relative` can hold every population SD.** + `theta_X_low_hyper_scale_fixed`, `theta_X_delta_hyper_scale_fixed` and + `theta_log_hill_K_hyper_scale_fixed` join the log(n) field below (above + 0: held, a deterministic site with no guide parameters). On the dev-data + growth-only MAP the learned SDs ran to 3.6, 4.3 and 11, against about + 0.45, 0.45 and 0.7 among genotypes with more than 1e4 reads, so the MAP + shrank nothing and its X at 100 reads or fewer was noise. + +### Fixed + +- **`test_nan_explosion` passes under numpyro 0.22.** numpyro 0.22 checks + distribution arguments by default (0.19 did not), so `svi.init` refused + the test's NaN starting loc with a `ValueError` before the optimization + loop's own NaN check ran. Every CI runner failed (numpyro 0.22.0, jax + 0.11.2), while local runs on numpyro 0.19.0 passed. The test now fits + under `validation_enabled(False)` + (`tests/tfscreen/tfmodel/inference/test_run_optimization.py`). A fit run + on numpyro 0.22 that starts from a non-finite point now stops at init with + numpyro's `ValueError`, not tfscreen's "model exploded" `RuntimeError`. + +- **The Laplace path no longer applies the mini-batch likelihood scale at + full batch.** `scale_vector` is built for the configured batch size, so a + model configured with `batch_size` below the library weighted every + growth likelihood by `num_genotype / batch_size` (53x on the dev-data + screen at 4096) even when the Hessian was taken over the whole library. + `RunInference._unscaled_batch` removes it. The full Laplace had only run + on simulations with `batch_size` at or above the library, so no result + was affected. + +- **`theta: hill_geno` and `hill_relative` take + `theta_log_hill_n_hyper_scale_fixed`.** Above 0, the population SD of + log(hill_n) is held at that value (a deterministic site, no guide + parameters) instead of learned. Set `theta.theta_log_hill_n_hyper_scale_fixed` + in the priors CSV; default 0 keeps it learned. On the dev-data screen + the learned SD ran to about 19 in every MAP fit, joint and growth-only. + n spread from 0.01 to 13, and in 42% of genotypes the curve changed more + between 0 and the lowest nonzero IPTG (0.1 µM) than across the whole + measured range. The data show no such step: for the binding genotypes + the fit predicted a 0.4-0.55 ln-unit gain relative to wt in kanR+kan, + and the counts show 0.00-0.08. + +- **`condition_growth: linear` takes per-condition `m_scale_plus` and + `m_scale_minus`.** Like `k_scale`, each may now be a per-condition array: + indexed rows in the priors CSV, joined to the conditions by + `condition_rep`. Each condition takes the entry its `+`/`-` flag names. + Before this, every selection condition shared one m prior width. That + could not express a tight prior from monoculture data for kan alongside + a loose one for pheS, which the growth-only dev-data fits need. + Scalars behave as before. + +- **`tfs-fit-model --init_from `** starts a fit at a MAP + point, the `{site}_auto_loc` arrays of an earlier fit's + `*_params.npz`, wherever they name a site of this model. They take + precedence over the guesses, and other sites start at their guesses. So + a model can be seeded from a simpler one, for example a + `sample_offset: level` fit from a `zero` fit. The option is refused with + `checkpoint_file`. Before this, guesses were the only way in, and + `read_configuration` silently drops guess rows for names outside the + configured guesses, including sites such as `condition_growth_k` and the + tube offsets. On the dev-data screen the level-offset MAP had fallen into + a mode with offsets of ±2.8. Holding the no-offset MAP's point and + fitting only small per-tube offsets beat it by 1.2e5 nats of count + likelihood (`planning/dev-data/real_fit`, gitignored). + +- **`sample_offset: level` takes `sigma_fixed`.** Above 0, the SD of the + per-tube offsets is held at that value (a deterministic site, no guide + parameters) instead of learned; set `growth.sample_offset.sigma_fixed` in + the priors CSV. On the first real-data fit the learned SD went from its + 0.2 prior scale to 0.53, and the offsets (-2.7 to +3.0, smooth in IPTG + and time) carried the population's growth in place of k and m. Default 0 + keeps the learned SD. + +- **`tfs-sample-posterior --map_point`** writes a MAP checkpoint's point + itself (one sample, `RunInference.get_map_posteriors`) instead of a + Laplace posterior, whose full Hessian is out of reach on a full library + (about 2 million parameters for the dev-data screen). +- **`tfs-sample-posterior --skip_growth_observations`** leaves the + per-observation growth sites (`growth_pred`, `growth_obs`) out of the + posterior file. They were 94% of its size on a 300-genotype subset and + would be about 100 GB per site at 500 draws on the full library; + `tfs-predict-growth` recomputes growth from the parameter draws. + +- **Compressed tables load by their inner extension.** `read_dataframe` + reads `.csv.gz`, `.tsv.bz2` and the like as CSV or TSV, with pandas + inferring the compression. They used to fall through to the slow + delimiter-sniffing reader. A full-library growth table from + `tfs-process-counts` is 5 GB as CSV and 0.7 GB gzipped. + +- **`linear` growth takes `k_pinned`.** Like `m_pinned`, it holds the + per-condition baseline k at `k_loc` (a deterministic site with no guide + parameters), set by `condition_growth.k_pinned` in the priors CSV. It is + for conditional fits at a fixed draw of k and m, such as the two-stage fit + in `planning/studies/svi-overconfidence/`. It is not meant for production + fits. + +### Fixed + +- **`ln_cfu0` extraction labeled rows with other genotypes' values.** The + model holds `ln_cfu0` as a (replicate, condition_pre, genotype) array, but + the extract spec indexed its flattened values with `map_ln_cfu0`, which + numbers only the combinations present in the data. Wherever a genotype + was missing from a replicate or library, every later row took another + cell's value: on an SVI-overconfidence run (2 of 200 cells missing) three + of four blocks correlated 0.09-0.25 with the presplit data they were fit + to, against 0.999 once labeled correctly. Fits were unaffected; the + `*_ln_cfu0.csv` outputs of `tfs-extract-params`, and anything downstream + of them, were wrong. Both `ln_cfu0` components now key rows by array + position (`hierarchical.ln_cfu0_extract_spec`). + +- **Simulated binding observations are no longer clipped to [0, 1].** + `generate_binding_df` and `generate_library_binding_df` clipped noisy + `theta_obs` to [0, 1], while the fit's binding likelihood is an unclipped + Normal, so the clipped anchors near 0 and 1 were biased (10-17% of the + observations in the SVI-overconfidence grids). `binding_data.clip_theta_obs: + true` reproduces the old simulations; the random draws are unchanged, so + only the formerly clipped values differ. + +- **SVI hyperparameter scales no longer start frozen.** The component + guide's hyperparameter scales are constrained `greater_than(1e-4)`, and + the guide start capped every scale at `guide_init_scale`, also 1e-4. They + started on the bound, `-inf` in unconstrained space, and never moved: in + every component-guide SVI fit since 2026-09-27 each hyperparameter's guide + SD stayed at 1e-4. The convergence monitor reported them as an infinite + drift, so conditional fits never stopped before their cap. + `initialization.component_guide_init` now starts a bounded scale at least + `init_scale` above its bound (`component_guide_map` records the bound). + Study grids fit with the component guide since then (count-likelihood v4, + relative-fit v3, SVI-overconfidence v2 and two-stage) carry the bug. +- **Laplace posteriors no longer blow up along a MAP's negative-curvature + directions.** `get_laplace_posteriors` floored Hessian eigenvalues at + 1e-3, a variance of 1000 in unconstrained units. A MAP stopped short of + its optimum always has a few negative eigenvalues, and where they touched + the growth slopes m the Laplace draws of m spread 50x wider than their + posterior. Eigenvalues are now floored at the prior's curvature along + each eigenvector (`laplace_eigenvalue_floor`, prior SDs from + `site_unconstrained_prior_sds` at the MAP), so no direction is wider than + the prior. On seed 8 of the two-stage grid the SD of m fell from 0.018 to + 0.0005. + +- **Breaking: `ln_cfu0: hierarchical_factored` is refused across libraries.** + It shares each genotype's starting abundance across every pre-condition of + a replicate. kanR and pheS are transformed and grown up separately, so + that is wrong for them. On simulations the fit pushed the per-genotype + difference, SD 0.17-0.29 ln units, into a confident per-genotype theta + error: 95% coverage above 1000 reads was 0.09, against 0.80 with + `hierarchical`. The orchestrator now refuses the factored model when a + replicate's pre-conditions come from different libraries. The study grids + and `examples/simulate-and-analyze/run.sh` now use `hierarchical`, and + their earlier calibration numbers carry the mismatch. + +- **Breaking: `ln_cfu0: hierarchical_factored` has a non-centered tube + offset.** It used to draw `tube_offset ~ Normal(0, tube_scale)` directly. + The 4 offsets (replicate x pre-condition) trade against the genotype + baselines, so they could sit at 0. The joint density then grew without + bound as `tube_scale` went to 0, so no MAP existed: the pre-MAP put the + scale at 5e-5 against a prior median of 0.34. The component now samples + `tube_offset_z ~ N(0, 1)` and keeps `tube_offset = tube_scale * + tube_offset_z` as a deterministic site. The guide's parameters are now + `ln_cfu0_tube_offset_z_locs/scales`, so checkpoints and guesses files + written before do not load. Reconfigure and refit. +- **A runaway loss is no longer called converged.** The convergence monitor + records the run's starting loss. A window whose loss falls below -1000 + times its magnitude now ends the run as `diverged`, not converged + (`convergence.RUNAWAY_LOSS_FACTOR`). A full-covariance guide had run to a + loss of -8e23 and was reported as converged. + +- **NUTS starts where it should.** `RunInference.run_nuts` passed + `initialize_model`'s `ParamInfo` tuple to `init_to_value`. No site + matched it, so every chain started at a uniform draw in [-2, 2] on the + unconstrained scale. `tfs-fit-model --analysis_method nuts` now starts + the chains at the MAP warm-up's point (`--pre_map_num_epoch`, as SVI + does), falling back to prior medians. It also passes the full data set + through `get_batch`, prints the worst split R-hat and least n_eff, and + takes `--nuts_dense_mass`. + +- **SVI no longer throws away the pre-MAP solution.** The component guide + started with every scale at 0.1 and a multiplicative 0.1 jitter on every + starting value. The scale is in each parameter's own units, so for growth + rates (per minute) and ln_cfu levels it was enormous. SVI started about + 1000x above the pre-MAP's loss, count and `lncfu` fits alike. It then + re-descended for tens of thousands of epochs into other optima: a + sign-flipped mirror mode, a halved slope `m`, or the whole library + shifted against wt. `tfs-fit-model --guide_init_scale` now defaults to + 1e-4 (`DEFAULT_GUIDE_INIT_SCALE`). It also sets the autoguides' + `init_scale` for a fresh fit, instead of numpyro's 0.1. SVI widens the + scales itself. `--init_param_jitter` and + `RunInference.run_optimization(init_param_jitter=...)` now default to 0. + SVI calibration results from before this fix (the count-likelihood and + relative-fit grids) need a rerun. + +### Added + +- **`tfs-calibrate-od600` (roadmap step 3).** Fits an OD600-to-CFU/mL + calibration from a lab's own two experiments: repeated OD600 readings of + a dilution series (reading noise and detection threshold) and plate + counts of cultures whose OD600 was read (CFU/mL from colonies, dilution + and plated volume, with counting and pipetting error). Writes the + polynomial's coefficients with their full covariance (the curve's error + is shared by every tube, so callers get it apart from the reading noise), + the reading noise, the detection threshold and the calibrated range, plus + a diagnostic PDF and CSVs. The method, file format and apply/invert + functions are in `tfscreen.process_raw.od600`; documented CSV inputs; + synthetic example in `examples/od600/`. It reproduces the lab + notebook's coefficients to 2e-9 when fed the notebook's CFU/mL, but the + notebook did not divide by the plated volume, so its constants (and the + CFU totals made with them) are 10x low. + +### Changed + +- **Breaking: the simulator's `od600.calibration` is a + `tfs-calibrate-od600` file.** The lab notebook's format (`A_CFU`, + `OD600_PCT_STD`, ...) is refused with a pointer to the new tool. + `examples/simulate/od600_calibration.yaml` is replaced by + `examples/od600/od600_calibration.yaml`, made by the tool from synthetic + data at a realistic scale (8e8 CFU/mL per OD600 unit, not the old + example's 8e7). `od600_in_range` now means at or below the top of the + calibrated OD600 range. + +- **Breaking: `tfs-configure-model` defaults to the count likelihood.** + `--growth_likelihood` now defaults to `counts` and + `--sample_offset_model` to `level` (was `lncfu` and `zero`). On the + count-likelihood simulation grid (`planning/studies/count-likelihood/`) + counts cut theta RMSE by 35-60% and dk_geno RMSE by 40-85% at about the + same coverage. The growth file then needs read counts, each tube's total + reads and its total cells (see `--growth_likelihood`), and + `--growth_noise_model` must stay `zero`: a call that passed + `--growth_noise_model normal_kt` without a likelihood now fails, and + should add `--growth_likelihood lncfu --sample_offset_model zero` to keep + its old model. `ModelOrchestrator` keeps `lncfu`/`zero` as its own + defaults, so a config written before the count likelihood existed (no + `growth_likelihood` key) still reads back as the model it was. + `tfs-build-empirical` inherits the new defaults. + +### Added + +- **Relative-X fit (`theta: hill_relative`; roadmap step 5).** A theta + component that fits growth alone on a wt-relative scale X instead of + claiming absolute occupancy (growth fixes it only up to an affine map): + hill_geno's per-genotype curve with real-valued baselines, gauged so that + wt's X is 1 at the low and 0 at the high gauge concentration + (`theta_gauge_conc`, an orchestrator setting and `tfs-configure-model + --theta_gauge_conc c_lo c_hi`; default the lowest and highest measured + concentration, recorded in the config). wt keeps its K and n free. It + refuses binding data, a learned activity, `logit` rescaling, non-linear + condition growth, theta noise and, under the congression mixture, any + theta rule but `max`. Outputs are labelled: deterministic sites + `theta_X_low`/`theta_X_high`, a `theta_scale = X` column from + `tfs-predict-theta`; `tfs-predict-epistasis` and `tfs-extract-epistasis` + allow only `--scale add` on X, and the former writes no `in_regime`. + `tfs-summarize-fit` puts simulated theta and `growth_k`/`growth_m` truth + on the X gauge, and `tfs-summarize-calibration` gives such rows + `theta_regime = X`. Prediction on a genotype subset keeps wt (the gauge + needs it) and drops it from the output. Validation grid: + `planning/studies/relative-fit/`. +- **Count likelihood (`growth_likelihood: counts`; roadmap step 7).** Growth + can be observed as read counts instead of `ln_cfu`: each genotype's reads + in a tube are negative binomial with mean + `depth * exp(ln_cfu_pred - ln(tube total))` and variance + `mu (1 + phi) + mu^2 inv_r` (both learned; study 0b found real counts 5-18x + Poisson), with no pseudocount (`generative/observe/growth_counts.py`). The + log-pmf is computed by Loader's algorithm (`CountNegativeBinomial`), + accurate to ~1e-4 nats in float32 where numpyro's negative binomial loses + several nats per observation at 1e4-1e6 reads. `tfs-configure-model + --growth_likelihood counts` (requires `growth_noise_model zero`); the + growth file needs `counts`, each tube's total reads (`sample_reads`, or + `adjusted_counts`/`frequency` to derive it) and total cells + (`sample_ln_cfu` or `sample_cfu`). Presplit data are still observed as + `ln_cfu`. +- **`sample_offset: level`.** One ln_cfu offset per tube, shared by every + genotype in it, with a learned constant SD (the tube's composition offset + and any error in its supplied total), in place of `normal`'s + time-scaled growth-rate offset. `tfs-configure-model` now exposes + `--sample_offset_model` (it was always `zero` before). +- **`counts_to_lncfu` writes `sample_reads`**, each tube's total reads + (`__unknown__` included, no pseudocounts). +- **Simulator sampling noise and OD600 (roadmap step 4).** Optional + simulate-config keys, all off by default so existing configs simulate + exactly as before: `founder_sampling` (Poisson founders per clone per + tube), `demographic_growth` (birth-death noise given the founders), + `shared_transformation` (one library assembly and transformation for all + replicates, as from one glycerol stock), `pcr_template_molecules` and + `pcr_amplification_cv` (a template bottleneck with PCR jackpotting before + sequencing). An `od600` block gives every tube one simulated OD600 reading + through an OD600-to-CFU calibration (`simulate/od600.py`), optionally + feeds the pipeline the OD600-derived total instead of the truth, adds + OD-only replicates, and writes `tfs_sim_od600.csv`. + `selection_experiment` gains `shared_state` and `sequence`. + `examples/simulate/od600_calibration.yaml` is a synthetic calibration for + examples; `od600.calibration` is a `tfs-setup-sim-grid` path key. +- **Growth-only models (no binding data).** `ModelOrchestrator` accepts + `binding_df=None` alongside growth data, and `tfs-configure-model` takes + `--binding_df` as an optional flag (at least one of `--binding_df` and + `--growth_df` is required). A growth-only model has no binding tensors, + no binding likelihood and no `theta_binding_noise` component; it refuses + a `binding_weight` or a non-`zero` binding noise model, and + `tfs-prefit-calibration` refuses a growth-only config. + `tfs-summarize-fit` then plots trajectories for wt, the spiked genotypes + and ten other genotypes chosen with a fixed seed, instead of every + genotype. Joint growth + binding models are unchanged (identical log + density at a fixed seed). Step 1 of `planning/analysis-roadmap.md`. +- **Soft-min congression dk rules (`congression_dk_rule` `softmin`, `min`).** + A congressed cell's dk_geno can now follow the whole soft-min family + `dk_cell = -(1/alpha) log sum_g x_g exp(-alpha dk_g)`, not only its + `alpha -> 0` limit: `dilution` (still the default, the share-weighted + mean), `softmin` (finite alpha, from the new `congression_dk_alpha`, in + units of 1/dk) and `min` (`alpha -> inf`, the worst variant sets the + cell's cost). Available in the simulator (`simulate/cell_rules.py`, + config keys `congression_dk_rule`/`congression_dk_alpha`) and the fit's + `mixture` transformation (`ModelOrchestrator` settings written to the + config and carried as static `GrowthData` fields; `tfs-configure-model + --congression_dk_rule`/`--congression_dk_alpha`). Alpha is required by + `softmin` and refused by the other rules. Congression physics plan, + step 5. +- **Limit on simulated Hill coefficients (`theta_sim_priors.max_hill_n`).** + hill_mut's simulation adds per-mutation and horseshoe-epistasis effects on + log(hill_n), so double mutants could reach Hill coefficients in the + hundreds (270 in the congression calibration study's library, with 50 of + 483 genotypes above 4). An optional `max_hill_n` in hill_mut's `SimPriors` + sets larger draws to the limit, in both `hill_mut.simulate` and + `binding_params.build_theta_gc_override_hill_mut` (the path used when + spiked binding curves come from a params file). Off by default, so + existing configs simulate as before. + +- **Guide selection for SVI (`tfs-fit-model --guide_type`).** With + `analysis_method=svi`, the variational family can now be any numpyro + autoguide as well as the component guide (still the default): + `delta`, `auto_normal`, `auto_diagonal_normal`, `auto_multivariate_normal` + or `auto_low_rank_multivariate_normal`. Numpyro class names such as + `AutoNormal` are accepted, case-insensitively. `--guide_rank` sets the rank + of `auto_low_rank_multivariate_normal`, and `--guide_init_scale` sets an + autoguide's initial scale. An option the chosen guide does not take is + refused before any fitting starts, as is any guide option with + `analysis_method` `map` or `nuts`. + - `RunInference.setup_svi` takes `guide_type`, `guide_kwargs` and + `init_values` (constrained site values an autoguide's location starts + from, via `init_to_value`). The registry is `AUTOGUIDES`/`GUIDE_TYPES` in + `inference/run_inference.py`, with `resolve_guide_type` and + `check_guide_kwargs`. + - An autoguide starts from the pre-MAP solution when `pre_map_num_epoch > 0`, + and from the configured guesses otherwise. + - Checkpoints record `guide_type` and `guide_kwargs`. + `restore_svi_from_checkpoint` rebuilds that guide; a checkpoint written + before this change is treated as a component-guide checkpoint. + `tfs-fit-model` refuses to resume a checkpoint with a different guide. + - `tfs-sample-posterior` sends `delta` checkpoints to the Laplace + approximation and every other guide to guide sampling. It routes by the + recorded `guide_type`, because `AutoNormal` parameters share AutoDelta's + `{site}_auto_loc` names. Older checkpoints still fall back to that name + check. + - `get_posteriors` drops autoguide auxiliary sites (names starting with + `_`, such as `AutoContinuous`'s `_auto_latent`) before the forward pass. + +- **`tfs-summarize-calibration`** (`analysis/calibration_grid.py`) pools + posterior calibration across a `tfs-setup-sim-grid` grid. It reads each + run's `combo.json` and the `tfs-summarize-fit` outputs in its `summary/` + directory, and reports coverage at several interval levels, calibration + error and bias, PIT uniformity (KS), interval width, and median RMSE and + Pearson r against the simulated truth. Rows are per run x quantity x + stratum: whether the genotype has binding data, its purity (`spike`: only + a spiked sequence encodes it, `bulk_fraction` 0; `bulk`: only the bulk + sub-libraries, `bulk_fraction` 1; `mixed`: both, like wt and the spiked + single mutants), and, for theta, whether the true value is resolvable or + saturated. Arms (runs sharing every grid variable except `--replicate_keys`) + are averaged, and `--baseline key=value ...` pairs each run with the + baseline run fit to the same simulated data. Unfinished runs are listed in + `{out_prefix}_run_status.csv`. No thresholds or grades. New console script: + reinstall to register it. + +- **`GrowthData.bulk_fraction`**: each genotype's fraction of cells from the + bulk (co-transformed) sub-libraries, built by `ModelOrchestrator` from the + library composition table (legacy `spiked_genotypes`: 0 for spiked, 1 + otherwise; `__unknown__` counts as bulk; a genotype missing from the table + is an error). This separates congression purity from the `ln_cfu0` prior + class, which previously shared one flag. Not yet used by the model, so fits + are unchanged. + +- **Library-ordered dk_geno and activity from their components.** Every + `dk_geno` (`fixed`, `pinned`, `hierarchical_geno`) and `activity` + (`fixed`, `hierarchical_geno`, `horseshoe_geno`, `hierarchical_mut`, + `horseshoe_mut`) component's `define_model` and `guide` take + `return_population=False`. When True they return `(tensor, population)`, + where `population` holds every genotype's value in library order, computed + from the same draw (no new sample sites), or `None` when the latents arrived + as a batch-sized substitution. The shared helper is + `generative/components/_population.py::per_genotype`. New + `GrowthData.external_dk_population` / `external_activity_population` + fields are the fallback for prediction. Groundwork for the congression + mixture's co-resident lookup; nothing requests the population yet, so fits + are unchanged. + +- **Fixed co-resident sets for the congression mixture.** `ModelOrchestrator` + draws, once per genotype, K co-resident plasmid sets + (`_draw_coresident_sets`) into `GrowthData.coresident_idx` + (`(num_genotype, K, N_max)` library indices, -1 = empty slot) and + `GrowthData.coresident_n` (co-residents per set). New settings + `congression_sets` (default `[12, 3, 1]`: 12 sets with one co-resident, + 3 with two, 1 with three) and `congression_seed` (default 0) are written + to the config's `components` block, so a refit reproduces the same draws. + Co-residents are drawn from the bulk share of the pool, + `pool_fraction * bulk_fraction`, over genotypes with growth data + (`__unknown__` excluded); the legacy `spiked_genotypes` path uses a + uniform pool over non-spiked genotypes. Not yet used by the model, so fits + are unchanged. + +### Changed + +- **`tfs-configure-model`: `binding_df` is now the `--binding_df` flag**, + no longer a positional argument (breaking for command-line callers; + `configure_model(binding_path, ...)` from Python still works). + `examples/simulate-and-analyze/run.sh`, the congression-calibration + study's `run.srun` and the docs are updated. +- **Breaking: a congressed cell's theta now follows a partition function + (`congression_theta_rule: homodimer`), and TF activity defaults to 1.** + The congression mixture (`transformation: mixture`) and the simulator + took a co-transformed cell's theta from its highest-theta plasmid. Both + now default to the homodimer partition function over equal plasmid + shares, `logit(theta_cell) = log sum_g x_g exp(l_g)` with `x_g = 1/M`; + `heterodimer` (`2 log sum_g x_g exp(l_g / 2)`) and the old `max` remain + selectable. The fit's rule is a new `ModelOrchestrator` setting + (`congression_theta_rule`, written to the config) and + `tfs-configure-model --congression_theta_rule`; the simulator key keeps + its name. The partition-function rules mix the variants' occupancies and + require TF activity 1: the simulator refuses otherwise, and + `ModelOrchestrator` refuses a mixture model with these rules and a + learned activity component. To match, the default activity component is + now `fixed` (was `horseshoe_geno`) in `ModelOrchestrator` and + `tfs-configure-model`. Configs that set neither key change behavior; + set `congression_theta_rule: max` and/or `activity_model: horseshoe_geno` + to keep the old ones. The example simulate configs now use `homodimer`. + See `planning/congression-physics-plan.md`, step 4. + +- **Gradient clipping is off by default in `tfs-fit-model` and + `tfs-prefit-calibration` (SVI, MAP, pre-MAP, prefit).** Every + congression-calibration fit on the step-3.5 grids ran to `max_num_epochs`, + and the lambda-1 seed-1 fits "blew up" to 4-5 times their best loss. + numpyro's `ClippedAdam` (the only optimizer, `adam_clip_norm=1.0`) clips each + gradient *element*. These losses have gradients of 1e3-1e6 per element, so + every element was clipped every step and each update followed the sign of + one draw's gradient; the fit settled where those signs balance, not where + the expected gradient vanishes. The ELBO was heavy-tailed: one binding + genotype (a step-like curve, `hill_n` 24.6, measured with SD 0.001) cost + ~2.4e5 nats whenever a guide draw crossed its step. At the sign-balance + point 5-90% of draws crossed it, depending on the run, and a step-size cut + cannot move such a point. The "blow-ups" were the block medians flipping + once that share passed 50%; the parameters barely moved. Resumed unclipped + from the blown-up checkpoints, the crossings fall to ~0 and the loss to + below the clipped runs' best, and a fresh unclipped fit converges (46k + steps). Study: `planning/studies/congression-calibration/diagnosis/`. + - The default optimizer is numpyro `Adam` (`inference/run_inference.py`, + `adam_optimizer`). `--adam_clip_norm` still selects `ClippedAdam`. + Step-size cuts keep whichever was chosen. Both share their state, so + checkpoints written under clipping resume unclipped. Checkpoints now + record `adam_clip_norm`. + - Results of fits made with the clip, notably the step-3.5 baseline and + lambda-profile grids, are biased toward whatever the clipped + equilibrium favored, and should be rerun. +- **The convergence stop also needs an unskewed loss, and posterior SDs used + to measure parameter movement are floored at 1% of the prior SD.** + - The loss test reads block medians, which ignore rare large penalties that + dominate the mean, the actual objective. Each window now also reports its + mean and its skew, `(mean - median) / (1.4826 MAD)` about the trend line + (`convergence.loss_skew`). At the final step size a window with skew above + `MAX_LOSS_SKEW` (3) is not a plateau, so such a run is reported rather + than called converged. Skew never forces a step-size cut. Benign ELBO noise + stays below ~1 (at most 0.4 on every recorded clean trace); the clipped + calibration runs gave 29-37. The statistic cannot see penalties carried + by most steps, but then the loss level shows them. + - The mean-field component guide collapses the guide SDs of hierarchical + scales (hill_mut `theta_sigma_d_*`, `theta_epi_tau`) to ~4e-4, far below + any honest posterior width. Measured in those units, a location creeping + ~2e-4 per window read as 0.3-0.5 SDs of movement at step size 1e-6 + forever. The SD used is now at least `PRIOR_SD_FLOOR` (0.01) times the + site's prior SD in the location's unconstrained units + (`initialization.site_unconstrained_prior_sds`), for the component guide, + AutoNormal and the AutoContinuous guides. + - `{out_prefix}_convergence.csv` gains `loss_mean`, `loss_skew` and + `loss_skewed`. A resume rewrites an older file under the new columns. + +- **Breaking: convergence and step size in `tfs-fit-model` and + `tfs-prefit-calibration`.** The stop rule divided the window-to-window loss + change by the improvement since the start of the run. On the step-3.5 + simulations the start-up loss was ~500 times the final loss, so fits stopped + (and printed "converged") while the loss was still falling 20-30% per 1000 + epochs; after a resume the same rule effectively never fired. The rule + before that (change relative to the loss itself) never fired on a noisy + ELBO. Both measured against an arbitrary scale instead of the loss's noise. + - Optimization now runs in windows of `--convergence_window_steps` optimizer + steps (default 2000, at least 10 epochs). A window's loss is *improving* + if a line through 20 block medians drops by more than `--convergence_z` + (3) standard errors and more than `--loss_rtol` (1e-6) of the loss. Each + parameter's trend is measured in posterior SDs (guide scale), prior SDs + (MAP) or log units (positive parameters), beyond noise and the step size's + resolution; it is *moving* above `--param_tolerance` (0.05). See + `inference/convergence.py`. + - The step size is now cut by `--adam_step_size_cut` (0.1) after + `--patience` (3) windows without loss improvement, down to + `--adam_final_step_size`. This replaces the exponential decay over + `max_num_epochs`, which collapsed the pre-MAP's step size within its own + 1000 epochs and made resumes depend on `max_num_epochs`. At the final step + size, `--patience` windows with neither loss improvement nor parameter + movement end the run. `--max_num_epochs` is only a cap, and a run that + reaches it says it has not converged and names the parameter still + moving. + - Removed flags: `--convergence_tolerance`, `--convergence_window` (replaced + by `--convergence_window_steps`, in optimizer steps) and + `--convergence_check_interval`. `--patience` now counts windows (default + 3, was 10 checks). `--init_param_jitter` was removed from + `tfs-prefit-calibration` (it never affected the MAP). Old templates that + pass the removed flags now fail at argument parsing. + - New output `{out_prefix}_convergence.csv`, one row per window (loss, drop, + standard error, parameter moving most, step size, decision). + `{out_prefix}_losses.txt` is now `epoch,loss,step,step_size`, with the + loss the median of each block. Checkpoints record the step size and the + monitor state; a resume continues at that step size. + - `RunInference.run_optimization` takes `convergence_window_steps`, + `patience`, `convergence_z`, `loss_rtol`, `param_tolerance`, + `final_step_size` and `step_size_cut`, in place of `convergence_tolerance`, + `convergence_window` and `convergence_check_interval`. +- **SVI and MAP start where they were meant to.** AutoDelta (MAP, the SVI + pre-MAP, `tfs-prefit-calibration`) ignored the configured guesses and + started at prior medians. The component guide ignored guesses keyed by site + name, and the pre-MAP result reached it under names it does not use + (`{site}_auto_loc`), so SVI never started from the pre-MAP. Now + `RunInference.site_values` collects site values from guesses and MAP + results, MAP runs start from them, and `RunInference.component_guide_start` + maps them onto the component guide's `{site}_loc(s)` parameters + (`inference/initialization.py`). The component guide's scales now start at + most `--guide_init_scale` (default 0.1; was the prior width, 1.0), as the + numpyro autoguides do. `--pre_map_num_epoch` is now a cap for a warm-up + that stops when it converges (default 10000, was a fixed 1000). On the smoke + data SVI starts ~18 times lower in loss, converges in 40k instead of 64k + steps, and ends ~185 nats lower. + +- **Breaking: congression is now an observable-level mixture + (`transformation: mixture`).** The old `empirical` component replaced each + bulk genotype's theta by the expected maximum over co-resident plasmids and + grew one trajectory with the genotype's own dk_geno. Averaging theta cannot + represent a mixture of growth rates, and in simulation that correction + removed almost none of the congression error once dk_geno varies + (`planning/studies/congression-estimator/`). The `mixture` component splits + each genotype's cells into a clean class and K congressed classes built from + the fixed co-resident sets: a congressed cell takes theta and TF activity + from its highest-theta plasmid (at each titrant concentration) and the mean + dk_geno of its plasmids (dilution). Every class is grown through the usual + `theta_rescale` → `calculate_growth` → `growth_transition` pipeline on a + leading class axis, and the classes are mixed as + `ln_cfu = ln_cfu0 + logsumexp_c(log w_c + G_c)`, with + `w_cong = bulk_fraction * (1 - P(M = 1))` split over the sets by + `P(M = n + 1)`, where a transformant carries `M` ~ zero-truncated + Poisson(lambda) plasmids that share its abundance (as in the simulator, so + the fit's lambda is the simulator's `transformation_poisson_lambda`; the + zero class of cells that never took up a plasmid is not observed). Theta noise acts on + the genotype's own theta before the classes are built; `theta_growth_pred` + is still the genotype's own theta. + - `empirical` and `logit_norm` are removed and refused by name with a + message; configs must switch to `transformation: mixture` (fits are not + comparable). + - New transformation interface: `define_model`/`guide` sample lambda only; + `cell_classes(focal, population, params, data)` returns the classes; + `NEEDS_POPULATION` replaces `NEEDS_FULL_POPULATION_THETA`. + - `GrowthData.congression_mask` is removed; purity is `bulk_fraction`. + - The `categorical_geno`/`empirical` incompatibility check is gone (fixed in + step 3.3a). + - A `mixture` model with bulk genotypes but an empty co-resident pool is + refused when built. + - `predict()` passes library-wide references and the fitted model's + co-resident sets to the genotype-subset model: dk_geno and activity are + posterior medians of their deterministic sites; theta is the theta + component evaluated at posterior-median parameters over the whole library + on the *prediction's* concentration grid (which can differ from the fit's). A raw MAP + checkpoint with a `mixture` model now raises (run `tfs-sample-posterior`) + instead of warning and predicting without a background. + - Growth tensors grow by a factor of 1 + K (K = 16 by default); use + mini-batching or a smaller `congression_sets` on large libraries. + +- **Breaking: the empirical pipeline's congression de-attenuation (Stage 1.5) + is retired.** `--congression_lambda` is removed from `tfs-fit-genotypes` + and `tfs-build-empirical`, together with `--library_config` and + `--save_theta_history` on `tfs-fit-genotypes` and the + `*_params_deattenuated.csv`, `*_theta_history.csv` and + `*_stage1p5_fits.csv` outputs. It corrected theta alone with the + expected-maximum operator the fit dropped above, and it used Poisson rather + than zero-truncated co-resident weights, a spiked/bulk split rather than + `bulk_fraction`, and no dk_geno dilution. Empirical distributions are now + built from uncorrected per-genotype fits. `genotype_fit/congression.py`, + `simulate/empirical/congression.py` and + `transformation/_congression.py` are deleted. A replacement is filed as + `planning/empirical-mixture-refit.md`. + +- **Test reports and badges are no longer committed.** `reports/` and + `docs/badges/` are gitignored and untracked; `run_all_tests.sh` now writes + everything, badges included, under `reports/`, and stops on the first + failing step. `README.rst` shows the live GitHub Actions status badge for + `tests.yml` instead. A new `.flake8` excludes `.claude/` (agent worktrees), + build and venv directories, which made up most of `reports/flake.txt`. + +- **Co-transformed cells in the simulator now grow by cell-level physics.** + A cell carrying several plasmids used to get the min/max/mean/... of its + plasmids' whole `k*t` (`multi_plasmid_combine_fcn`, per marker library). + Because `dk_geno` varies more across genotypes than the whole theta signal, + that let `dk_geno` decide which plasmid set the cell's growth. Now the + cell's theta and TF activity come from `congression_theta_rule` (`max`: + the highest-theta plasmid at each condition), its `dk_geno` from + `congression_dk_rule` (`dilution`: share-weighted mean), and its `k_pre`, + `k_sel` and `k*t` from the same growth model, theta rescale, growth + transition and tube noise as any genotype. Single-plasmid cells are + unchanged; at lambda = 0 output is identical to before. + - **Breaking:** `multi_plasmid_combine_fcn` is removed and a config that + sets it is refused with a message naming the new keys. Both new keys are + optional (defaults `max` and `dilution`). Example configs updated. + - With `transformation_poisson_lambda > 0`, `selection_experiment` needs + the config's `growth` block and phenotype columns `theta`, `activity`, + `dk_geno`, and checks that they reproduce the phenotype's `k_pre`/`k_sel` + (`_check_cell_components`), so the two paths cannot drift apart. + - New `simulate/cell_rules.py` (`THETA_RULES`, `DK_RULES`). New + `thermo_to_growth.growth_rates`/`growth_rate_one_condition` are the one + theta/activity/dk_geno -> k formula, used per genotype and per cell; + they replace `_apply_growth_params`. + +### Fixed + +- **Step-size cuts during a slow descent.** The convergence monitor cut the + step size after `patience` windows each without a significant loss trend, + even when those windows together were clearly still falling: on the + count-likelihood v2 grid, 20 of 23 runs made their first cut mid-descent + (pooled t = 4-14), and one (run 0012) was cut into a wrong optimum at + twice its twin's ELBO. Before a cut or a stop the monitor now fits one + line through all the stalled windows (`convergence.pooled_loss_trend`) and + starts the count over if that is a significant descent. Fits spend longer + at the large step size; more may reach `max_num_epochs`. New + `pooled_loss_t` column in `{out_prefix}_convergence.csv`; checkpoints + carry the stalled windows. +- **NaN from horseshoe priors far in the tail.** The slab-regularized + horseshoe scale `sqrt(c2 lam^2 / (c2 + tau^2 lam^2))` and numpyro's + HalfCauchy log density both square the local scale; a draw near 1e20 + overflowed float32 and turned the fit to NaN (hill_mut epistasis for + low-read doubles, whose guide local scales widened to ~11 in log space; + count-likelihood v2 run 0007). New `components/_horseshoe.py` + (`regularized_scale`, a `HalfCauchy` with a `hypot`-based log density), + used by hill_mut, activity `horseshoe_mut`, `thermo/horseshoe.py` and the + thermo `PK`/`PnnC`/`PddG` variants; same values, no overflow. +- **`tfs-prefit-calibration` took its Hessian at the wrong point.** + `RunInference.compute_hessian_sigmas` treated the MAP values it was given + as unconstrained, but its caller passes `svi.get_params(...)`, which + AutoDelta returns constrained; every positive site (hyper-scales, noise + scales, the count likelihood's `phi`) was evaluated at `exp(value)`. The + pre-fit's Hessian-derived `k_scale`/`m_scale_plus` were therefore off + (on a count-likelihood grid run, k sigmas of 0.0012-0.0029 against 0.0003-0.0007 at the + right point, so the written `k_scale` was ~0.0028 where the 0.002 floor + now applies), and on the count likelihood `phi` overflowed and wrote + NaN scales that crashed the fit. It now maps the values to unconstrained + space first. `tfs-sample-posterior` and `tfs-extract-params` read the + optimizer state (already unconstrained) and were not affected. +- **Count likelihood NaN for genotypes predicted near extinction.** The + negative binomial's concentration `mu / phi` underflowed float32 when the + expected reads fell below about `e^-40` (reached by genotypes with zero + reads everywhere, whose likelihood keeps rising as `mu` falls), and the + log-pmf's gradient and Hessian went to NaN; SVI then exploded within its + first 250 steps. The concentration now has a floor of `e^-30` + (`growth_counts._LOG_C_FLOOR`), which changes only rows expecting under + ~1e-12 reads. +- **Prediction with a per-tube `sample_offset`.** `tfs-predict-growth` (and + anything calling `analysis.prediction.predict` on a new time or + concentration grid) failed to reshape the per-tube offsets of + `sample_offset: normal` (and now `level`) onto the prediction grid's + tubes. Per-tube offsets are now set to zero there: a prediction is for a + typical tube. +- **Absent prior groups survive the priors CSV.** `write_configuration` + wrote a `None` prior group (growth priors of a binding-only model, binding + priors of a growth-only one) as a `None` row that reloaded as NaN; it is + now left out and reloads as `None` (`configuration_io._extract_scalars`, + `_update_dataclass`). +- **MAP parameter movement silently lost all its prior-SD units on models with + a horseshoe slab.** `initialization.site_prior_sds` did not catch the + `ZeroDivisionError` numpyro raises for the variance of an `InverseGamma` + with Python-float concentration 2 (the slab prior with the default + `slab_df` 4). The whole calculation then fell back to unconstrained units + for every parameter ("Could not compute prior SDs for MAP parameter + movement" in the pre-MAP log of every hill_mut epistasis fit). An undefined + variance now skips only its own site. + +- **Library enumeration counted every codon of a tile as a site, flooding + partially degenerate libraries with wt.** `LibraryManager._prepare_indexes` + (`genetics/library_manager.py`) indexed every block in a tile, so each + non-degenerate codon (and each out-of-frame flank) added one wt copy to every + single-mutant sub-library, and each such codon paired with a degenerate one + added a full single-mutant expansion to the double-mutant sub-libraries. A + site is now a codon whose designed sequence differs from wt (a degenerate or + explicit mutant codon). Fully degenerate tiles, including the real design and + `examples/process_raw`/`examples/simulate`, enumerate exactly as before. + Partially degenerate ones change a lot: in + `planning/studies/congression-calibration/simulate_config.yaml`, `single-1` + goes from 46 sequences (30 wt) to 16 (0 wt), `single-2` from 67 (36 wt) to + 32 (1 wt), and `double-1-2` from 3082 (1080 wt) to 512 (0 wt), and wt's share + of the co-resident pool drops from 0.39 to 0.003 (0.59 to 0.015 for + `examples/simulate-and-analyze`). Everything built on the enumeration moves + with it for such libraries: the simulator's transformation draws, + `tfs-report-cfu0`, and `tfs-configure-model`'s library snapshot + (`pool_fraction`, `bulk_fraction`) and hence the fit's co-resident sets. + Simulations and fits made from partially degenerate configs should be + regenerated. Genotype names are unchanged. `tfs-process-fastq` no longer + expects the exact wt DNA sequence unless a site's expansion or a spiked + sequence contains it (it used to come from the non-degenerate codons). A + tile with no + site is now an error, and a design that encodes no wt at all (no site's + codon encodes the wt amino acid and no spiked sequence is wt) warns, since wt + is then not an expected genotype. + +- **The SVI loss file and epoch checkpoints were numbered from the end of the + pre-MAP**, because the step counter carried over from the pre-MAP to SVI on + the same `RunInference`; the SVI `losses.txt` also lost its header. A run + started without a state now resets the counter. + +- **`tfs-setup-sim-grid` grids could not be moved, and nested file paths were + not resolved.** Each run's `tfs_sim_config.yaml` and rendered template + referred to input files by a path relative to their original location, so + moving the grid (on or off a cluster, to another partition) broke every run. + Nested file keys (`binding_data.*.choose_by` params files, + `empirical.phenotype_model`) were not resolved at all and so needed the run + template to copy the file in. Setup now copies every input file into + `/inputs/` once and each run refers to it as + `../inputs/`, so the grid directory is self-contained. The file-path + keys are listed in `_SIM_PATH_KEYS` + (`simulate/scripts/setup_sim_grid_cli.py`), nested ones included; the + `stratified`/`random` `choose_by` keywords are left alone, and template + variables naming a file are copied the same way. Two different files with + the same name get a numeric suffix, and a changed file never overwrites a + copy that existing runs use. Setup fails before writing anything if an input + file is missing or is a directory, or if a config value outside + `_SIM_PATH_KEYS` names an existing file. The congression-calibration study's + `run.srun` no longer copies `hill_params.csv`, and its grid YAMLs drop the + `hill_params_file` template block. +- **`tfs-setup-grid` grids could not be moved.** Each run's + `tfs_configure_config.yaml` and rendered template referred to the input + files (`growth_df`, `binding_df`, `library_config`, ...) by a path relative + to their original location, so moving the grid directory (on or off a + cluster, to another partition) broke every run. Setup now copies every input + file into `/inputs/` once and each run refers to it as + `../inputs/`, so the grid directory is self-contained. The inputs are + the `configure_model` file arguments in `_PATH_KEYS` + (`tfmodel/scripts/setup_grid_cli.py`) and template variables naming a file; + the written config names the copy everywhere it recorded the file, + `library.source` included. The priors/guesses/library CSVs are per-run + outputs and are unchanged. Two different files with the same name get a + numeric suffix, and a changed file never overwrites a copy that existing + runs use. Setup now fails before writing anything if an input file is + missing or is a directory (previously a missing data file quietly skipped + the combination), if a `configure_model` value outside `_PATH_KEYS` names a + file, or if the template does not render. The staging helpers moved from + `setup_sim_grid_cli.py` into `util/grid_utils.py` (`InputStager`, + `check_no_outside_paths`, `stage_template_vars`, `render_run_template`), + and both grid CLIs use them; the unused `relativize_*` path helpers were + removed. +- **Growth prediction failed on every model built from a library file.** + `tfs-predict-growth`, and everything else that goes through + `prediction.predict` (prior-predictive growth, the genotype trajectory + plots), raised. `prediction.copy_orchestrator` rebuilt the prediction model from + `orchestrator.settings`, which carries both `library_file` and the spiked + list derived from it, and `ModelOrchestrator` refuses the pair. It now + drops the derived list when `library_file` is set, as + `write_configuration` does. Configs from `tfs-configure-model + --library_config` (the only current path) were affected; legacy + `spiked_genotypes` configs were not. + +- **Per-genotype latents scrambled across genotypes under autoguides.** + Several components sampled their per-genotype latents in a plate sized + `data.batch_size`, so every numpyro autoguide -- including the `AutoDelta` + behind MAP, pre-MAP and `tfs-prefit-calibration` -- held one parameter per + batch *position* rather than per genotype. The full-batch index is + binding-first and reshuffled every step, so this happened even without + mini-batching: binding genotypes kept their positions, but every bulk + genotype's latents were overwritten by whichever genotype landed in that + position. The component SVI guide was not affected, since it holds + library-sized parameters and slices them itself. + - `activity/hierarchical_geno`, `activity/horseshoe_geno`, + `dk_geno/hierarchical_geno`, `ln_cfu0/hierarchical`, + `ln_cfu0/hierarchical_factored`, `theta/categorical_geno` and + `noise/logit_normal` now sample these latents in a library-sized + `{name}_genotype_plate` and slice them to the batch with + `data.batch_idx`. The latent priors and guides that were wrapped in + `scale_vector` no longer are; only the likelihood is scaled. + - The theta-noise components (`zero`, `beta`, `logit_normal`) take an + optional `data` argument, which `generative/model.py` now passes. + - `noise/beta` is a documented exception: its `{name}_dist` latent is the + noisy theta itself and remains batch-shaped. + - `RunInference.get_map_posteriors` and `get_laplace_posteriors` now raise + on a MAP checkpoint whose genotype-indexed latents are not library-sized, + instead of clipping the indices and reusing one genotype's value for + another. **MAP checkpoints written before this fix must be refit.** + - `draw_prior` no longer returns plate sites as if they were latents. +- **New `tfmodel/inference/batch_safety.py`** detects batch-dependent latents + by tracing a model at two batch sizes (abstractly, under `jax.eval_shape`). + `tests/tfscreen/tfmodel/inference/test_batch_safety.py` runs it across the + component registry. `orchestrator_latent_dimension` gives the size of the + flattened latent vector an `AutoContinuous` guide works in. +- **`RunInference.run_optimization` refuses unsafe autoguide fits.** When the + guide is any numpyro `AutoGuide` (including the `AutoDelta` behind MAP, + pre-MAP and `tfs-prefit-calibration`), it first runs the batch-safety check + and raises a `ValueError` naming the offending sites -- for example, a MAP + fit with `theta_growth_noise=beta`. The component guide is not checked. For + an `AutoMultivariateNormal` guide it also reports the latent dimension and + the memory the dense covariance needs, and warns above 4 GB. + +- **`tfs-summarize-fit` could pick the wrong config or losses file in a grid + run directory.** It matched `*_config.yaml` and `*_losses.txt`, which there + also match the simulate config and the prefit/pre-MAP losses; the right file + won only by alphabetical order. It now takes the config that has `data` and + `components` sections, and ignores `*_prefit_losses.txt` and + `*_premap_losses.txt`. + +- **Simulator counted a co-transformed cell once per plasmid.** + `_sim_sequencing` and `_calc_genotype_cfu0` + (`simulate/selection_experiment.py`) gave every plasmid in a cell that + cell's whole abundance, so a cell carrying n plasmids contributed n cells' + worth of reads and `ln_cfu_0`. Bulk genotypes (transformed with + `transformation_poisson_lambda`) were inflated relative to the monoclonal + spikes by `E[n | n >= 1] = lambda / (1 - exp(-lambda))` (1.19 at + lambda = 0.357), so a genotype's simulated spike/bulk make-up depended on + lambda. A cell's abundance is now split among its plasmids (1/n each; new + helper `_plasmid_shares`), so each cell counts once and the simulated + composition matches the design `pool_fraction`/`bulk_fraction` from + `genetics.library_composition_table` at any lambda. **Simulated read counts + and `ln_cfu_0` change for any config with lambda > 0.** Growth itself + (`_sim_growth`) is unchanged. + +- **Batch-order bug in three more components.** `activity/hierarchical_mut` + and `activity/horseshoe_mut` sliced activity to the batch only when + `batch_size < num_genotype`, and `theta/_simple` indexed its per-genotype + theta by the batch-relative `geno_theta_idx` alone. The full-batch training + index is binding-first and reshuffled every step, so at full batch these + returned library-ordered values for reordered batch positions: bulk + genotypes got other genotypes' activity (or, for `_simple` with + per-genotype curves, theta). The shape-based batch-safety check could not + see it. All three now slice through `batch_idx`. Fits using + `hierarchical_mut`/`horseshoe_mut` activity without mini-batching should be + rerun. The pre-fit's `_simple` model contains only binding genotypes, whose + index is library order and never reshuffled, so pre-fit results are + unaffected. + - New `find_batch_order_mismatches` / + `find_orchestrator_batch_order_mismatches` in + `inference/batch_safety.py`: trace two orderings of the full batch with + the latents held fixed and check that the batch-positional predictions + follow the index. `test_batch_safety.py` runs it across the component + registry and the epistasis and binding-only variants. + +- **`theta/categorical_geno` full-population theta.** The component sliced + its per-genotype latent to the batch when sampling, so the full-population + theta `generative/model.py` builds for the congression background (by + calling `run_model` with `batch_idx = geno_theta_idx = arange(num_genotype)`) + came back in batch order at full batch and indexed past the batch under + mini-batching, repeating the last genotype. The empirical correction's + background CDF is order-free, so full-batch fits were unaffected; fits + combining `categorical_geno`, `transformation=empirical` and mini-batching + used a corrupted background and should be rerun. theta now stays + library-sized and `run_model` indexes it through + `batch_idx[geno_theta_idx]` (the `hill_geno` pattern). The `theta_theta` + deterministic site is now stored for every genotype. + - New `tests/tfscreen/tfmodel/generative/test_population_theta.py` + checks, for every theta component (model and guide; shuffled full batch + and mini-batch), that the full-population theta is library-ordered. + +- **Posterior files depended on `forward_batch_size`.** `get_posteriors`, + `get_laplace_posteriors` and `get_map_posteriors` run the model over + genotype chunks, and sliced every genotype-indexed latent to the chunk + first. Components that index library-sized parameters by library position + (`hill_geno`, `thermo.*`, and `categorical_geno`) then read past the sliced + arrays for every chunk after the first, and JAX clamped the index instead + of raising. So for any library larger than one chunk (512 genotypes by + default), the deterministic sites written for genotypes past the first + chunk (`theta_growth_pred`, `growth_pred`, and everything + `tfs-predict-theta`/`-growth` derive from them) were wrong. The forward pass + now receives library-sized latents whole, as in training, and slices only + batch-positional latents (`noise/beta`) and the saved output + (`RunInference._batch_positional_latents`). Posterior files for libraries + larger than `forward_batch_size` should be regenerated with + `tfs-sample-posterior`. + - New `tests/tfscreen/tfmodel/inference/test_forward_chunking.py` checks + that posterior and MAP posterior files are identical at + `forward_batch_size` 512 and 2. ## [0.4.4] - 2026-09-22 @@ -677,7 +2139,8 @@ genotypes are handled. - **Fitting routines** (`tfscreen.fitting`): OLS, WLS, GLS, GEE, GLM, NLS, matrix NLS/WLS, and Kalman / unscented Kalman filter estimators. - Walkthrough notebook (`notebooks/tfscreen-walkthrough/`). -[Unreleased]: https://github.com/harmslab/tfscreen/compare/v0.4.4...HEAD +[Unreleased]: https://github.com/harmslab/tfscreen/compare/v0.5.0...HEAD +[0.5.0]: https://github.com/harmslab/tfscreen/compare/v0.4.4...v0.5.0 [0.4.4]: https://github.com/harmslab/tfscreen/compare/v0.4.3...v0.4.4 [0.4.3]: https://github.com/harmslab/tfscreen/compare/v0.4.2...v0.4.3 [0.4.2]: https://github.com/harmslab/tfscreen/compare/v0.4.1...v0.4.2 diff --git a/CLAUDE.md b/CLAUDE.md index 29637e29..0fd24059 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,6 +29,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ~/miniconda3/bin/coverage run --branch -m pytest tests/tfscreen --runslow ``` +### Dependencies + +`pyproject.toml` floors every dependency at the oldest versions the suite has passed with (no caps: CI installs the newest). No lock or constraints file lives in the repo (user, 2026-10-20: not at the top level beside `src`, `docs`, `tests`); an analysis records its own `pip freeze`. tfscreen needs Python 3.12+ (jax 0.11 dropped 3.11); CI tests 3.12, 3.13 and 3.14. The `cuda12` extra is GPU jax on Linux, where pip's plain jax is CPU-only. `planning/studies/fresh-install/fresh_install.srun` checks a clean install on a cluster GPU node. + ### Linting ```bash # Fatal errors only @@ -40,34 +44,37 @@ flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 ### CLI Entry Points ``` -tfs-process-fastq # FASTQ → read counts -tfs-process-counts # counts → ln_cfu DataFrames -tfs-process-presplit # Process pre-split count files -tfs-configure-model # Generate YAML config template (+ library composition snapshot) +tfs-process-fastq # FASTQ → read counts (library_config f1 f2 --out_dir; library YAML first) +tfs-process-counts # per-tube counts → growth table (tube table + counts dir; tube totals from --od600_file/--od600_calibration_file/--tube_volume_mL or supplied sample_cfu; every library in one call; --presplit writes the presplit table) +tfs-calibrate-od600 # Fit an OD600-to-CFU/mL calibration from a dilution series of replicate readings + plate counts (process_raw/od600.py) +tfs-check-tubes # Tube QC on the growth table: each tube's apparent titrant from its genotype composition (centered log-ratio, robust row effect + per-row-scaled concentration profile, so tube totals cancel and incomplete rows are not shifted); flags tubes out of order in their replicate x time row (leave-one-out isotonic order test + level test vs the label's expected score, --flag_z noise SDs), a likely tube/sample-sheet swap; rows of < 3 tubes are never flagged; process_raw/tube_qc.py +tfs-correct-tubes # Apply a YAML corrections file (swap/relabel/exclude, each with id, reason, by, date, evidence) to a tube table before tfs-process-counts; writes a new table (+ correction_id), {out_prefix}_changes.csv and provenance with input hashes; only DESIGN_COLUMNS move, measurements stay with the physical sample (process_raw/tube_corrections.py) +tfs-process-binding # Lab binding table (one row per measurement) -> binding_df at one protein concentration (--protein_conc, default 2 uM): concentrations rounded to 6 significant digits so days line up, replicates (--replicate_cols, default biorep,date) averaged, theta_std = pooled replicate SD / sqrt(n) or --theta_std; titrant written in mM (process_raw/binding.py) +tfs-configure-model # Generate YAML config template (+ library composition snapshot); --set_priors name=value ... sets scalar priors (held SDs, sigma_fixed); --growth_priors / --growth_priors_wt_rates set per-condition linear k/m priors where the pre-fit does not run, and --growth_priors's 0/1 k_pinned/m_pinned columns clamp chosen conditions (priors_edit.py); a library spike with no growth data is an error unless --allow_missing_spikes tfs-prefit-calibration # Pre-fit linking function via MAP -tfs-fit-model # Main hierarchical Bayesian inference -tfs-fit-genotypes # Per-genotype MLE fits of the growth model (+ optional congression de-attenuation; spiked genotypes come from --library_config) -tfs-sample-posterior # Draw posterior samples from fitted model +tfs-fit-model # Main hierarchical Bayesian inference (--guide_type component|delta|auto_normal|auto_diagonal_normal|auto_multivariate_normal|auto_low_rank_multivariate_normal; --guide_rank, --guide_init_scale; convergence: --convergence_window_steps, --patience, --convergence_z, --loss_rtol, --param_tolerance, --adam_step_size -> --adam_final_step_size by --adam_step_size_cut; --adam_clip_norm opts back into elementwise gradient clipping, off by default; --init_from starts at an earlier MAP's {site}_auto_loc values, e.g. a level-offset fit from a zero-offset fit, since guesses cannot name sites outside the configured guesses; --stage_offsets auto|on|off, --staged_step_size: a fresh MAP with level tube offsets runs the staged MAP, inference/staged_map.py) +tfs-fit-two-stage # The two-stage relative fit as one command (stage 1 wt+spikes+singles with the congression mixture -> jackknife of stage 1 -> stage 2 full library with k/m pinned -> X (the jackknife and k-shift refits that add k's uncertainty as a shared per-draw shift are off by default, two_stage.kshift: false, since validation run 2 found the shift a third or less of the true one; when on: refits from stage 2's MAP at kshift_step_size 1e-5, dk_geno population held at stage 2's MAP, a k-unchanged control refit, kshift/drift.csv) -> marker-consistency check -> summary: {out_dir}/tfs_x.csv, X quantiles + k_shift + reads (total, per library) + num_mutations + consistency z/flag per genotype x concentration, the users' table); YAML = tfs-configure-model's args + a two_stage block (examples/tfmodel/two_stage.yaml); resumable, --step, --jackknife_task for arrays, --no_consistency; tfmodel/two_stage.py, planning/two-stage-fit.md +tfs-fit-genotypes # Per-genotype MLE fits of the growth model (no congression correction) +tfs-sample-posterior # Draw posterior samples from fitted model (MAP: --laplace auto|full|arrowhead|blocks|point; auto = full up to --laplace_max_params 20000 MAP parameters, arrowhead above; arrowhead writes {out_prefix}_held_directions.csv; blocks holds the shared parameters at the MAP; point = the MAP point, no Hessian; --skip_growth_observations: leave out growth_pred/growth_obs, most of the file on a large library) tfs-sample-prior # Draw prior predictive samples -tfs-extract-params # Extract parameters from checkpoint +tfs-extract-params # Extract parameters from checkpoint (+ {out_prefix}_model_params.csv: learned model-side pyro.params, e.g. the thermo PnnC NN weights) tfs-predict-growth # Predict growth from fitted model -tfs-predict-theta # Predict operator occupancy -tfs-predict-epistasis # Joint second-order epistasis from the theta posterior (per-draw ep, then quantiles; captures cross-genotype posterior covariance the marginal tfs-extract-epistasis path drops) +tfs-predict-theta # Predict operator occupancy (--shared_shift adds one shared per-draw X shift, the two-stage fit's k uncertainty; analysis/shared_shift.py) +tfs-predict-epistasis # Joint second-order epistasis from the theta posterior (per-draw ep, then quantiles; captures cross-genotype posterior covariance the marginal tfs-extract-epistasis path drops; --shared_shift as tfs-predict-theta) tfs-cat-response # Fit categorical response curves tfs-extract-epistasis # Calculate second-order epistasis from a long-form observable table (--scale add|mult|logit; --scale_constant rescales the transform before epistasis, e.g. -RT to put logit onto a free-energy scale) tfs-compare-runs # Cross-run agreement statistics for any quantile-summarized estimate -- predicted features (theta/growth/epistasis) or fitted parameters (log_hill_K/dk_geno/growth_k/k_ref) -- across N runs (seeds / k-fold dropouts). Reports raw numbers (rms_sd, overdispersion + p/q, n_present/n_runs); no thresholds, no grades. --index_by picks the entity, --group_by breaks it out further, --match_by overrides key detection -tfs-diagnose-nan # Diagnose NaN issues in inference -tfs-simulate # Simulate a full experiment -tfs-report-cfu0 # Report average ln_cfu0 by genotype class from a simulate config +tfs-simulate # Simulate a full experiment (config_file --out_prefix tfs_sim → {out_prefix}_{name}.csv; also the raw experiment, {out_prefix}_counts/ + _tubes.csv [+ _tube_od600.csv, _od600_calibration.yaml], for tfs-process-counts, whose command it prints; --no_write_raw skips it; --no_write_growth skips the multi-GB direct growth table; cfu0 may be a {library: cells} map; total_num_reads is the total over all replicates, split by tube count (replicate_read_totals; until 2026-10-08 every replicate got the full total); unassigned_read_fraction (scalar or per library) sends that share of each tube's reads to __unknown__; single-plasmid transformants are merged into one weighted row per genotype (_sim_transform_collapsed, exact for founder/demographic noise), so transform_sizes can be realistic; with an od600 block, tubes read below the detection threshold are dropped from the direct growth table as from the raw tube table (raw_output.undetectable_tubes; logged), so both routes hold the same tubes (until 2026-10-11 the direct table kept them), and max_congressed_cells caps the multi-plasmid cells; lib_assembly_skew_sigma may be per origin; thermodynamic theta models also write {out_prefix}_mutation_parameters.csv and, with multi-mutants, _pair_parameters.csv: the per-mutation d_*/ddG_* and per-pair epi_* truth; library_genotypes (list file) / library_subsample ({origin: N}, needs library_subsample_seed) restrict the library before anything is simulated, see **Library subsample**) tfs-build-empirical # Fit real data → empirical phenotype-generating distribution (Stages 1-2) tfs-setup-sim-grid # Set up grid of simulation runs tfs-setup-grid # Set up grid of model configs tfs-summarize-grid # Summarize grid results -tfs-summarize-fit # Summarize a fitted model -tfs-summarize-sbc # Summarize simulation-based calibration runs -tfs-subset-genotypes # Subset genotype data +tfs-summarize-fit # Summarize a fitted model (and, with tube offsets, the offset-mode check: offsets in prior SDs, Spearman trend vs titrant/time per condition, BH-flagged; tfmodel/analysis/tube_offsets.py) +tfs-summarize-calibration # Pool posterior calibration (coverage, PIT, width, RMSE) across a tfs-setup-sim-grid grid, by arm and stratum; --baseline pairs runs fit to the same simulated data ``` +Removed in 0.5.0: `tfs-process-presplit` (now `tfs-process-counts --presplit`), `tfs-diagnose-nan`, `tfs-subset-genotypes`, `tfs-report-cfu0`, `tfs-summarize-sbc` (the library function `error_calibration.summarize_sbc` remains). `docs/source/cli.rst` is the generated reference of every command's flags. + ## Architecture ### Data Flow @@ -110,10 +117,13 @@ The hierarchical Bayesian inference engine. Key files: - **`generative/model.py`** — Numpyro probabilistic model definition. The generative model is: `ln_cfu = ln_cfu0 + (k_pre + dk_geno + m_pre·A·θ)·t_pre + (k_sel + dk_geno + m_sel·A·θ)·t_sel` where θ = operator occupancy, A = per-genotype TF activity, dk_geno = pleiotropic growth effect. + With `transformation: mixture` each genotype's cells are split into classes (clean, plus K congressed classes built from fixed co-resident sets) with cell-level θ/A/dk_geno (θ by `congression_theta_rule`, default the homodimer partition function over equal plasmid shares; dk_geno by `congression_dk_rule`, default dilution); every class is grown through the same pipeline on a leading class axis and the classes are mixed as `ln_cfu = ln_cfu0 + logsumexp_c(log w_c + G_c)`. The transformation component's `cell_classes(focal, population, params, data)` builds the classes (`single`: one class); `NEEDS_POPULATION` makes `jax_model` supply library-ordered θ/activity/dk_geno (`return_population=True`, or `GrowthData.external_*_population` in prediction). - **`model_orchestrator.py`** — `ModelOrchestrator`: top-level class orchestrating data loading, inference, and prediction. -- **`inference/run_inference.py`** — `RunInference`: coordinates JAX/Numpyro sampling (SVI or NUTS) and MAP estimation (optax). +- **`inference/run_inference.py`** — `RunInference`: coordinates JAX/Numpyro sampling (SVI or NUTS) and MAP estimation (optax). **Guide selection:** `setup_svi(guide_type=..., guide_kwargs=..., init_values=...)` takes any name in `GUIDE_TYPES` — `component` (the guide assembled from the model components) or a numpyro autoguide from the `AUTOGUIDES` registry (snake_case of the class name, `delta` = `AutoDelta`; CamelCase class names accepted via `resolve_guide_type`). `tfs-fit-model --guide_type` exposes it for `analysis_method=svi`; an autoguide's location starts at the pre-MAP point (`init_to_value`), and `init_params` (component-guide param names) is not passed to it. Checkpoints record `guide_type`/`guide_kwargs`; `restore_svi_from_checkpoint` rebuilds that guide (legacy checkpoints → `component`) and `tfs-sample-posterior` routes `delta` → Laplace, everything else → guide sampling. `run_optimization` refuses any autoguide (AutoDelta included) when `inference/batch_safety.py` finds batch-dependent latents. Because per-genotype latents are library-sized, the `AutoContinuous` guides (incl. `auto_multivariate_normal`) are valid under mini-batching; the dense MVN just costs O(D²) memory (reported, warned above `_DENSE_GUIDE_WARN_GB`). + +- **`inference/convergence.py`** — when to cut the step size and when to stop; see **Convergence and step size** below. - **`tensors/tensor_manager.py`** — `TensorManager`: maps ragged per-genotype observations into JAX-compatible tensors. @@ -121,18 +131,36 @@ The hierarchical Bayesian inference engine. Key files: - **`generative/components/`** — Pluggable model components selected via the YAML config's `components:` section (registry key noted in parens where it differs from the directory name): - `activity/`: `fixed`, `hierarchical_geno`, `hierarchical_mut`, `horseshoe_geno`, `horseshoe_mut` - - `growth/` (registry key `condition_growth`): `linear`, `power`, `saturation`. Their per-condition prior loc/scale fields accept a scalar (broadcast to all conditions) **or** a per-condition array; the pre-fit calibration writes per-condition arrays to pin the baselines (see **Per-condition growth priors** below). Each declares `get_scale_bounds()`. + - `growth/` (registry key `condition_growth`): `linear`, `power`, `saturation`. `linear` also takes `m_warp` (bool or per-condition tuple, a 0/1 `--growth_priors` column: that condition's growth follows a logistic warp f(theta), f(0) = 0 and f(1) = 1, steepness a and threshold x0 sampled per warped condition; for markers whose occupancy-to-growth maps differ in shape, `planning/studies/relative-m/`; extracted as `warp_log_a`/`warp_x0` per warped condition; runs with `hill_geno` and the mixture's partition rules, acting on each cell class's theta), `k_titrant_loglinear_scale` (> 0: a selection-phase k shift linear in log titrant concentration from a tenth of the lowest nonzero one, in selection conditions only, shared by every genotype, 0 at no titrant; needs the full growth-tensor condition map; a free shift per concentration was dropped as unidentified, since it trades with m along wt's curve; `planning/studies/within-tube-m/`). Their per-condition prior loc/scale fields (in `linear`, `m_scale_plus`/`m_scale_minus` too) accept a scalar (broadcast to all conditions) **or** a per-condition array; the pre-fit calibration writes per-condition arrays to pin the baselines (see **Per-condition growth priors** below). Each declares `get_scale_bounds()`. - `growth_transition/`: `instant`, `memory`, `baranyi`, `baranyi_k`, `baranyi_tau`, `two_pop` - - `transformation/`: `empirical`, `logit_norm`, `single` - - `theta/`: `categorical_geno`, `hill_geno`, `hill_mut`; thermodynamic partition-function variants under `theta/thermo/` (lac dimer and MWC dimer, with/without unfolded state, PK/PnnC/PddG parameterizations) + - `transformation/`: `single`, `mixture` (`empirical`/`logit_norm` were removed; `ModelOrchestrator` refuses them by name with a message, `_RETIRED_TRANSFORMATIONS`). The old E[max] θ operator (`_congression.py`) is gone, along with Stage 1.5 of the empirical pipeline that used it + - `theta/`: `categorical_geno`, `hill_geno` (with `theta_growth_log_conc_shift_scale` > 0, a global log s site `theta_growth_log_conc_shift`: growth sees each curve at s times nominal titrant, binding at nominal, applied in `model.py`'s `growth_titrant` to the growth data only, including the mixture population and `prediction._population_theta`; off by default; `tfs-predict-theta` reports the curve in binding units; joint-fit plan step 4a), `hill_relative` (wt-relative X, see **Relative-X fit** below), `hill_mut`; `hill_geno`/`hill_relative` take `theta_log_hill_n_hyper_scale_fixed` (> 0 holds the log(n) population SD; `hill_geno` also `theta_{logit_low,logit_delta,log_hill_K}_hyper_scale_fixed`, joint-fit-sim run 5; learned, a MAP on real data ran it to about 19, and n near 0.01 hid each genotype's response below the lowest nonzero concentration, in the 0-titrant tubes); `hill_relative` also takes `theta_{X_low,X_delta,log_hill_K}_hyper_scale_fixed` (the dev-data MAP ran them to 3.6, 4.3 and 11 against about 0.45, 0.45 and 0.7 among well-measured genotypes, so it shrank nothing; `planning/dev-data/real_fit/make_init_hs.py` rescales a MAP's non-centered offsets for a warm start at the held values); thermodynamic partition-function variants under `theta/thermo/` (lac dimer and MWC dimer, with/without unfolded state, PK/PnnC/PddG parameterizations; in the U1 PddG/PnnC models mutations shift each folded structure by ΔΔG_X and leave U alone (ΔΔG_U = 0), so with K_U = [U]/[H] the per-mutation `d_ln_K_U` (C12: `d_ln_K_u`) is ΔΔG_H (`_PROJ_U`, `_project_unfolding`; `epi_ln_K_U`/`epi_ln_K_u` per pair), the names PK uses for its free shift; until 2026-10-10 they held ln_K_U at wt for every genotype, which moved U with H and made every mutation stability-neutral; like `hill_mut`, all twelve skip their per-mutation sites when `num_mutation == 0`, in `define_model`, `guide` and `prior.sample_ddG`, since numpyro plates need size > 0, so a wt-only library draws; **physics** (`theta/thermo/mwc.py`, shared by all twelve): the standard MWC dimer, two independent equivalent effector sites per conformation, P_X = (1 + K_x_e·e)² with K_x_e per site (M⁻¹); cooperativity only from the H/L equilibrium. Until 2026-10-11 C12 used Sochor's 1 + K·e + K²·e² (K2/K1 = 1, not 1/4) and C4 a one-step L + 2E with an M⁻² K_E. M3 = C12 (K_h_l, K_h_o, K_h_e, K_l_o, K_l_e); M2 = C4 is M3 with K_h_e = K_l_o = 0 (ln_K_op ↔ K_h_o, ln_K_HL ↔ K_h_l, ln_K_E ↔ K_l_e per site), computed by the same `mwc.mwc_theta` (`theta_c4`/`theta_c12`), so the nesting is exact; the C4/K3 labels are historical. Bookkeeping: TF ≫ operator, so operator-bound TF is out of the TF mass balance and there is no `op_total` anywhere (removed from priors, ThetaParam, build_calc_df, extraction); h = r/F(e), r = tf_total/2 (monomer units), F = B0 + B1·e + B2·e² (B0 = 1 + K_h_l + K_u, B1 = 2(K_h_e + K_h_l·K_l_e), B2 = K_h_e² + K_h_l·K_l_e²), W = h·(K_h_o·P_H + K_h_l·K_l_o·P_L), θ = W/(1+W); free effector from the cubic B2x³ + (B1 + B2(2r − e))x² + (B0 + B1(r − e))x − B0·e = 0. **All of it in log space** (since 2026-10-11; `mwc.log_W`, `mwc_theta_log`): log-sum-exp for every partition-function sum, log P = 2·softplus(ln K + log e), θ = sigmoid(log W), so no ln K is exponentiated and any finite ln K gives a finite θ in [0, 1] in float32 with the right limits. Before, θ came from exp(ln K): K² overflows float32 above ln K 44, and Laplace draws of the PddG pilot fits (ln K to ±60) wrote NaN θ for whole genotypes (15-71 per run). The free effector is solved for log e by safeguarded Newton (`_solve_log_free_effector`: bracket [x_lo, log e_total], rtsafe bisection fallback, `NEWTON_ITERATIONS` = 16; 12 converged exactly in float64 over ln K uniform on ±150) on a residual that stays conditioned on both sides of the equivalence point e_total = 2r (`_mass_balance`: log(e + r·n̄) − log e_total for e_total ≤ r, log(e + c⁻) − log(c⁺ + r·(2 − n̄)) above, c = e_total − 2r); at e_total ≈ 2r with huge K_E θ is genuinely ill-posed at float precision, so tests avoid comparing there. Derivatives of the root come from the implicit function theorem (`jax.custom_jvp`), not from the iterations, so gradients and the Laplace Hessian are exact and finite at extreme values. `mwc_theta`, `free_effector` and `theta_saturated` keep their linear-K signatures (tests) and log their inputs. `tfs-predict-theta` warns (`warn_nonfinite_theta`) if any output row is still NaN or infinite. Projection rows (`mwc.PROJ_C4/PROJ_C12/PROJ_U_*`, imported as each PddG/PnnC `_PROJ`): effector rows are per site, (ΔΔG_X − ΔΔG_XE2)/2, so C4 Δln K_E = (ΔΔG_L − ΔΔG_LE2)/2. **Defaults**: tf_total 1e-5 M monomer; C12 keeps Sochor's K_h_l 6.3, K_h_o 4.2e8, K_l_o 0.1 but not his K_l_e/K_h_e ≈ 13.6, which left the default wt at θ_sat 0.64 (residual HE2 repression at 1e-5 M TF). Since 2026-10-10 ρ = K_l_e/K_h_e = `MWC_EFFECTOR_SELECTIVITY` ≈ 79.6 is solved exactly from W_sat = r(K_h_o + K_h_l·K_l_o·ρ²)/(1 + K_h_l·ρ²) for `CALIBRATION_THETA_SATURATED` = 0.05 (`effector_selectivity_for_theta_saturated`), then K_l_e = Sochor's 7.6e5 × `MWC_EFFECTOR_SCALE` ≈ 0.393 (≈ 3.0e5) and K_h_e = K_l_e/ρ (≈ 3.7e3), the scale chosen for an 80 µM midpoint; C4 uses the same K_h_l, K_h_o and calibrates ln K_E (≈ 12.41); both give a default wt midpoint (halfway between θ(0) and `mwc.theta_saturated`) of 80 µM with depletion. Default wt curves: C12 θ 0.997 → 0.05, EC10/50/90 27/80/339 µM, Hill-like n 1.73; C4 0.997 → 0, 29/80/229 µM, n 2.14. Prior-predictive fraction of wt draws (PK wt priors, U0, 200k) with Δθ(0→1 mM) ≥ 0.5 / ≥ 0.2 / Δθ(0→10 mM) ≥ 0.5: C12 0.61/0.71/0.65 (0.42/0.52/0.43 with Sochor's ρ), C4 0.72/0.81/0.87. **units**: all twelve read `titrant_conc` in mM and multiply it by `theta_conc_unit_scale` (1e-3) so the equations run in M. Old priors CSVs/checkpoints are not supported (the legacy `theta_conc_unit_scale` reader is gone; an old priors CSV with a `theta_op_total_M` or other stale row now fails to load, see **CSV (de)serialization**). Tests: `theta/thermo/test_mwc.py` (brute-force species sum, Newton vs brentq, exact nesting with depletion, 80 µM midpoints, C12 θ_sat 0.05, projection rows), `test_conc_units.py`) - `theta_rescale/`: `passthrough`, `logit` - `dk_geno/`: `fixed`, `hierarchical_geno`, `pinned` - - `noise/` (theta observation noise; registry keys `theta_growth_noise`: `zero`/`beta`/`logit_normal`, `theta_binding_noise`: `zero`/`beta`) + - `noise/` (theta observation noise; registry keys `theta_growth_noise`: `zero`/`beta`/`logit_normal`, `theta_binding_noise`: `zero`/`beta`/`genotype_offset`; `genotype_offset`: one additive theta offset per binding genotype, constant over titrant, `tau * z_g`, tau learned (HalfNormal(0.2)) or held by `offset_scale_fixed`; plate `{name}_binding_plate` over the binding genotypes only, indexed by `BindingData.binding_ordinal` (arange in joint models, the batch indices binding-only); prediction remaps a `*_binding_plate` site by binding genotype label; joint-fit plan step 4b) - `growth_noise/`: `zero`, `normal_kt` - - `ln_cfu0/`: `hierarchical`, `hierarchical_factored` - - `sample_offset/`: `zero`, `normal` + - `ln_cfu0/`: `hierarchical` (one starting abundance per replicate x pre-condition x genotype; use this), `hierarchical_factored` (one genotype baseline per replicate shared by every pre-condition, plus a per-tube offset; tube offsets non-centered, `tube_offset = tube_scale * tube_offset_z`, because the centered form had an unbounded density as `tube_scale` went to 0). The orchestrator refuses `hierarchical_factored` when a replicate's pre-conditions come from different libraries (`_check_factored_ln_cfu0`): kanR and pheS are transformed and grown up separately (user, 2026-09-28), so their starting abundances differ per genotype. On simulations the fit pushed that difference into a confident per-genotype theta error: 95% coverage 0.09 above 1000 reads, against 0.80 with `hierarchical` (`planning/studies/svi-overconfidence/`). Every study grid used the factored model until then. + - `sample_offset/`: `zero`, `normal` (per-tube growth-rate offset, scaled by elapsed time), `level` (per-tube ln_cfu offset, constant prior; `sigma_fixed` > 0 holds its SD instead of learning it: learned, it grew to a free level per tube on the first real-data fit and absorbed the population's growth). Both extract one row per measured tube, labeled from the growth tensor's `{dim}_idx` columns in the components' reshape order (`_tubes.py`; `test_tubes.py` checks the labels against the model's own tensor) + +- **`generative/observe/`** — *not* under `components/`, and not swappable via YAML. Holds the four observation-likelihood layers (`binding`, `growth`, `presplit`, `base_growth`), registered under flat `model_registry` keys `observe_binding`/`observe_growth`/`observe_presplit`/`observe_base_growth`. `ModelOrchestrator` wires in `observe_binding` whenever binding data were supplied and (unless `binding_only`) the growth observer chosen by `growth_likelihood` (`lncfu` → `observe_growth`, Student-t on `ln_cfu`; `counts` → `observe_growth_counts`, see **Count likelihood** below), plus `observe_presplit`/`observe_base_growth` only when the corresponding data (`presplit_df`/`base_growth_df`) was supplied — these are parallel, independently-gated observers, not alternative choices for one axis. **Growth-only models** (`binding_df=None`, `tfs-configure-model` without `--binding_df`) have `data.binding = None`, `priors.binding = None`, no `theta_binding_noise` component and no binding sites in the model (`jax_model` gates them on `data.binding is not None`, static structure); the orchestrator refuses a `binding_weight` or a non-`zero` `theta_binding_noise` without binding, and `tfs-prefit-calibration` refuses a growth-only config. Absent prior groups are left out of the priors CSV (`configuration_io._extract_scalars` / `_update_dataclass` skip `None`), so they reload as `None`, not NaN. + +### Count likelihood (`growth_likelihood: counts`) + +Roadmap step 7. **The `tfs-configure-model` default** (with `sample_offset: level`), since the count-likelihood study (2026-09-27: theta RMSE 35-60% lower than `lncfu` at similar coverage). `ModelOrchestrator`'s own defaults stay `lncfu`/`zero` on purpose: configs are read back through it, and one written before step 7 has no `growth_likelihood` key, so changing the orchestrator default would silently turn old `lncfu` configs into count models (tests with synthetic `ln_cfu` tables also rely on it). `observe/growth_counts.py` observes reads directly: `reads ~ NegBin(mu, var = mu (1 + phi) + mu^2 inv_r)` with `log mu = ln_sample_reads + ln_cfu_pred - sample_ln_cfu` (predicted frequency times the tube's reads), learned `growth_phi`/`growth_inv_r` (LogNormal priors, medians 5 and 0.01), no pseudocount. The log-pmf is `CountNegativeBinomial`, Loader's algorithm (R's `dnbinom_mu`: `bd0` deviance + `stirlerr`): numpyro's own negative binomial loses several nats per observation to float32 cancellation at 1e4-1e6 reads; keep any rewrite of it tested against scipy at high depth (`test_observe_growth_counts.py`), and guard `jnp.where` branches so the unused one cannot overflow (NaN gradients). The concentration has a floor (`c + e^-30`, `_LOG_C_FLOOR`): without it `mu / phi` underflows for genotypes predicted near extinction (all-zero genotypes drift there) and the gradient/Hessian go NaN; `test_finite_value_gradient_and_hessian` covers log mu down to -200. Data: `GrowthData.counts`/`ln_sample_reads`/`sample_ln_cfu` (same layout as `ln_cfu`, None for `lncfu`; sliced by `tensors/batch.py`) and the static `GrowthData.growth_likelihood`, built by `model_orchestrator._add_count_columns` (tube reads from `sample_reads`, else `adjusted_counts / frequency`, about 1% high from pseudocounts, the same for every genotype in a tube; tube totals from `sample_ln_cfu` or `sample_cfu`). The orchestrator refuses `growth_noise` other than `zero` with counts. Pair it with `sample_offset: level` (one constant-prior ln_cfu offset per tube), since an error in a tube's supplied total otherwise moves every genotype in it. Presplit is still observed as `ln_cfu`. Validation grid: `planning/studies/count-likelihood/`. + +### OD600 calibration (`tfs-calibrate-od600`) + +Roadmap step 3. `process_raw/od600.py` holds the method (C10: the lab's data never enter the repo): `reading_noise`/`detection_threshold` from repeated readings of a dilution series (largest relative SD; midway between the two most dilute means), `plate_counts_to_cfu` (`colonies * dilution / plated_volume_mL`, Poisson counting + one pipetting error per step), `fit_polynomial` (weighted least squares, covariance scaled by the reduced chi-square as in `curve_fit`), `calibrate`, `write_calibration`/`read_calibration`/`check_calibration` (`kind: od600_to_cfu_per_mL`, coefficients, full covariance, `reading_rel_sd`, `detection_threshold`, `calibrated_od600_range`; the lab notebook's `A_CFU`... format is refused with a pointer), `od600_to_cfu_per_mL`, `cfu_per_mL_error_components` (curve SD `sqrt(J C J^T)`, shared by every tube (C11), kept apart from the per-reading SD) and `cfu_per_mL_to_od600` (bisection). It reproduces the lab notebook's coefficients to 2e-9 given the notebook's CFU/mL, but the notebook never divided by the plated volume (0.1 mL), so its constants are 10x low; the tool divides. + +`tube_totals_from_od600(od600_df, cal, tube_volume_mL)` turns one reading per tube into `sample_cfu` (CFU/mL times the culture volume: totals are cells per tube), `sample_cfu_std`, `sample_cfu_curve_std`, `sample_cfu_reading_std` and `od600_in_calibrated_range`; a reading below the detection threshold is an error (kept deliberately: dropping a real tube is the lab's call, and the simulator's raw and direct outputs both leave such tubes out before this step). `tfs-process-counts` calls it (`add_tube_totals`) when given `--od600_file` (or an `od600` column) with `--od600_calibration_file` and `--tube_volume_mL`, and refuses OD600 plus supplied totals. This is roadmap/pipeline step 1 (P3: the OD600 table stays a first-class input so the population model can replace the totals step). Synthetic inputs in `examples/process_raw/` (`make_example_data.py`). + +### Relative-X fit (`theta: hill_relative`) -- **`generative/observe/`** — *not* under `components/`, and not swappable via YAML. Holds the four observation-likelihood layers (`binding`, `growth`, `presplit`, `base_growth`), registered under flat `model_registry` keys `observe_binding`/`observe_growth`/`observe_presplit`/`observe_base_growth`. `ModelOrchestrator` wires in `observe_binding` and (unless `binding_only`) `observe_growth` unconditionally, plus `observe_presplit`/`observe_base_growth` only when the corresponding data (`presplit_df`/`base_growth_df`) was supplied — these are parallel, independently-gated observers, not alternative choices for one axis. +Roadmap step 5. Growth alone fixes an occupancy-like variable only up to an affine map (`k`/`m` absorb it), so `theta/hill_relative.py` fits a wt-relative growth variable X: hill_geno's curve with real-valued baselines (`X_low`, `X_delta`), wt gauged to `X = 1` at `c_lo` and `X = 0` at `c_hi` (`gauge_baselines`, from wt's own K and n; `data.growth.theta_gauge_log_conc`). `theta_gauge_conc` is an orchestrator setting (`tfs-configure-model --theta_gauge_conc`; default the measured min/max, resolved and written to the config; refused for other theta components). `model_orchestrator._check_relative_theta` refuses binding data, activity other than `fixed`, `theta_rescale` other than `passthrough`, non-`linear` condition growth, theta noise other than `zero` and, with `mixture`, any `congression_theta_rule` but `max` (the only rule invariant to an increasing map; `mixture`'s `max` branch does not clip). The module's `THETA_SCALE = "X"` is how downstream code tells: `tfs-predict-theta` adds `theta_scale = X`, `extract_theta_epistasis`/`tfs-extract-epistasis` allow only `scale='add'` (no `in_regime`), `tfs-summarize-fit` maps simulated truth to the gauge (`hill_relative.x_scale_truth`/`x_scale_growth_truth`, activity from `*_sim_parameters.csv`), `calibration_grid` labels those rows `theta_regime = X`. `prediction.copy_orchestrator` always keeps wt in a genotype subset (its K/n set the gauge) and `predict` drops it again. `tfs-prefit-calibration` refuses it (growth-only). **For a screen the route is settled: the two-stage fit** (`tfs-fit-two-stage`, documented in `docs/source/screen-analysis.rst`; `planning/two-stage-fit.md`): a single full-library MAP compresses X about 1.45-fold through the doubles' pull on m, so k and m come from a stage 1 of wt, spikes and singles. The rest of this paragraph is the history of the single-fit comparison, kept for the reasons behind it. Growth alone constrains mostly `m·(X_g - X_wt)`, so the posterior is a curved ridge along `m·X` that only wt's absolute growth change (via the tube totals) pins. Runs 2 and 3 of `planning/studies/relative-fit/` (2026-09-28) found the mean-field component guide sliding along it (smaller |m|, stretched X) while the low-rank guide (`--guide_type auto_low_rank_multivariate_normal`) recovered X, and MAP + Laplace blowing up in 5 of 6 runs. Both comparisons ran with bugs fixed on 2026-09-29: every component-guide hyperparameter scale frozen at its 1e-4 bound (2026-09-27 16:48 to 2026-09-29; the low-rank autoguide was unaffected) and the flat 1e-3 Laplace eigenvalue floor (the blowups); the learned log(hill_n) spread was also free then. The svi-overconfidence study's relative arm, in the same window on another library, found the two guides about equal (X 95% coverage 0.68 vs 0.74, RMSE 0.071 vs 0.072). Every guide collapses k and m. The post-fix rerun (`planning/studies/relative-fit/grid_v4.yaml`, run 4, 2026-10-01, 30 runs) confirmed the guide verdict: the component guide is biased even with the fixes (X RMSE 0.29-0.30, 95% coverage 0.06 Poisson / 0.38 realistic), and the low-rank guide recovers X (RMSE 0.07 / 0.12, coverage 0.81 / 0.76); rank 20 matches numpyro's default rank. MAP + floored Laplace now works and covers X and k/m best, but with the widest intervals and the noisiest point. Two-stage is the best calibrated under Poisson noise (0.91) and no better than low-rank under realistic noise (0.78). Every guide collapses k and m. So: the low-rank guide, rank 20, expecting X intervals to undercover somewhat and k/m intervals to be too narrow. On a full library the low-rank guide slid along the ridge (dev data, 3.3e5 nats worse than the MAP), and the MAP plus the arrowhead Laplace (below) covered best on run 4 (X 0.97 / 0.94, k/m about the full Laplace's), so that is the full-library route. Validation grid: `planning/studies/relative-fit/`. + +### Structural inputs (`thermo_data`, `thermo_options`) + +`theta/thermo/io.py` reads `thermo_data` for the PddG (ΔΔG CSV) and PnnC (LigandMPNN HDF5) models. `load_thermo_data(path, mut_labels, pair_labels, theta_name, thermo_options)` is the one entry point (orchestrator growth and binding paths, `sim_data_class.build_sim_data`): a `.PddG` component reads the CSV keeping only its `STRUCTURE_NAMES` (via `required_struct_names`, from the registry) in that order, a `.PnnC` component reads the HDF5 (structures in its `structure_names` order; the C12 PnnC permutes them into `STRUCTURE_NAMES`, the C4 PnnC requires that order), and with no or another component the extension decides (`.csv` keeps every column in file order). Whatever the file order, everything per structure downstream (`theta_ddG`, `theta_ddG_prior_mean`, `theta_ddG_sigma`, the `ddG_*` truth and extracted tables) is in `STRUCTURE_NAMES` order; see **Per-structure ΔΔG**. `thermo_options` (`normalize_thermo_options`; `components.thermo_options` in a fit config, `tfs-configure-model --thermo_ddG_units/--thermo_temperature_K/--thermo_column_map/--thermo_residue_offset/--thermo_max_missing_fraction`; a top-level `thermo_options` block in a simulate YAML, in `SIMULATE_KNOWN_KEYS`; the simulate-only `ddG_truth_sd` is a separate top-level key, see **Per-structure ΔΔG**): `ddG_units` `kT` (default, values used as-is) / `kcal/mol` / `kJ/mol`, divided by RT once at load at `temperature_K` (298.15; 1 kcal/mol = 1.6878 kT); `column_map` `{model_name: csv_column}` or `"HD=HO,..."` (`mut` mappable too); `residue_offset` added to the file's residue numbers (CSV labels and HDF5 `residue_nums`); `max_missing_fraction` (0.1). None/absent keeps the pre-2026-10-10 reading (`test_default_ddG_units_leave_the_draw_unchanged`: default and `kT` give the same draw). Sign convention: ΔΔG_X adds to state X's free energy (positive destabilises X), Δln K_{A→B} = ΔΔG_A − ΔΔG_B, ΔΔG_U = 0. CSV: mutation column `mut`/`Mutation`/`mutation`, `Unnamed: *` index columns dropped, extra columns ignored with a printed note, missing required columns listed in the error, non-single-mutation labels skipped; prints `ΔΔG for n/M library mutations`; raises on zero coverage, on a file wt residue that disagrees with the library's at the same (offset) position, on two wt residues at one file position, on duplicate labels; zero-fills absent/blank mutations and warns (list truncated at 10) above `max_missing_fraction`. The wt/zero-coverage errors search offsets ±50 and suggest the `residue_offset` that lines the wt residues up. HDF5: any library residue absent from a structure is an error (no neutral feature value), with coverage counts; an optional per-group `wt_aa` dataset (now written by `scripts/generate_struct_ensemble.py` from the PDB) is checked like the CSV's wt. Until 2026-10-10 the loader had none of this: kcal/mol went straight into ln K, absent mutations (e.g. a 0-based file against a 1-based library) were silently 0, the PddG `_get_struct_perm` demanded exact column-set equality and `mut` exactly. Now the four PddG `_get_struct_perm` accept extra structures and use `len(STRUCTURE_NAMES)` instead of `data.num_struct`; PnnC still requires its exact names. The one real-ish file, `notebooks/other-data/ddG_ensemble_clean_with_effector.csv` (main checkout, untracked): `Unnamed: 0`, `Mutation`, `unfolded` (all 0) and 12 states, 0-based numbering (V0 = LacI residue 1; a near-copy of chain A sits at 1000-1338, within 0.12, and goes unused), units not recorded; it serves the O2_C12 PddG models with `residue_offset: 1` and the O2_C4 ones only with `column_map: {HD: HO}` as well. Its H/HO, L/LO, HE2/HOE2, LE2/LOE2 columns are identical, so it carries no operator-binding information (every d_ln_K_h_o/l_o prior mean is 0). ### Adding a New Model Component @@ -141,9 +169,46 @@ The hierarchical Bayesian inference engine. Key files: 3. Register it in `tfmodel/generative/registry.py` under the appropriate category key 4. Add a test in `tests/tfscreen/tfmodel/components//` 5. For a new `condition_growth` component: make its per-condition prior loc/scale fields accept scalar-or-array (broadcast-then-index by the condition plate) and implement `get_scale_bounds()` so the pre-fit can pin its per-condition baselines — see **Per-condition growth priors** above. +6. Per-genotype latents must be **mini-batch safe**: sample them in a library-sized plate (`pyro.plate(f"{name}_genotype_plate", data.num_genotype)`), then slice with `[..., data.batch_idx]` whenever the value is library-sized (a smaller value is an already-sliced substitution from the posterior forward pass). Never size a latent's plate with `data.batch_size`, and do not wrap latent priors/guides in `scale_vector` (only the likelihood is scaled). A batch-sized latent silently aliases genotypes onto batch positions under every numpyro autoguide, including `AutoDelta` (MAP, pre-MAP, `tfs-prefit-calibration`) — even at full batch, because the full-batch index is binding-first and reshuffled each step. Slice **unconditionally**: never `if data.batch_size < data.num_genotype`, and never index a library-ordered array by the batch-relative `geno_theta_idx` alone (translate through `data.batch_idx[data.geno_theta_idx]`, the `hill_geno` pattern). At full batch the index is a permutation, not the identity, so skipping the slice pairs batch positions with the wrong genotypes while every shape stays right. `inference/batch_safety.py` checks both failure modes: `find_batch_dependent_latents` traces at two batch sizes (shape; `RunInference.run_optimization` runs it before fitting with any autoguide and refuses an unsafe model), and `find_batch_order_mismatches` traces two orderings of the full batch with the latents held fixed and checks that `growth_pred`/`theta_growth_pred`/`binding_pred`/`theta_binding_pred` permute with the index (order). `tests/tfscreen/tfmodel/inference/test_batch_safety.py` enforces both registry-wide. Known exception: `noise/beta`'s `{name}_dist` (its latent is the noisy θ itself). + +7. A new `dk_geno` or `activity` component must accept `return_population=False` in `define_model` and `guide` and, when True, return `(tensor, population)`: the library-ordered per-genotype values from the same draw, or `None` if the latents arrived batch-sized. Build both with `generative/components/_population.py::per_genotype` (default path = the old slice-then-compute, so no extra cost when not requested). The congression mixture looks co-resident genotypes up in these by library index; `tests/tfscreen/tfmodel/generative/components/test_component_population.py` checks every registered variant. + +8. Horseshoe priors use `components/_horseshoe.py`: its `HalfCauchy` (log density via `hypot`, finite far in the tail) and `regularized_scale(lam, tau, c2)` (the slab-regularized `sqrt(c2 lam^2 / (c2 + tau^2 lam^2))` without squaring `lam`). Never write `lam ** 2` there: a mean-field guide can widen a local scale until a draw near 1e20 squares to inf, and the whole fit goes NaN (count-likelihood grid v2, run 0007, hill_mut epistasis on low-read doubles). + +9. Guide sites should follow the `{site}_loc(s)` / `{site}_scale(s)` naming convention (a Normal or LogNormal guide whose location/scale params are named after the site). The MAP warm-up hands its solution to the component guide through it (`inference/initialization.py`); `tests/tfscreen/tfmodel/inference/test_initialization.py::test_component_guides_follow_loc_scale_convention` fails for a new variant that does not, unless it is added to `_KNOWN_UNMAPPED` with a reason. (This applies to `components/` categories. `generative/observe/` is not a `/` registry entry — see above.) +### Convergence and step size + +`RunInference.run_optimization` is the one optimization loop (SVI, MAP, the SVI pre-MAP warm-up, `tfs-prefit-calibration`). It runs in tumbling **windows** of `convergence_window_steps` optimizer steps (default 2000; raised to ≥ `MIN_WINDOW_EPOCHS` = 10 epochs so mini-batched genotypes are each visited several times; split into `PARAM_BLOCKS` = 8 blocks). `inference/convergence.py` holds the decisions as host-side numpy, testable on synthetic traces (`tests/tfscreen/tfmodel/inference/test_convergence.py`, including a recorded real-data and a simulated trace in `loss_traces/`). + +- **Loss test** (`loss_trend`): OLS line through the medians of 20 blocks of the window's per-step losses → drop per window ± SE. *Improving* iff `drop > z·SE` and `drop > loss_rtol·|loss|`. Judged against the loss's own noise: never against the start-up transient (the old rule's `|Δ|/(loss_start − loss_best)` stopped runs ~500× above their final loss and never stopped after a resume) nor against `|loss|` (the rule before that, which never fired on a noisy ELBO). A rising or oscillating loss is *not* improving. +- **Exact loss (MAP)**: `run_optimization(exact_loss=None)` (auto: on for `AutoDelta`, so MAP, every staged-MAP stage, the SVI pre-MAP and the prefit) computes `RunInference.full_batch_loss` at each window end: `-log_density` over genotype chunks of the full batch (chunk = batch size) with the mini-batch scale removed, the chunk-repeated part (library-sized latents' priors, unsliced binding data) measured as `f(a) + f(b) - f(a, b)` on one-genotype batches and counted once (`test_full_batch_loss.py` checks chunk invariance with binding data). `ConvergenceMonitor.end_window(..., exact_loss=)` then judges the window by a line through the last `patience + 1` exact losses (drop > `z`·SE and > `loss_rtol`·|loss|; SE = the optimizer's jitter), skips the skew check and the separate pooled check, and records `loss_exact` in the convergence CSV. Why: on the dev data (218k genotypes, batch 4096) the mini-batch trend SE was ~5e4 nats per window, so a 1e5-per-window descent read as a stall and every MAP was cut to 1e-6 by ~40k steps while still descending (`planning/studies/staged-map/`). SVI keeps the mini-batch test. +- **Loss skew** (`loss_skew`, reported by `loss_trend`): `(mean − median) / (1.4826·MAD)` of the window's losses about the trend line. Medians hide rare huge penalties that dominate the mean (the objective); benign ELBO noise, skewed or heavy-tailed, stays < ~1 (all recorded clean traces ≤ 0.4), rare penalties give 10–40. At the floor a window with skew > `MAX_LOSS_SKEW` (3) is not a plateau; skew never forces a cut (a smaller step size does not remove penalties). Blind once penalized steps are the majority (median and MAD follow them) — then the loss level shows it instead. +- **Parameter test** (`param_drift`): each tracked param's block means (accumulated on the device inside the `lax.scan` carry — no per-step host transfer) → per-element trend over the window, in **normalizer** units (`RunInference._param_normalizer_specs`): paired guide scale for a location (`_loc`/`_locs`/`_auto_loc` ↔ `_scale`/`_scales`/`_auto_scale`; the posterior SD), derived SD for AutoContinuous `auto_loc`, prior SD for an AutoDelta location on a real site, and 1 for positive/bounded params (log/logit units, so relative). Excess = `|drift| − z·SE − floor`, where `floor = MIN_STEP_COHERENCE (0.01) · step_size · window` is the optimizer's resolution (Adam moves ~step size per step whatever the gradient; without the floor, jitter of a location whose guide scale collapsed below the step size reads as hundreds of SDs of drift). The posterior SD is floored at `PRIOR_SD_FLOOR` (0.01) × the site's prior SD in the location's unconstrained units (`initialization.site_unconstrained_prior_sds`: exact SD for real sites, robust MC `IQR/1.349` of prior draws otherwise; `RunInference._posterior_sd_floors` maps component-guide locations via `{site}_loc(s)`, AutoNormal via `{site}_auto_loc`, and AutoContinuous `auto_loc` via numpyro's `_ravel_dict` latent order): the mean-field component guide collapses hierarchical scales' guide SDs to ~4e-4 (hill_mut `theta_sigma_d_*_scale`, `theta_epi_tau_scale`), and in those units a negligible creep never settles. Summarized per array (max if ≤ 100 elements, else 99th percentile); *moving* iff any summary > `param_tolerance` (0.05). `scale_tril`/`cov_factor` are not tracked. +- **Decisions** (`ConvergenceMonitor.end_window`): above the floor step size, `patience` consecutive windows without loss improvement → cut (`step_size_cut`, default 0.1), unless the stalls pooled are a descent: before a cut or a stop, `pooled_loss_trend` fits one line through the block medians of all `patience` stalled windows (same `z`/`loss_rtol`), and a significant drop resets the stall count (`pooled_loss_t` in the convergence CSV; the stalled windows' medians are kept in checkpoints). A single window misses a slow, noisy descent: on the count-likelihood v2 grid, 20 of 23 first cuts came during a descent that pooled to t = 4-14, one (run 0012) into a wrong optimum (`loss_traces/sim_svi_counts_noisy_descent_run0012.npy`). **Parameters and skew never block a cut**: some directions never settle (a horseshoe local scale's MAP drifts to 0 forever at ~no loss change) and holding the step size high for them only wastes time. At `final_step_size`, `patience` consecutive windows with no loss improvement, no parameter movement and no loss skew → converged; a still-moving parameter or a skewed loss keeps the run going and is named in the log, so a slide, degenerate direction or hidden penalty is reported, never called converged. `max_num_epochs` is only a cap. A hierarchical MAP generally has no finite optimum (hyper-scale funnel), so MAP/pre-MAP runs often end at the cap — the honest outcome. +- **Optimizer** (`run_inference.adam_optimizer`): plain numpyro `Adam` by default; `adam_clip_norm` (every CLI and `setup_svi`) opts into `ClippedAdam`. **Do not re-enable clipping by default.** `ClippedAdam` clips each gradient *element* to ±`clip_norm`; these losses have gradients of 1e3–1e6 per element, so every element is clipped every step and Adam follows each draw's gradient *sign* — its fixed point is where the per-draw signs balance, not where the expected gradient is zero. With a heavy-tailed ELBO (step-like binding curve M42C/K84N, `hill_n` 24.6, measured with SD 0.001; a guide draw crossing it costs ~2.4e5 nats) that held 5–90% of draws in violation on every congression-calibration run (2026-09-25; the "blow-ups" were block medians flipping once the rate passed 50%, and step-size cuts cannot move a sign-balance point). Unclipped, the same fits drop to ~0 violations and converge (`planning/studies/congression-calibration/diagnosis/`). The two optimizers share their state, so old checkpoints resume unclipped. +- **Step-size cuts** swap `svi.optim` for a new optimizer of the same kind (clipped or not, read off the current `svi.optim`) at the smaller constant step (the optimizer state is step-size independent) and re-trace the scan (a fresh closure, so no stale compiled step size). `setup_svi` records `_step_size`/`_adam_clip_norm` (the latter written to checkpoints); a step-size *schedule* is still accepted but refuses cuts. The old `optax.exponential_decay` over `max_num_epochs` is gone (it collapsed the pre-MAP's step size within its 1000 epochs and tied resumes to `max_num_epochs`). +- **Runaway loss** (`RUNAWAY_LOSS_FACTOR` = 1e3): a window whose loss falls below -1000 times the magnitude of the run's first block median ends the run as `diverged` (not converged; `monitor.diverged`, kept in checkpoints with `reference_loss`). It means the objective is unbounded: a hierarchical density singularity (the old centered `hierarchical_factored` tube offset), or float32 error in a dense guide's own log density (`auto_multivariate_normal` on the small SVI-overconfidence library, where `log q` of its own draws reached -4e22 through an ill-conditioned `scale_tril`). +- **Records**: `{out_prefix}_convergence.csv` (one row per window: loss (median), drop ± SE, t, `loss_mean`/`loss_skew`/`loss_skewed`, worst param and its excess/drift, step size, plateau count, decision; a resume rewrites an older file under the current columns, `_migrate_convergence_columns`); `{out_prefix}_losses.txt` is `epoch,loss,step,step_size` (loss = block median). Checkpoints carry `step_size` and `convergence` (the monitor's `state_dict`); a resume continues at that step size and plateau count (a checkpoint without them resumes at `adam_step_size`). A run started with `svi_state=None` resets `_current_step` to 0 (the pre-MAP no longer shifts SVI's epoch numbering). + +**Starting points** (`inference/initialization.py`). Configured guesses are keyed mostly by *site* name and partly by component-guide param name; AutoDelta and the component guide both ignored site-keyed guesses (AutoDelta started at prior medians; the MAP → component-guide hand-off matched no names, so the pre-MAP was discarded). Now `RunInference.site_values` turns any mix of guesses / `{site}_auto_loc` MAP params / guide locations into constrained site values; MAP (`_run_map`, prefit, pre-MAP) starts there through `setup_svi(init_values=...)` (fallback `init_to_median` for AutoDelta); `RunInference.component_guide_start` maps site values onto the component guide via the `{site}_loc(s)`/`{site}_scale(s)` convention (LogNormal locations take the log; each match verified against the traced distribution) and caps every mapped scale at `guide_init_scale` (`fit_model_cli.DEFAULT_GUIDE_INIT_SCALE`, 1e-4; also the autoguides' `init_scale` for a fresh fit). A scale param with a lower-bounded constraint (the hyperparameter scales' `greater_than(1e-4)`) starts at least `guide_init_scale` above the bound: starting on it is `-inf` unconstrained, and the param never moves (every hyperparameter scale stayed frozen at 1e-4 from 2026-09-27 to 2026-09-29). The scale is one number in every site's unconstrained units, so it must be small: at the old 0.1 (numpyro's autoguide default) an SD of 0.1 on a growth rate per minute is 17 ln units over a selection, SVI started ~1000x above the pre-MAP's loss and re-descended into other optima (mirror mode, halved m, the library shifted against wt; relative-fit grid, 2026-09-27). SVI widens the scales itself (about e-fold per 1000 steps at step size 1e-3). `init_param_jitter` defaults to 0 for the same reason (it is multiplicative, so 0.1 moves an ln_cfu0 location near 15 by ~1.5 ln units). `test_default_guide_start_stays_near_its_point` checks the start on a count model; a fit's first logged SVI loss should sit near its pre-MAP loss. `tests/tfscreen/tfmodel/inference/test_initialization.py` checks the convention registry-wide; exceptions: horseshoe_geno (`activity_*_offset_*` param names) and prior-guided noise latents. A new component guide should follow the convention. `read_configuration` keeps only guess rows whose names are in the orchestrator's `init_params`; it silently drops any others, such as a bare `condition_growth_k` or `sample_offset_offset` row. To start at a point the guesses cannot name, use `tfs-fit-model --init_from `: an earlier MAP's `{site}_auto_loc` arrays, which `site_values` ranks first, plus its model-side params (every other array; see **Model-side params**). The dev-data level-offset MAP is the case: Adam moves every parameter about one step size per step, so in the first window it carried the tube offsets and k/m (k prior SD 0.002 per minute) by whole units into a ±2.8 offset mode. The step-size cuts then froze it there. Starting from the no-offset MAP plus small per-tube offsets was 1.2e5 nats better (`planning/dev-data/real_fit`). + +**Staged MAP** (`inference/staged_map.py`, pipeline plan step 3). `tfs-fit-model` builds that start itself: a fresh MAP (no `--checkpoint_file`, no `--init_from`) of a model with `sample_offset: level` runs (1) the MAP with `sample_offset_offset` held at 0 (and a learned `sample_offset_sigma` held at `sigma_prior_scale`: with every offset at 0 its MAP is 0 and the density unbounded), files `{out_prefix}_stage1_*`; (2) the offsets alone, every other site held at stage 1's MAP (`{out_prefix}_stage2_*`; the tubes do not couple given the rest, so this is `small_offsets.py`); (3) the joint MAP from stage 1 + stage 2's offsets at `--staged_step_size` 1e-4, writing `{out_prefix}_*`. A stage is an ordinary `_run_map` on `HeldModel(orchestrator, held)`, whose `jax_model` is `numpyro.handlers.condition` on the held constrained values (as jax arrays: the model indexes per-genotype values with traced batch indices); the AutoDelta then has no parameters for them. Fit a `HeldModel` with autoguides only (its `jax_model_guide` is the wrapped model's, read by `site_values` for names). A stage whose `_params.npz` exists is reused on rerun; one with only a checkpoint resumes. `--stage_offsets auto` (default) stages only that case; SVI's pre-MAP is not staged. Validation on the dev data: `planning/studies/staged-map/`. **The offset mode is the objective's better optimum, not only a trap** (run 2, 2026-10-05, every MAP converged on the exact loss): the reference fit run to convergence moved into it and beat the staged MAP by 7.9e4 nats, with offsets that follow IPTG in the selection conditions (kanR selection +2 to -4 from 0 to 1 mM, pheS the reverse), constant over time, and unphysical k/m. The staged MAP keeps the physical basin only by stopping in a local optimum. Working hypothesis (IPTG is added at `-t_pre`, so markers are expressed before selection): a selection-onset transient that `instant` cannot express. Filed as `planning/offset-mode-growth-transition.md`, taken up after pipeline steps 6 and 7. + +**MAP values: constrained or not.** AutoDelta's `svi.get_params(state)` (what `run_optimization` returns and `write_params` saves) is *constrained*; `svi.optim.get_params(state.optim_state)` (what `tfs-sample-posterior`, `tfs-extract-params` and `checkpoint_io` read from checkpoints) is *unconstrained*. `compute_hessian_sigmas` takes the former, `get_laplace_posteriors`/`get_map_posteriors`/`_map_params_to_constrained` the latter. Mixing them up silently evaluates positive sites at `exp(value)` (the pre-fit did until 2026-09-27). + +**Model-side params** (`inference/model_params.py`). A few components register learned point estimates with `pyro.param` inside the *model*, not only in a guide: the thermo PnnC structure-MLP weights (`theta/thermo/nn.py`, `theta_nn_{struct}_{W1,b1,W2,b2}`). Until 2026-10-11 the PddG/PnnC per-structure ddG scale was one too (`theta_ddG_sigma_s`); it is now the sample site `theta_ddG_sigma` (see **Per-structure ΔΔG**), so it is an ordinary MAP site, in the Laplace Hessian and in the guides. They are empirical-Bayes estimates that live in the param store next to the guide's params, under their bare names (constrained in `svi.get_params` and `_params.npz`, unconstrained in a checkpoint's optimizer state). Anything that runs the model after a fit must hand them over, or every one falls back to its init value (NN = 0). Until 2026-10-10 none did: a PnnC posterior's prior mean was the untrained NN's 0 instead of the MAP's (off by up to 0.99 on a C4_U0 PnnC fit). Now `RunInference.model_params(params, unconstrained=...)` picks them out of any param dict (by tracing the model's param sites), and the post-fit entry points (`get_map_posteriors`, `get_laplace_posteriors`, `get_posteriors`, `compute_hessian_sigmas`) set `RunInference.fixed_model_params` and run `hold_model_params(model, params)` (substitute plus `block`, so no outer `substitute` or param store can replace them). The Laplace holds them fixed at the MAP: they are not in the Hessian (`num_params` counts `_auto_loc` only). Posterior files carry them in the `model_params` HDF5 group (absent for models without such params; iterate samples with `model_params.sample_keys`); `analysis/prediction.predict` (`tfs-predict-growth`) reads that group, or the bare names of a raw MAP `_params.npz`. `tfs-extract-params` writes them to `{out_prefix}_model_params.csv` (`parameter`, `index`, `value`). Hand-offs carry them too: `--init_from` reads every non-`_auto_loc` array of the npz, the pre-MAP passes them to SVI (component guide and autoguides alike, as `init_params`), the staged MAP holds them at stage 1's values in stage 2 (`HeldModel(..., params=)`) and starts stage 3 from them, and the window's exact loss (`full_batch_loss`) evaluates the model at their current values. Not covered: NUTS fits the model with them at their init values (no optimizer), and its pre-MAP's values are not handed to it. + +**Subset predictions keep the fitted mutation index** (`analysis/prediction.py`). `predict(genotypes=...)` rebuilds the model on the subset (`copy_orchestrator`), and `tfs-predict-growth` always does this in genotype chunks (auto-sized batch, `--genotypes_file`). Per-mutation and per-pair sites (`*_mutation_plate`, `*_pair_plate` and the like: hill_mut, activity `hierarchical_mut`/`horseshoe_mut`, all 12 `thermo.*` models, and the structure data built from the labels) are indexed by `mut_labels`/`pair_labels`. Until 2026-10-10 the rebuilt model derived those labels from the subset alone, so the fitted arrays landed on the wrong mutations and pairs; the predict loop only remaps plates named after a growth-tensor dimension. hill_mut chunked predictions were off by up to 1.6 ln_cfu on a real run, and PddG/PnnC subsets crashed on a shape mismatch. Now `copy_orchestrator` passes the fit's labels as `ModelOrchestrator(mut_labels=, pair_labels=)` (not a setting, never written to a config), `build_mut_geno_matrix(..., mut_labels=, pair_labels=)` builds the subset's genotype columns in that fixed index space (absent mutations get zero rows; a mutation or pair outside it is a `ValueError`), and `predict` traces the subset model and refuses any posterior site whose shape does not match it, so a plate it cannot remap fails loudly instead of misindexing. `tests/tfscreen/tfmodel/analysis/test_prediction_mutation_subsets.py` checks chunked and subset predictions against the full one. + +**Block Laplace.** `get_laplace_posteriors(block_genotypes=True)` (`tfs-sample-posterior --laplace blocks`) holds the shared parameters at the MAP and gives each genotype the Laplace of its own parameters. Given the shared parameters the genotypes do not couple (checked: every cross-genotype Hessian entry was 0 on a relative-fit run, wt included, since the X gauge fixes only wt's own baselines), so `block_laplace_factors` gets every B x B block from B Hessian-vector products whose probe is 1 on one slot of every genotype, over genotype chunks, and refuses a model whose blocks leak. Each block is floored like the full Laplace. Both Laplace paths take the Hessian on `_unscaled_batch`, the full batch with the mini-batch scale removed: `scale_vector` carries `num_genotype / batch_size` even at full batch. On relative-fit run 4 the block Laplace covered X 0.93 / 0.84 (Poisson / realistic) and k/m 0. The MAP point's X is poor at low depth under realistic noise (RMSE 1.3 at <= 100 reads, against 0.22 for the low-rank guide's point; the MAP's hyperscales run away, so nothing shrinks it), but the intervals are wide enough to cover there. + +**Arrowhead Laplace.** `block_shared=True` (`--laplace arrowhead`; what `--laplace auto` picks above 20,000 MAP parameters) keeps the shared parameters' uncertainty: `_genotype_hessian_parts` adds one probe per shared element (its response on a chunk's genotype rows is their coupling; on the shared rows it is summed over chunks, less `(chunks - 1)` times the chunk-invariant part, measured as `H(a) + H(b) - H(a + b)` on two one-genotype batches since every chunk's potential carries the priors and any unsliced binding data). `arrowhead_laplace_factors` then draws the shared parameters from the Schur complement and each genotype from its conditional, the full Laplace (`test_arrowhead_laplace_matches_full_laplace`). Two departures for MAPs short of an optimum, both needed: a floored direction of a genotype block carries no coupling (a short dev-data MAP floored 167 of 308 blocks, and the kept couplings subtracted 6e10 from m's curvature of 1e10), and a negative direction of the Schur complement is held at the MAP, named in the log and recorded on `RunInference.held_shared_directions` (`tfs-sample-posterior` writes it to `{out_prefix}_held_directions.csv`) (floored at the prior, the k/dk_geno slide gave k its prior SD and X widths of 1-3 on run 4). On run 4 it covered X 0.97 / 0.94 and k/m 0.92 / 1.0 (Poisson) and 0.50 / 0.42 (realistic), about the full Laplace's. + +**Laplace floor.** `get_laplace_posteriors` floors each Hessian eigenvalue at the prior's curvature along its eigenvector (`laplace_eigenvalue_floor`; prior SDs from `site_unconstrained_prior_sds` on one trace with every latent at its MAP, `trace_model_sites(..., unconstrained=)`; `LAPLACE_MIN_EIGENVALUE` = 1e-3 where no prior SD is usable, named by site in a warning), so no direction of the Laplace is wider than the prior. The full Laplace computes the prior SDs before the Hessian. If they cannot be computed it is an error (`RuntimeError`), unless `allow_flat_floor=True` (`tfs-sample-posterior --allow_flat_laplace_floor`), which warns and floors everything at 1e-3. Until 2026-10-11 the supports came from a first trace with every site drawn from the prior, and the failure was a one-line print: on the PddG pilot fits the held `sigma_s` widened the ddG prior, the draw (ln K −38 to 61) gave NaN θ and ln_cfu, the count likelihood refused it (`CountNegativeBinomial ... invalid log_mu`), and all four runs fell back to the flat floor, turning 16-38 negative eigenvalues into directions with SD ~30 (wt ln K_op 95% 14.9-24.6 against PK's 19.4-20.4, medians moved up to 28 kT). **Structure traces** (`initialization.structure_trace`: seeded trace with numpyro argument validation off) are used wherever a trace only reads site shapes, supports or distributions: `trace_model_sites`, `model_param_constraints`, the site-transform traces in `run_inference`, and the two traces in `analysis/prediction.predict` (the same refusal stopped `tfs-summarize-fit`'s trajectory plots). Validation never runs under jit anyway; eagerly it rejects a prior draw at an observation nothing evaluates. The old flat 1e-3 floor turned a MAP's negative eigenvalues (a hierarchical MAP stopped at its epoch cap always has a few) into variances of 1000 that leaked into whatever they touched: on the two-stage grid the draws of the growth slope m spread 50x (seed 8). + ### Per-condition growth priors The `condition_growth` components (`linear`/`power`/`saturation`) carry a per-condition **additive baseline** (`k` for linear/power, `min` for saturation) that is only jointly identified with the shared per-genotype `dk_geno`: the growth likelihood `g = k_condition + dk_geno + A·m·θ` is invariant to `k += C, dk_geno −= C`. With only weak/rare anchors (wt's pinned `dk_geno=0`, `base_growth`) the whole system slides by a global constant `C`, inflating all condition baselines and `k_ref` and making genotypes with constrained `dk_geno` (notably wt) badly mis-fit their `ln_cfu`. The fix is to pin each condition baseline with a per-condition prior — a prior on the 4 baseline params acts at full strength, unlike the abundance-diluted genotype anchors. @@ -151,12 +216,12 @@ The `condition_growth` components (`linear`/`power`/`saturation`) carry a per-co Mechanism, spanning three files: - **Components** (`generative/components/growth/*.py`): `ModelPriors` loc/scale fields accept a scalar (broadcast, the default/back-compat) or a length-`num_condition_rep` array; `define_model`/`guide` broadcast-then-index by the condition plate. Each component declares `get_scale_bounds() → {suffix: {floor, ceiling, scale_field}}`, giving the pre-fit per-parameter scale floors (tight for the baseline term, looser for e.g. `power`'s log-exponent `n`). -- **CSV (de)serialization** (`configuration_io.py`): the priors CSV supports per-condition **indexed rows** (`flat_index` + `condition_rep`/`replicate` label columns) alongside scalar rows. On load, indexed rows are **name-joined** to the model's `map_condition_rep` order (`_read_priors_flat`/`_assemble_condition_array`) and **fail fast** on an unknown or missing condition. A fresh `tfs-configure-model` still writes scalar rows (2-column CSV, legacy path). +- **CSV (de)serialization** (`configuration_io.py`): the priors CSV supports per-condition **indexed rows** (`flat_index` + `condition_rep`/`replicate` label columns) alongside scalar rows. On load, indexed rows are **name-joined** to the model's `map_condition_rep` order (`_read_priors_flat`/`_assemble_condition_array`) and **fail fast** on an unknown or missing condition. A fresh `tfs-configure-model` still writes scalar rows (2-column CSV, legacy path). A priors row whose name is not a prior of the configured model is an error that lists every such row and says to regenerate the file with `tfs-configure-model` (`_update_dataclass` → `_check_unknown_keys`, all components). Known names are every dataclass field (including one left at None and the static tuples, which are kept, not overwritten) and one-level `{field}.` entries of a dict field (`pinned`). Until 2026-10-10 such rows were silently ignored, so a priors file written for an older model loaded and its rows were read with the meanings they have now. Guesses are not checked this way: `read_configuration` keeps only the guess rows the model's `init_params` names, on purpose (see **Starting points**). `test_configuration_io_helpers.py::TestUnknownPriorRows` also checks every registered component's own rows round-trip. - **Pre-fit** (`prefit_calibration_cli.py`): after the MAP calibration, `_build_csv_updates` writes each `condition_growth` site's per-condition MAP **loc** array into the priors CSV (the baseline pin), and `_build_hessian_scale_updates` writes a tight **scalar** scale (floored via `get_scale_bounds()`). `_apply_priors_updates` expands a scalar prior row into per-condition indexed rows tagged with `condition_rep`. `growth_transition` sites keep warm-start-only behavior. Net flow: `configure` (scalar) → `prefit` (per-condition `k_loc` indexed rows + tight scalar `k_scale`) → `fit` (loads per-condition priors that hold the baselines, closing the slide). This is the primary mechanism; the `base_growth_data`/`k_ref` anchor (below) is complementary but insufficient alone because `k_ref` is itself free to slide. -**Hard clamp on `m` (`tfs-prefit-calibration --pin_m`).** A soft Normal prior on the slope `m` — however tight — is only a KL penalty in SVI, and the growth likelihood over many observations can override it (empirically m walked ~5σ off even a 0.0005-scale pin). `linear`'s `ModelPriors.m_pinned` (bool, static) makes `m` a `deterministic` site clamped to its per-condition `m_loc` instead of a sampled site (guide drops the `m` variational params); `--pin_m` sets `condition_growth.m_pinned=1` in the priors CSV. `m` is safe to clamp because its calibration MAP loc is unbiased (dk_geno is uncorrelated with θ). **`k` is intentionally never clamped this way**: it carries real per-experiment tube-noise variance (`tube_noise_sigma`; mirrored by `k_scale_floor`) and sits in the additive k/dk_geno slide, so it keeps a floored soft prior — pin its loc, not its scale. +**Hard clamp on `m` (`tfs-prefit-calibration --pin_m`).** A soft Normal prior on the slope `m` — however tight — is only a KL penalty in SVI, and the growth likelihood over many observations can override it (empirically m walked ~5σ off even a 0.0005-scale pin). `linear`'s `ModelPriors.m_pinned` (bool, static) makes `m` a `deterministic` site clamped to its per-condition `m_loc` instead of a sampled site (guide drops the `m` variational params); `--pin_m` sets `condition_growth.m_pinned=1` in the priors CSV. `m` is safe to clamp because its calibration MAP loc is unbiased (dk_geno is uncorrelated with θ). **`k` is not clamped this way in production fits**: it carries real per-experiment tube-noise variance (`tube_noise_sigma`; mirrored by `k_scale_floor`) and sits in the additive k/dk_geno slide, so it keeps a floored soft prior — pin its loc, not its scale. Both flags also take one value per condition (a tuple; indexed 0/1 rows in the priors CSV, from `--growth_priors` columns `k_pinned`/`m_pinned`): the pinned conditions are clamped, the free ones are sampled as `{name}_{k,m}_free`, and `{name}_{k,m}` is a deterministic site over every condition (`configuration_io._static_flag` turns the rows into a hashable tuple; uniform flags collapse to the bool, so old configs trace unchanged). `planning/studies/m-bias-factorial/` pins kanR's k and m at the no-doubles fit while pheS's are fitted. `ModelPriors.k_pinned` exists for conditional fits at a fixed draw of (k, m): the two-stage fit (`planning/studies/svi-overconfidence/two_stage.py`) refits everything else by SVI at Laplace draws of k and m and pools the conditional posteriors, which carries the k/m uncertainty the guides collapse into each genotype's interval. ### Key Abstractions @@ -172,15 +237,19 @@ See **YAML Standards** below for the full conventions. The tfmodel config (`tfs_ `tfs-configure-model` takes `--library_config`, the library YAML that describes the screened library — **the same file handed to `tfs-process-fastq`**. It is required whenever `growth_df` is given (a binding-only model has no congression correction and no ln_cfu0 latents, so it refuses one). It replaced the old `--spiked` list, which could not express the central fact that the spiked controls are *also* encoded in the bulk sub-libraries: wt and each spiked single mutant arrive both as a monoclonal spike and as bulk library members, and they cannot be told apart by sequence. -Keys read (via `LibraryManager`): `reading_frame`, `first_amplicon_residue`, `wt_seq`, `degen_sites`, `tiles`, `tile_combos`, `spiked_seqs`, plus `library_mixture`. Everything else (a full simulate config, say) is ignored. +Keys read (via `LibraryManager`): `reading_frame`, `first_amplicon_residue`, `wt_seq`, `degen_sites`, `tiles`, `tile_combos`, `spiked_seqs`, plus `library_mixture` and the subsample keys (`library_genotypes`, `library_subsample`, `library_subsample_seed`, `library_subsample_depth`; see **Library subsample**). Everything else (a full simulate config, say) is ignored. Flow: `genetics.library_composition_table(library_config)` (`genetics/library_design.py`) → one row per genotype with `is_wt`, `in_spiked_origin`, `pool_fraction` and `bulk_fraction` → written to `{out_prefix}_library.csv`. The **snapshot CSV is what `tfs-fit-model` reads**, never the YAML; this mirrors the priors/guesses CSV pattern and makes the table the place to override the design's numbers. `ModelOrchestrator(library_file=...)` derives today's `spiked_genotypes` from `in_spiked_origin` and exposes the full table as `orchestrator.library_df`. `spiked_genotypes` is still accepted directly (and is what `tfs-prefit-calibration` uses when it clears both keys for its calibration-subset model), but the two are mutually exclusive, and a config carrying `spiked_genotypes` is the legacy path. **Two checks, two different guarantees.** The *genetics* keys are attestable: they determine the genotype names, so `tfs-configure-model` fails fast if any genotype in `growth_df`/`presplit_df`/`base_growth_df` is absent from the library (`check_genotypes_in_library`; `__unknown__` excepted). A residue-numbering shift mismatches essentially every mutant and is caught immediately. `library_mixture` is **declared-only**: no upstream artifact ever saw it, so nothing can cross-check it. It is recorded verbatim in the config's `library` block, and that is the whole defense — the values should be the best available estimate of the realized pool, not just the intended design. -A library-derived spiked genotype that has no growth data is **reported, not fatal** (spikes can drop out of a real dataset), unlike the hand-supplied `spiked_genotypes` list, which still raises. +A library-derived spiked genotype that has no growth data is an **error** at configure time (`configure_model_cli.check_spikes_in_data`; the real-data fit's processing config named three spikes, one with the wrong codon, and nothing caught it), unless `--allow_missing_spikes`, which reports it (spikes can drop out of a real dataset). The orchestrator itself still only reports it. + +`pool_fraction`/`bulk_fraction` are computed and snapshotted. `bulk_fraction` is carried into the model as `GrowthData.bulk_fraction` (library-sized float per genotype, indexed by `batch_idx`; built by `ModelOrchestrator._build_bulk_fraction`: the table's value, `__unknown__` → 1.0, any other missing genotype raises; legacy `spiked_genotypes` → 0 spiked / 1 otherwise) and is the `f_g` of the `mixture` transformation: a genotype's congressed fraction is `f_g (1 - P(M = 1))`, with `M` ~ zero-truncated Poisson(λ) plasmids per transformant (a cell's abundance is shared among its plasmids, so abundance by co-resident count n is `P(M = n + 1)`; λ is the simulator's `transformation_poisson_lambda`). Purity (`bulk_fraction`) and the `ln_cfu0` prior class (`ln_cfu0_spiked_mask`, from `in_spiked_origin`) are separate fields; the old binary `congression_mask` is gone. The orchestrator also draws each genotype's fixed co-resident plasmid sets for the mixture — `GrowthData.coresident_idx`/`coresident_n`, via `_draw_coresident_sets`, from the bulk share `pool_fraction * bulk_fraction` of genotypes with growth data (legacy: uniform over non-spiked); `congression_sets` (default `[12, 3, 1]`; `tfs-configure-model --congression_sets`, mixture only) and `congression_seed` are orchestrator settings written to the config, so draws are reproducible. `congression_theta_rule` (default `homodimer`; also `heterodimer`, `max`; `tfs-configure-model --congression_theta_rule`) is a third such setting, carried into the model as the static `GrowthData.congression_theta_rule`; the partition-function rules require `activity='fixed'` (the default activity component; the orchestrator refuses a mixture with these rules and a learned activity). `congression_dk_rule` (default `dilution`; also `softmin`, which needs `congression_dk_alpha` > 0, and `min`; `tfs-configure-model --congression_dk_rule`/`--congression_dk_alpha`) sets a congressed cell's dk_geno the same way (static `GrowthData.congression_dk_rule`/`congression_dk_alpha`). -`pool_fraction`/`bulk_fraction` are computed and snapshotted but **not yet consumed by the model** — `bulk_fraction` is the `f_g` the observable-level congression mixture will use (see `future/congression-physics-plan.md`), and the current backend still treats spiked as a binary via `congression_mask`/`ln_cfu0_spiked_mask`. +### Library subsample (`genetics/library_subsample.py`) + +Four optional library-YAML keys, read by `tfs-simulate` (and `run_simulation`) and by `library_composition_table` (so `tfs-configure-model --library_config` snapshots the same subset), never by `LibraryManager` itself or `tfs-process-fastq`: `library_genotypes` (text file, one genotype per line, relative to the YAML's own directory: `build_library`/`library_composition_table`/`library_prediction` take `base_dir`, default the YAML's dir for a path; simulate_cli passes the config file's dir; every unlisted genotype is dropped from every origin; a listed genotype outside the library is an error), `library_subsample` (`{origin: N}`, N random non-protected genotypes per origin; `spiked` refused, N >= 1), `library_subsample_seed` (required with `library_subsample`, deliberately not `seed`: configure never sees `tfs-simulate --seed`; each origin draws from `default_rng([seed, crc32(origin)])` and keeps a prefix of one permutation, so N=500 ⊂ N=1000 and adding an origin leaves the others' draws alone) and `library_subsample_depth`. wt and every spiked-origin genotype are protected (kept in every origin, not counted in N); a `double-x-y` origin's lower-order members are drawn like any other; an origin left empty is an error. `subsample_library_df` keeps whole genotypes (all their sequences) and renormalizes `weight` per origin; it returns a per-origin summary (genotypes/sequences, full and kept). **Depth** (`subsample_design`): `full_library` (default) multiplies each origin's `library_mixture` and `transform_sizes` by its kept fraction of sequences and `total_num_reads`/`pcr_template_molecules` by the kept pool share (the per-sequence rule of `scale_library_design`), leaving `cfu0` (tube totals/OD600 realistic; cells per kept genotype ×1/share); `as_written` changes nothing (the subsample is the library). simulate_cli builds the library only when `has_subsample` (otherwise `library_prediction` builds it as before, so mocks of it keep working), prints `format_subsample_summary`, and runs `library_prediction(..., library_df=)`, the replicates, read split and presplit with the updated `sim_cf`; `{out_prefix}_input-config.yaml` stays the config as given. `library_composition_table` uses the same depth rule, so kept genotypes have the simulation's composition; configure records `subsample_record` (list file by basename) as `library.subsample`. Keys are in `SIMULATE_KNOWN_KEYS`; `library_genotypes` is in `_SIM_PATH_KEYS`. `tfs-setup-grid` needs nothing: configure reads the original library YAML at setup. Pilot that motivated it: `research/projects/2026-10-07_lasr-bgmp/pilot/2026-10-10_structural-prior-pilot` (`scripts/tfs.py` monkeypatched `get_libraries`; `make_configs.py` did the depth scaling by hand). Tests: `tests/tfscreen/genetics/test_library_subsample.py`, `tests/tfscreen/simulate/test_library_subsample_sim.py`, grid staging in both grid CLI tests. ### Pre-fit model accounting (`tfmodel/model_stats.py`) @@ -211,13 +280,23 @@ The simulation pipeline generates ground-truth phenotypes for a synthetic TF scr | `genotype_theta_df` | Long-form theta predictions (one row per genotype × titrant_conc) | | `parameters_df` | One row per genotype; per-genotype Hill/theta params + dk_geno + activity | | `binding_theta_df` | Theta at binding concentrations for calibration genotypes (`None` if not configured) | +| `library_binding_theta_df` | Only with `return_binding_theta=True` (what `tfs-simulate` passes), always last: theta of every library genotype at the binding concentrations (`None` without `binding_data`); `library_binding` reads it | +| `mutation_truth` | Only with `return_mutation_truth=True` (what `tfs-simulate` passes): `{"mutation": df or None, "pair": df or None}`, see below | The core calculation happens in `thermo_to_growth` (`simulate/thermo_to_growth.py`): 1. **Prior-predictive theta sampling** — `sample_theta_prior` draws a `theta_gc` matrix of shape `(G, C)` where G = number of library genotypes and C = number of unique titrant concentrations. The genotype order here matches the library (`sim_data`) order. 2. **theta_gc_override injection** — specific rows of `theta_gc` can be replaced before any further computation (see below). 3. **Growth rate calculation** — theta is mapped to growth rates via the configured `growth_params`. -4. **parameters_df assembly** — per-genotype Hill parameters extracted from the `theta_param` pytree, then patched with `theta_params_override` (see below). +4. **parameters_df assembly** — per-genotype Hill parameters extracted from the `theta_param` pytree, then patched with `theta_params_override` (see below). `_theta_param_to_df` takes both (T, G) fields (one column, `_T{t}` suffixes when T > 1) and (G,) fields. The thermodynamic models' titrant-independent constants (`ln_K_op`, `ln_K_HL`, `ln_K_U` for C4; `ln_K_h_l`, `ln_K_h_o`, `ln_K_l_o`, `ln_K_u` for C12) are (G,); until 2026-10-10 they were dropped, so only `ln_K_E`/`ln_K_h_e`/`ln_K_l_e` had truth. + +### Per-mutation and per-pair truth (`tfs_sim_mutation_parameters.csv`, `tfs_sim_pair_parameters.csv`) + +`ThetaParam` carries only assembled per-genotype values. For components drawn on the prior-predictive path (all thermodynamic models), `sample_theta_prior(..., return_deterministic=True)` traces the draw (the trace does not change it) and `thermo_to_growth._mutation_truth_tables` turns the deterministic sites into two tables: one row per mutation with every `theta_d_*` site (prefix dropped, so `d_ln_K_op` etc.) plus, for PddG/PnnC, `ddG_{structure}` from `theta_ddG` (the per-state truth draw, see **Per-structure ΔΔG**); and one row per pair (canonical `A/B` label) with every `theta_epi_*` site. Column names are the `get_extract_specs` names, so they join `*_params_d_*.csv` on `mutation` and `*_params_epi_*.csv` on `pair`. `tfs-simulate` writes them when present; hill_geno/hill_mut go through `simulate()` (no trace) and write neither. The `ddG_*` columns are labelled with the component's `STRUCTURE_NAMES` (the order `theta_ddG` is in); until 2026-10-10 they took `sim_data.struct_names`, the CSV's column order, so a CSV whose columns were not in model order mislabelled them. + +`tfs-summarize-fit` (`_summarize_params`) joins a `*_params_d_X.csv` / `*_params_epi_X.csv` table to these files when they have a `d_X` / `epi_X` column (pair labels standardized on both sides), and otherwise falls back to the per-genotype truth: sim[mutation] − sim[wt] and the additive epistasis of the per-genotype values (the hill_mut path). `tfs-summarize-calibration` reads the resulting `ref` columns with no change of its own. The per-mutation file still scores a mutation whose single mutant is not in the library. + +**Per-structure ΔΔG** (`theta/thermo/prior.py`). PddG/PnnC put the structural prior on the constants: per state `theta_ddG_sigma` (σ_s, S) ~ LogNormal(`theta_ddG_sigma_loc` 0, `theta_ddG_sigma_scale` 1.2; median 1 kT, 95% 0.095-10.5 kT; priors fields of all 8 modules, defaults `prior.DEFAULT_DDG_SIGMA_LOC/SCALE`), then per mutation `theta_dlnK` (M × C, sample site, mut plate dim -1, event C) ~ MultivariateNormal(P·μ_m, P·diag(σ_s²)·Pᵀ), μ the per-state prior mean (`theta_ddG_prior_mean`, M × S deterministic: the CSV values for PddG, the MLP output for PnnC), P = each module's `DLNK_PROJ` (`prior.projection(mwc.PROJ_*, mwc.PROJ_U_*)`: the folded rows incl. the per-site effector rows, then for U1 the unfolding row Δln K_U = ΔΔG_H; C = 3/4 for C4 U0/U1, 5/6 for C12), so the constants come out in `d_ln_K_*` order with `d_ln_K_U`/`d_ln_K_u` last. This is the distribution the per-state model ΔΔG ~ N(μ, σ_s) induces on what the data see. `scale_tril` is the Cholesky factor from a QR of (P·diag σ)ᵀ, once per evaluation, shared over the M mutations (`constant_scale_tril`); everything is in `STRUCTURE_NAMES` order (PddG and the C12 PnnC permute the data's columns into it, the C4 PnnC requires it). Why (2026-10-11, after c4f03d42): that commit sampled per-state `theta_ddG` ~ N(μ, σ_s) and projected it; in U0 every constant is a difference of states, so an equal shift of all states (P's null vector (1,…,1)) changes nothing, and joint MAP used it to zero one state's residuals and collapse its σ_s (C4_U0, true σ 0.3 kT: σ_LE2 0.001, ΔΔG RMSE 0.28). With the prior on Δln K there is no per-state latent and one σ_s → 0 keeps the covariance PD (the null vector has no zero entry); same problem now gives σ_s 0.28-0.32 on every state, Δln K RMSE 0.0497 (per-state form 0.0512, flat 0.0502). U1: P square and invertible, a pure change of variables (constant Jacobian), same MAP as the per-state form to 1e-6 (`test_ddG_constant_prior.py`). σ_s identifiability: σ_s enters only via PΣPᵀ; every σ_s is identified in all four families (the rank-one p_s p_sᵀ are independent: C4_U0 σ_H² = cov(K_op, K_HL), σ_L² = −2·cov(K_HL, K_E), HD/LE2 from the K_op/K_E variances; C12_U0 H and L each own off-diagonal terms, HO/LO/HE2/LE2 a variance each; `prior.identifiable_sigma`, `TestSigmaIdentifiability`). Remaining MAP limit (docstring): with weak data a σ_s can still drift toward 0 (density unbounded only where the covariance turns singular with all residuals inside it), so a σ_s near 0 from MAP means 'not resolved'. **Held σ_s** (`theta_ddG_sigma_fixed`, kT, default 0, a `pytree_node=False` field of all 8 ModelPriors, read by `prior.ddG_sigma_fixed(priors)` like hill's `_fixed()`): > 0 makes `sample_dlnK(..., sigma_fixed=)` record `theta_ddG_sigma` as a deterministic (S,) of the held value (the MVN uses it) and `guide_dlnK(..., sigma_fixed=)` register no σ_s params or site; AutoDelta/Laplace then have no σ_s, posteriors still carry the deterministic site, so `ddG_sigma` is still extracted (every row the held value) and the summary truth join still works; `get_guesses` keeps its σ_s entry (no sample site matches it). One scalar for every state, always kT (not converted by `ddG_units`); for the simulation-study oracle arm. Set with `tfs-configure-model --set_priors theta_ddG_sigma_fixed=1.69` (full name `theta.theta_ddG_sigma_fixed`). Tests: `theta/thermo/test_ddG_sigma_fixed.py`. **Post-hoc per-state ΔΔG**: deterministic `theta_ddG` (M × S, `prior.posthoc_ddG`): U1 = P⁻¹·Δln K exactly (ΔΔG_U = 0); U0 = the conditional mean μ + ΣPᵀ(PΣPᵀ)⁻¹(Δln K − Pμ), whose common mode is the table's, not the data's (reproduces the constants exactly; Δln K = Pμ gives μ in every family). Score per-state ΔΔG only for U1. Component guides call `prior.guide_dlnK` (`theta_ddG_sigma_locs/_scales` LogNormal; `theta_dlnK_locs/_scales` Normal(...).to_event(1), (M, C), locs start at P·μ for PddG, 0 for PnnC); `get_guesses` uses `prior.dlnK_guesses` (Δln K = P·μ, σ_s = 1 kT). AutoDelta/Laplace see `theta_dlnK` and σ_s as ordinary sites (Hessian included). `predict_unmeasured` is unchanged: it assembles from the `d_ln_K_*` posteriors, which for a library mutation without data are its prior (P·μ); a mutation outside `mut_labels` is still NaN. The pairwise horseshoe (`horseshoe.sample_pair_ddG`) stays per state and non-centered on purpose: its scales are sample sites with proper heavy-tailed priors (finite MAP optimum), and a centered horseshoe would give MAP an unbounded spike at τλ̃ → 0 that zeros weakly measured pairs; in U0 its common-mode direction is unconstrained by the data and regularized only by the shrinkage (docstring). `prior.ddG_extract_specs` (called by all 8 `get_extract_specs`) writes one table per structure, `ddG_{structure}` (from the post-hoc site) and `ddG_prior_mean_{structure}`, keyed by `mutation`, plus one `ddG_sigma` table keyed by `structure`; it uses the spec key `out_names` (`extraction._extract_param_est`), which names a spec's output tables when several read slices of one site. `tfs-summarize-fit` joins a direct table named for a per-mutation truth column (`ddG_H`) on `mutation`, and scores a prior-mean table against the same truth (`ddG_prior_mean_H` vs `ddG_H`: how well the structural prior alone predicts it) without a calibration, since it is a point (its quantiles coincide; `tfs-summarize-calibration` still lists it, so read only its rmse and pearson_r). For a U0 fit (`_ddG_per_state_identified(config_yaml)`, from the theta module's `DLNK_PROJ`) the `ddG_{structure}` summary is still written but with the truth in `ref_unidentified` (no `ref`, so `tfs-summarize-calibration` skips it) and `per_state_identified = False`, no plot, no calibration, and one printed note: score U0 fits on `d_ln_K_*`. `_summarize_ddG_sigma` joins `*_params_ddG_sigma.csv` to `*_sim_ddG_sigma.csv` on `structure`. This is the with/without-structural-priors comparison: a PddG/PnnC fit's `ddG_*` (U1) and `d_ln_K_*`, against a PK fit of the same simulation, which has no ΔΔG and is scored on `d_ln_K_*` (PddG/PnnC simulations write the `d_ln_K_*` truth too; PK's per-titrant `d_ln_K_E` rows join the per-mutation truth, which is uniform across effectors). For a PK fit, summarize prints one note that the truth's `ddG_*` have no table instead of warning per structure. Until 2026-10-10 nothing extracted `theta_ddG`, so the `ddG_*` truth was written but never scored. **Truth-noise dial** (`ddG_truth_sd`, a top-level simulate key in `SIMULATE_KNOWN_KEYS`, not in `thermo_options` so the fit-side reader never sees it and a grid varies it as an ordinary top-level variable): the simulator conditions the σ_s site (`numpyro.handlers.condition` inside `seed`, so the conditioned site draws no key), draws the per-state truth ΔΔG = prior mean + σ_s·z (`prior.per_state_truth`, key `fold_in(rng_key, 7919)`) and conditions `theta_dlnK` on P·ΔΔG in a second traced draw with the same key (`sample_theta._per_state_truth_conditions`; σ_s from the first draw, so an unconditioned call keeps its one σ_s draw), so σ_s is the SD of truth − table and the constants have exactly the model's MVN (`test_constant_covariance_matches_the_model`). `sample_theta.ddG_truth_conditions(sd, theta_component, thermo_options)` turns it into `{"theta_ddG_sigma": (S,)}` in kT, in `STRUCTURE_NAMES` order: scalar or `{structure: sd}` naming every structure, in `ddG_units` (converted with `ddG_to_kT_factor`, like the table), finite and >= 0 (0 gives truth = table exactly: the ΔΔG Normal is built with `validate_args=False` for this), refused for non-PddG/PnnC components and with `phenotype_source: empirical`. Absent = 1 kT per structure (`DEFAULT_DDG_TRUTH_SD`; chosen over drawing σ_s from its prior so the truth noise is always a known number). `library_prediction` always passes it for PddG/PnnC as `thermo_to_growth(theta_conditions=)` → `sample_theta_prior(conditions=)` (a name that is not a sample site is an error); a direct `sample_theta_prior` call without conditions draws σ_s from its prior. `sample_theta_prior(return_deterministic=True)` returns the deterministic sites plus the sample sites in `_TRUTH_SAMPLE_SITES` (`theta_ddG_sigma`), with `theta_ddG` replaced by the per-state truth draw (the model's post-hoc site cannot recover the U0 common shift). The truth tables: `ddG_{structure}` columns of `mutation_parameters` and, from `_ddG_sigma_truth_table`, `tfs_sim_ddG_sigma.csv` (`structure`, `ddG_sigma`, kT); `thermo_to_growth`'s truth dict has keys `mutation`, `pair`, `ddG_sigma`. The fit side is untouched (σ_s is a free sample site there), and `theta_ddG_prior_mean` stays the unperturbed table. Both 2026-10-11 changes moved the seeded PddG/PnnC truth (σ_s a site and ΔΔG drawn (M, S) rather than (S, M) offsets; then the redraw through `theta_dlnK`); the truth files keep their schema (`ddG_{structure}`, `d_ln_K_*`, `ddG_sigma`). For PnnC the sim's prior mean is the untrained MLP's output, 0. Tests: `tests/tfscreen/simulate/test_ddG_truth_sd.py`. ### Binding data and calibration genotypes @@ -225,11 +304,15 @@ The optional `binding_data` YAML block configures calibration genotypes for whic **`spiked_binding`** — clean, monoclonal (congression-free) controls; **pool = `spiked_seqs`**. - `choose_by: ` — the named genotypes get the file's Hill params as their true phenotype (see the measured-params bullets below). `num` is forbidden with a file. -- `choose_by: stratified|random` — assign diverse (greedy-maximin) / random prior-predictive theta curves to `num` spiked genotypes (default all); `wt` keeps its natural reference. Their theta at *growth* concentrations is injected via `theta_gc_override` so growth matches binding. +- `choose_by: stratified|random`, Hill and other non-thermodynamic components — assign diverse (greedy-maximin) / random prior-predictive theta curves to `num` spiked genotypes (default all); `wt` keeps its natural reference. Their theta at *growth* concentrations is injected via `theta_gc_override` so growth matches binding. Their `parameters_df` truth is not patched (no `theta_params_override`), so it still describes the replaced draw. +- `choose_by: stratified|random`, thermodynamic components (`thermo.*`, `library_prediction._spiked_keeps_own_theta`) — no override: the spikes keep their own model-generated theta, so binding theta, growth theta and the K truth all come from one draw (pasting a pool curve over a genotype whose theta follows from shared per-mutation K's would leave its K truth describing a curve that was never simulated). `stratified`/`random` only choose *which* `num` spikes are measured (`_select_own_theta_spiked`: greedy maximin over the spikes' own binding curves, or a random draw), after `thermo_to_growth`. Until 2026-10-10 this path built a wt-only stratified pool, which the thermodynamic models could not draw (a numpyro plate of size 0). **`library_binding`** — in-library controls drawn from the **bulk** (congression-affected growth); **pool = library genotypes NOT in `spiked_seqs`**. Omit to disable. -- These are regime-matched to the bulk: on the fit side they are simply extra `binding_df` rows and, because they are *not* in `--spiked`, they get `congression_mask=True` automatically. No fit-side change. +- These are regime-matched to the bulk: on the fit side they are simply extra `binding_df` rows and, because they are not spiked, their growth gets the bulk `bulk_fraction` automatically (binding itself is never congressed). No fit-side change. - **Pre/post-sim split** (`simulate/library_binding_data.py::generate_library_binding_df`): the `file` path injects the genotypes' phenotype *pre-sim* (in `library_prediction`, via the same override machinery), but the binding *measurement* — and, for `stratified`/`random`, the *selection* — happen **post-growth-sim** so selection is restricted to genotypes that actually survived with growth data (guaranteeing `num` usable anchors). `file`-specified genotypes that don't survive are **warned** and dropped. `simulate_cli.py` writes the selected set to `tfs_sim_library_binding.csv`. +- **Binding theta is evaluated, not looked up.** `thermo_to_growth(..., binding_titrant_conc=...)` returns (last) the theta of every genotype at the binding concentrations (`_theta_at_concs`: the component's own `run_model` on the drawn `theta_param` with `sim_data_class.with_titrant_conc`; genotypes overridden with complete Hill `theta_params_override` follow those params; overridden ones without are NaN). `library_prediction(..., return_binding_theta=True)` passes it on, and `generate_library_binding_df(..., theta_df=...)` selects and measures from it, so `library_binding` works for any theta component and at binding concentrations that are not growth concentrations. Without `theta_df` it falls back to the Hill columns of `parameters_df` (until 2026-10-10 the only path, so `library_binding` raised `KeyError: 'theta_low'` for the thermodynamic models). + +**Noise** (`binding_data.noise`): Gaussian on `theta_obs`, not clipped to [0, 1] (the fit's binding likelihood is an unclipped Normal; clipped anchors biased the absolute theta scale). `binding_data.clip_theta_obs: true` reproduces simulations made before 2026-09-29. **Validation** (`library_prediction._validate_binding_config`, fail-fast): `spiked_binding` file genotypes must be ⊆ `spiked_seqs`; `library_binding` file genotypes must be disjoint from `spiked_seqs`; `num` + a file is an error; a file requires a Hill theta component; `spiked_binding.num` ∈ `[1, num_spiked]`; `library_binding` stratified/random requires `num`. @@ -260,10 +343,20 @@ The optional `base_growth_data` YAML block generates a simulated `base_growth_df - Each growth model uses different YAML parameter names than the fit's extract names, and the module hand-maintains the mapping (`b,m` → `growth_k,growth_m` for `linear`; `b,a,n` → `growth_k,growth_m,growth_n` for `power`; `kmin,kmax` → `growth_min,growth_max` for `saturation`) by comparing `simulate/growth/growth_linkage.py`'s numpy formulas against the corresponding JAX formulas in `generative/components/growth/*.py`. If a new `condition_growth` component is added, this mapping must be extended too, or its parameters won't get ground-truth comparison in `tfs-summarize-fit`. - On the `tfmodel` side, `tfmodel/scripts/summarize_fit_cli.py::_summarize_condition_growth_params` and `_summarize_k_ref` join these two files against the fit's extracted params (on `condition_rep`, and trivially for the single-row `k_ref`, respectively) to annotate them with a `ref` column, mirroring `_summarize_params`'s genotype-keyed comparison but for growth's condition-keyed/global-scalar parameters. +### Joint-fit simulator keys (joint-fit plan step 5) + +All off by default. A `linear` growth entry takes `warp_a` (> 0) and `warp_x0` (default 0.5): k = b + A·m·f(θ), f the fit's `m_warp` logistic (`simulate/growth/growth_linkage.warp_theta`), applied per cell in `selection_experiment` too; truth `warp_log_a` = log a and `warp_x0` in `tfs_sim_growth_parameters.csv`. Top-level `growth_titrant_scale` (s; in `SIMULATE_KNOWN_KEYS`): `library_prediction` replaces sim_data's growth concentrations by s × nominal (`with_titrant_conc`) right after building it, so the prior draw, `_pin_wt_reference`, the measured-params and empirical overrides and (through a scaled `sample_df` copy) the spiked stratified pool all evaluate growth theta at s × c, while `_theta_at_concs` and the binding pools use the nominal binding concentrations; `tfs_sim_genotype_theta.csv` is therefore the theta growth sees, and the Hill truth in `tfs_sim_parameters.csv` is in binding units, like the fit's curve; truth log s in `tfs_sim_growth_titrant_scale.csv`. `binding_data.genotype_offset_sd`: one Normal offset per binding genotype added to every theta_obs of it (`binding_data.apply_genotype_offsets`, after spiked and library binding are assembled; no draw at 0), truth `tfs_sim_binding_offsets.csv`. Tests: `tests/tfscreen/simulate/test_joint_fit_sim.py`. + +### Sampling noise (roadmap step 4) + +Optional simulate-config keys, all off by default (a config without them simulates byte-for-byte as before): `founder_sampling` (`_sim_growth`: each tube gets a Poisson number of cells per transformant clone, drawn per tube), `demographic_growth` (given founders, Gamma(n0, e^kt) growth or Binomial(n0, e^kt) death; needs `founder_sampling`), `shared_transformation` (one library assembly and transformation shared by all replicates through the `shared_state` dict `simulate_cli` passes to `selection_experiment`; without it each replicate redraws both, unlike the real protocol's single glycerol stock), `pcr_template_molecules` / `pcr_amplification_cv` (`_sim_sequencing`: reads drawn from a multinomial set of template molecules, each amplified by a Gamma factor; count variance grows by about `(reads/templates)(1 + cv^2)`). `selection_experiment(..., sequence=False)` simulates totals (and OD600) without reads, for OD-only replicates. Real counts are 5-18x Poisson at 100-3,000 reads (`planning/studies/noise-anatomy/`); matching that needs templates on the order of reads per tube / 10. + ### theta_gc_override and theta_params_override These two dicts are the mechanism by which binding data is "pinned" into the growth simulation. +Neither is used for spiked stratified/random binding with the thermodynamic components, whose spikes keep their own theta (see the binding section). + **`theta_gc_override`** (`dict[str, np.ndarray]`): - Keys are genotype strings; values are 1-D arrays of theta at the growth titrant concentrations (sorted-unique order from `sample_df`). - Applied in `thermo_to_growth` *after* prior-predictive sampling but *before* noise and all downstream computations. @@ -288,26 +381,29 @@ These two dicts are the mechanism by which binding data is "pinned" into the gro | `simulate/base_growth_data.py` | Generates simulated direct growth-rate calibration data (`base_growth_data` YAML block) and the single-row `k_ref` ground-truth echo | | `simulate/growth_parameters_output.py` | Generates per-condition `condition_growth` ground truth (`tfs_sim_growth_parameters.csv`) from the `growth` YAML block | | `simulate/presplit_data.py` | Generates simulated pre-split (t = -t_pre) data (`presplit_data` YAML block; `generate_presplit_df`) | +| `simulate/od600.py` | Simulated OD600 per tube (`od600` YAML block): inverts a `tfs-calibrate-od600` calibration (read/apply/invert live in `process_raw/od600.py`) to get each tube's true OD600, adds reading noise, flags detectable/in-calibrated-range, and (`sample_cfu_from_od600`) runs it forward to the lab's estimate. `simulate_cli` writes `tfs_sim_od600.csv` for every replicate, including `num_od_only_replicates` extra unsequenced ones | | `simulate/sample_theta.py` | `sample_theta_prior` (prior-predictive) and `sample_theta_stratified` (greedy maximin) | | `simulate/sim_data_class.py` | `SimData` container and `build_sim_data` factory | -| `simulate/build_sample_dataframes.py` | Constructs sample/timepoint DataFrames from simulation config | -| `simulate/selection_experiment.py` | Models the selection experiment (growth + sequencing) | +| `simulate/build_sample_dataframes.py` | Constructs sample/timepoint DataFrames from simulation config: `build_sample_dataframes` (the `condition_blocks` grid) or, with the `design` key, `read_design`/`design_conditions` (a tube table in tfs-process-counts format; `library_prediction` simulates the union of its conditions and `simulate_cli` gives each replicate exactly its own tubes, carrying the design's tube names into the raw output as `design_sample`; refused with `condition_blocks` or `od600.num_od_only_replicates`) | +| `simulate/selection_experiment.py` | Models the selection experiment (transformation, growth, sequencing). A co-transformed cell splits its abundance among its plasmids (`_plasmid_shares`) and has one growth rate built from cell-level physics (`_cell_kt`); `_check_cell_components` fails fast if the phenotype's `k_pre`/`k_sel` can't be rebuilt from its theta/activity/dk_geno | +| `simulate/cell_rules.py` | Pluggable rules combining a co-transformed cell's plasmids: `THETA_RULES` (`homodimer`, the default: `logit θ_cell = log Σ x_g exp(l_g)` over equal shares; `heterodimer`: `2 log Σ x_g exp(l_g/2)`; both require activity 1 and raise otherwise; `max`: highest-θ plasmid sets θ and activity) and `DK_RULES`, the soft-min family `dk_cell = -(1/alpha) log sum_g x_g exp(-alpha dk_g)` (`dilution`, the default: share-weighted mean, alpha -> 0; `softmin`: finite `congression_dk_alpha`; `min`: the worst variant, alpha -> inf), chosen by `congression_theta_rule`/`congression_dk_rule`. The fit's mixture uses the same rules (`mixture.cell_classes`, `THETA_EPS` shared). Growth is then `thermo_to_growth.growth_rate_one_condition`, the same formula used per genotype. Never combine whole growth rates across plasmids | | `simulate/scripts/simulate_cli.py` | `tfs-simulate` CLI entry point; runs `library_prediction` + `selection_experiment` across replicates and writes all output CSVs, delegating the optional `binding_data`/`presplit_data`/`base_growth_data` blocks to their respective generator modules above | -| `simulate/scripts/report_cfu0_cli.py` | `tfs-report-cfu0` CLI entry point; reuses `library_prediction` + `selection_experiment` (same pattern as `simulate_cli.py`) across `num_replicates` to report mean `ln_cfu_0` and surviving-genotype counts by class (`wt`/`spiked`/`single`/`double`), for tuning `transform_sizes`/`library_mixture`/`cfu0` against observed real-library values | +| `simulate/raw_output.py` | `write_raw_experiment`: the simulated experiment in the raw formats real data come in (pipeline plan step 6), called by `tfs-simulate`. One `counts_tubeNNNN.csv` per sequenced tube in `tfs-process-fastq`'s format (every library genotype, zeros included, plus `__unknown__` = 0), the tube table (design columns only; `sample_cfu`/`sample_cfu_std` without an `od600` block), and with one the OD600 table and a copy of the calibration; tubes read below the detection threshold are left out (`undetectable_tubes`, which `simulate_cli` also applies to the direct growth table and to the in-library binding survivors; `tfs_sim_od600.csv` keeps every reading and presplit is unaffected; `test_undetectable_tubes.py` checks both routes give the same tubes). Fixed-width tube names keep the `counts*{sample}*.csv` glob unique. `tfs-process-counts` on these files reproduces `tfs_sim_growth.csv` row for row (`test_raw_output.py`); ln_cfu differs only by the pseudocounts of library genotypes never seen, which a real counts file lists | | `simulate/run_simulation.py` | Simpler, non-CLI orchestrator (`run_simulation`); does not support `binding_data`/`presplit_data`/`base_growth_data` | ### Empirical phenotype pipeline (`simulate/empirical/`) An alternative to prior-predictive phenotype sampling: instead of drawing theta from made-up priors, fit **real** screen data to an empirical phenotype-generating distribution and resample from it, so the simulated library's phenotype *distribution* matches reality while ground truth stays known. Targets `hill_geno` + linear growth; asserts `A ≡ 1` (repressor: blocks or not, leaky binding absorbed into `theta_low`). Three stages: -**Module location note.** Stage 1 (per-genotype MLE fit) and Stage 1.5 (congression de-attenuation) live in `tfmodel/genotype_fit/` (`fit.py`, `congression.py`) — they are a general per-genotype inference engine for the growth model, exposed standalone as **`tfs-fit-genotypes`** (`tfmodel/scripts/fit_genotypes_cli.py`). `simulate/empirical/fit_phenotypes.py` and `.../congression.py` are back-compat re-export shims. `tfs-fit-genotypes` takes `growth_file` + `calibration_file` (positional; a prefit priors CSV or wide `condition_rep,growth_k,growth_m`) and writes `_params.csv` (raw fit), `_theta.csv` (predicted θ vs `[genotype,titrant_name,titrant_conc]`), and — with `--congression_lambda` — `_params_deattenuated.csv` + a `theta_deattenuated` column (and, with `--save_theta_history`, the fixed-point trajectory `_theta_history.csv`). Reusable helpers `fits_to_results_df(fits)` (rebuild the params table from any fits dict) and `predict_theta(fits, growth_df)` back both this CLI and `tfs-build-empirical`. **`_stage1_fits.csv` from `tfs-build-empirical` is the RAW (pre-de-attenuation) fit; the de-attenuated fits are now also written to `_stage1p5_fits.csv` when `--congression_lambda` is given.** The Stage-1.5 correction is inherently a *population* operation (the background CDF couples all bulk genotypes), even though the per-genotype MLE fits themselves are independent/parallel. +**Module location note.** Stage 1 (per-genotype MLE fit) lives in `tfmodel/genotype_fit/fit.py` — a general per-genotype inference engine for the growth model, exposed standalone as **`tfs-fit-genotypes`** (`tfmodel/scripts/fit_genotypes_cli.py`). `simulate/empirical/fit_phenotypes.py` is a back-compat re-export shim. `tfs-fit-genotypes` takes `growth_df` + `growth_calibration_file` (positional; a prefit priors CSV or wide `condition_rep,growth_k,growth_m`) and writes `_params.csv` and `_theta.csv` (predicted θ, column `theta_raw`, vs `[genotype,titrant_name,titrant_conc]`). Reusable helpers `fits_to_results_df(fits)` (rebuild the params table from any fits dict) and `predict_theta(fits, growth_df)` back both this CLI and `tfs-build-empirical`. + +**No congression correction (Stage 1.5 retired, 2026-09-24).** The Stage 1 fits of bulk genotypes carry the congression bias (small in the measured regime, about 0.05 ln units), and Stage 2 builds the distribution from them as-is. The old Stage 1.5 de-attenuated θ with the θ-only `E[max]` operator the main fit dropped in step 3.3c (Poisson rather than zero-truncated co-resident weights, binary spiked/bulk purity, no dk dilution); `--congression_lambda` was removed from `tfs-fit-genotypes` and `tfs-build-empirical`. A proper replacement (an iterated per-genotype refit under the same observable-level mixture as the fit) waits on a review of the whole empirical pipeline (`planning/empirical-mixture-refit.md`). - **Stage 1** (`fit_phenotypes.py`): per-genotype MLE of the growth model on a real `ln_cfu` DataFrame (derives `ln_cfu_std` from the processed `ln_cfu_var` via `get_scaled_cfu`, like `model_orchestrator`), calibration (`growth_k`/`growth_m` per `condition_rep`) frozen. Fits `(dk_geno, theta_low, theta_high, log_hill_K, hill_n)` in transformed coords (logit theta bounds, log n) so `run_least_squares` returns covariance in the space Stage 2 needs. Per-genotype independent (no cross-genotype coupling); weak `dk_geno` Tikhonov prior; ±16 logit clamp. Returns `GenotypeFit(estimate, covariance)` per genotype. The fits are embarrassingly parallel — `num_workers` (`-1` = `cpu_count-1`) runs them over a `ProcessPoolExecutor`; the parallel path strips the genotype `Categorical` first (it carries all categories on every group → O(N²) pickling). Worker startup pays a ~2s JAX import, so parallelism is a wash for tiny/fast runs and near-linear for large real libraries. `_hill_theta` is numerically identical to `hill_geno.run_model` and `binding_params._hill_theta` (verified — same `_ZERO_CONC_SENTINEL`). - **Stage 2** (`population.py`): measurement-error EM (`z_i~N(mu,Σ)`, `y_i~N(z_i,S_i)`) that **deconvolves estimation noise** (`Cov(y)=Σ+mean(S_i)`), so it recovers a narrower population than a naive KDE on the point estimates. `fit_population()` → `PopulationModel` (single MV-Normal in transformed space; `wt_ref` field holds wt's actual Stage-1 fit; `.sample(n)` → natural-space DataFrame; `.save()`/`.load()`). -- **Stage 1.5** (`congression.py`, optional; between Stage 1 and Stage 2, gated by `--congression_lambda`): de-attenuates the **bulk** genotypes' theta curves for co-transformation. Reuses the inference's own θ-level operator `transformation._congression.update_thetas` (the `E[max(x,M)]` **dominant-max occupancy** map — the tightest-bound operator sets effective θ, so `E[max]` only, never the min variant — with an empirical background CDF). `correct_theta_matrix` is the fixed point: `θ_true ← θ_obs`; iterate `θ_true += gain·(θ_obs − update_thetas(θ_true; background=θ_true))` per-concentration until converged (the background *is* the corrected population, hence the iteration). `deattenuate_congression` evaluates each bulk genotype's Stage-1 Hill at the growth `titrant_conc` grid, corrects, and refits Hill (dk_geno and the Stage-1 covariance are left untouched — a deliberate bias-only correction; Stage 2's estimation-noise deconvolution still runs after). λ passes straight through **unconverted**: a focal barcode is size-biased into its cell, so co-residents are Poisson(λ) at the *same* zero-truncated `transformation_poisson_lambda` (`_sim_transform` uses `zero_truncated_poisson` + i.i.d. plasmid draws). Spiked genotypes are congression-free → excluded from correction and the background CDF, and pass through unchanged. This is the θ-level analogue of the simulator's growth-level congression (agree to first order, exact at λ→0); spiked-only vs bulk distribution is the external check. - **Stage 3** (`resample.py`): `resample_phenotypes` draws one i.i.d. phenotype per genotype (wt pinned to `dk_geno=0` + `wt_ref` Hill); `make_empirical_overrides` reuses `build_theta_gc_override_hill_geno` → `(theta_gc_override, theta_params_override, dk_geno_override)`. `thermo_to_growth` gained a `dk_geno_override` param (the one growth-path change; `theta_params_override` only patches θ columns). `build_empirical_binding_theta` rebuilds spiked `binding_theta_df` from resampled params (selection from the `binding_data` config; θ from resampled Hill, not `sample_theta_stratified`). -Integration: `library_prediction` gains a `phenotype_source: empirical` branch (needs an `empirical: {phenotype_model: }` block pointing at the single self-contained `_phenotype_model.json`; `_resolve_phenotype_model_path` accepts that path with/without `.json` or the bare ``, and the fit prints the absolute path — use it so no file-copying is needed) that resamples all genotypes, injects the overrides, forces `theta_component=hill_geno` (warns + ignores any other value; the resampled phenotypes are per-genotype Hill curves and the discarded prior draw must match the `parameters_df` schema — this also avoids running/overflowing an e.g. `hill_mut` draw), drops the ignored `theta_priors`/`theta_sim_priors`, and forces `activity=fixed/1`. `phenotype_source`/`empirical` are in `selection_experiment.SIMULATE_KNOWN_KEYS`. Resampled ground truth flows into `parameters_df`/`genotype_theta_df` automatically (no new return value); `library_binding` regenerates for free (reads `parameters_df`). `tfs-build-empirical` (`simulate/scripts/build_empirical_cli.py`) is a **one-command orchestrator**: given the experimental inputs (`growth_file` and `seed` positional; `--binding_file` required; optional `--spiked_file`/`--base_growth_file`/`--thermo_data`/`--congression_lambda`/`--num_workers`) it internally calls `configure_model` (linear + hill_geno defaults — no model choices exposed, since here they'd only be wrong) then `run_prefit_calibration` (MAP-calibrates per-condition k/m), then Stages 1-2 (plus optional Stage 1.5 congression de-attenuation when `--congression_lambda` is given), saving the deliverable `_phenotype_model.json` (one self-contained, human-readable file = the generating distribution; `PopulationModel.save`/`.load`) + the diagnostic `_stage1_fits.csv`, plus the `_configure_*`/`_prefit_*` intermediates. The configure/prefit imports are lazy (the heavy JAX stack loads only on this path). The MAP prefit is the slow step, so `--calibration_file` skips configure+prefit and reuses a calibration for fast Stage-1/2 iteration — either a prefit **priors CSV** (read via `fit_phenotypes.read_calibration`, which pivots the `growth.condition_growth.k_loc`/`growth.condition_growth.m_loc` per-`condition_rep` rows — prefit writes these via `_csv_row_name` = `growth.{component}.{field}` — to wide k/m) or a wide `(condition_rep, growth_k, growth_m)` CSV. This **complements** the fully-synthetic prior path (the *accuracy* benchmark) as a *realism* benchmark. +Integration: `library_prediction` gains a `phenotype_source: empirical` branch (needs an `empirical: {phenotype_model: }` block pointing at the single self-contained `_phenotype_model.json`; `_resolve_phenotype_model_path` accepts that path with/without `.json` or the bare ``, and the fit prints the absolute path — use it so no file-copying is needed) that resamples all genotypes, injects the overrides, forces `theta_component=hill_geno` (warns + ignores any other value; the resampled phenotypes are per-genotype Hill curves and the discarded prior draw must match the `parameters_df` schema — this also avoids running/overflowing an e.g. `hill_mut` draw), drops the ignored `theta_priors`/`theta_sim_priors`, and forces `activity=fixed/1`. `phenotype_source`/`empirical` are in `selection_experiment.SIMULATE_KNOWN_KEYS`. Resampled ground truth flows into `parameters_df`/`genotype_theta_df` automatically (no new return value); `library_binding` regenerates for free (reads `parameters_df`). `tfs-build-empirical` (`simulate/scripts/build_empirical_cli.py`) is a **one-command orchestrator**: given the experimental inputs (`growth_df` positional; `--binding_df` and `--seed` required; optional `--library_config`/`--base_growth_df`/`--thermo_data`/`--num_workers`) it internally calls `configure_model` (linear + hill_geno defaults — no model choices exposed, since here they'd only be wrong) then `run_prefit_calibration` (MAP-calibrates per-condition k/m), then Stages 1-2, saving the deliverable `_phenotype_model.json` (one self-contained, human-readable file = the generating distribution; `PopulationModel.save`/`.load`) + the diagnostic `_stage1_fits.csv`, plus the `_configure_*`/`_prefit_*` intermediates. The configure/prefit imports are lazy (the heavy JAX stack loads only on this path). The MAP prefit is the slow step, so `--growth_calibration_file` skips configure+prefit and reuses a calibration for fast Stage-1/2 iteration — either a prefit **priors CSV** (read via `fit_phenotypes.read_calibration`, which pivots the `growth.condition_growth.k_loc`/`growth.condition_growth.m_loc` per-`condition_rep` rows — prefit writes these via `_csv_row_name` = `growth.{component}.{field}` — to wide k/m) or a wide `(condition_rep, growth_k, growth_m)` CSV. This **complements** the fully-synthetic prior path (the *accuracy* benchmark) as a *realism* benchmark. ## Categorical response assessment (`analysis/cat_response/`) @@ -396,6 +492,58 @@ overconfident ones, exactly what Axis 2 exists to detect. arguments are direct CSV paths (`rep1.csv rep2.csv`); a single argument is a manifest file (one path per line) *unless* it ends in `.csv`. +## Planning (`planning/`) + +Ideas, active plans and studies live in `planning/` (called `future/` before +2026-09-23); `planning/README.md` has the conventions. **Read +`planning/findings-ledger.md` before proposing a plan or study on the +growth/binding model, lambda, the k/dk_geno slide or m:** it lists what is +established, refuted and open, with sources (2026-10-18: a week of +joint-fit work had re-found July and early-October results). Ideas and plans are one +Markdown file each with a YAML header whose `status` (`idea`/`active`/ +`promoted`/`dropped`/`done`) tells them apart. An active plan keeps its step +list current. Studies live in `planning/studies//` (a `README.md` with +question, decision fed, how to run, inputs, commit and results, plus the +script); once a plan cites a study it is frozen. `dev/` is untracked scratch; +anything a plan cites moves into `planning/studies/`. **The current +priority is `planning/release-0.5.0.md`** (active, 2026-10-19): the +close-out checklist for 0.5.0 (never tagged; the version stays 0.5.0), +the release Natalie's manuscript uses, so that someone in the lab can +take a screen from counts to X with reasonable intervals through +`tfs-fit-two-stage`. Its decisions are settled (repair the k shift or ship +without it; no marker warp; the reversed pheS row excluded; the measured +lambda prior; kanR's m from the tube totals); validation round 2 runs in +`planning/studies/two-stage-validation/` (run 2). The paper's analysis, +thermodynamic models with a floating protein concentration, and +experimental design come after it. `planning/joint-fit-return.md` is +paused (binding cannot yet set X's scale; ledger), and +`planning/relative-m.md` is an idea. The other active plans are +`planning/congression-physics-plan.md`, `planning/analysis-roadmap.md` +(steps 2, 6, 7b, 8 and 9 left; outcome so far in +`planning/analysis-roadmap-summary.md`) and `planning/experiment-pipeline.md` +(one experiment file from raw data to posterior; steps 0-4, 6 and 7 done, +step 3 as machinery; step 5 (orchestrator) deferred; next, step 8, a +science prioritization that takes up +`planning/offset-mode-growth-transition.md`). The single-MAP real-data +recipe is `planning/studies/real-data-fit/fit/run_recipe.srun` (it applies +`planning/dev-data/tube_corrections.yaml` with `tfs-correct-tubes` before +`tfs-process-counts`); the two-stage fit has replaced it for X. The full-size +simulation (`planning/studies/full-size-sim/`) is step 8's baseline: X +undercovers on a full library, worse with read depth, because the k/m/ +dk_geno shared errors are held at the MAP by the arrowhead Laplace. +`planning/deep-coverage.md` (active, 2026-10-09) works on that: honest +intervals for the well-measured genotypes from this experiment's data, +starting with k/m priors from the wt monoculture rates +(`planning/studies/km-anchor/`). The full-library MAP compresses X about +1.45-fold (each double pulls m steeper); the fix is a two-stage fit +(stage 1: wt, spikes and singles with the congression mixture, which +congression's pheS bias needs; stage 2: the full library with every +condition's k and m pinned there), with k's uncertainty from a stage-1 +jackknife added as one X shift shared by every genotype +(`planning/studies/m-bias-factorial/`). `planning/two-stage-fit.md` +(active) makes it one command; its pieces are in `tfmodel/two_stage.py`. `planning/design-noise-sweep.md` (idea) +holds the next screen's noise-source sweep. + ## YAML Standards All YAML files in this codebase follow these conventions. Apply them when creating or modifying any YAML file. @@ -453,10 +601,13 @@ template: - Multi-key variants (multiple keys in one dict) always travel together and are never split. - `simulate` / `configure_model` variables go to the config; `template` variables go to the Jinja2 template only. To share a variable, list it in both sections. - Variable names in blocks: `snake_case`. +- File paths are the grid tool's job, never the run template's, and a grid output directory must stay movable as a unit (on/off a cluster, onto a data partition). Both grid CLIs (output directory `--out_dir`) copy every input file into `/inputs/` once (`grid_utils.InputStager`; a numeric suffix keeps two different files with the same name apart, and a changed file never overwrites an existing copy) and each run refers to it as `../inputs/`. Relative paths resolve against the grid YAML's directory (overrides, `configure_model` variables and template variables) or, for `tfs-setup-sim-grid`, the base config's directory (base config values). A template variable naming an existing file is copied the same way. Setup fails, before writing anything, on a missing or directory input, on a config value outside the known path keys that names an existing file (`grid_utils.check_no_outside_paths`), or on a template that does not render. +- `tfs-setup-sim-grid`: the config file paths are the key paths in `_SIM_PATH_KEYS` (`setup_sim_grid_cli.py`; `library_genotypes` and nested keys such as `binding_data.*.choose_by` and `empirical.phenotype_model` included, `choose_by` keywords excepted). Add any new file-valued simulate key there. +- `tfs-setup-grid`: the inputs are the `configure_model` file arguments in `_PATH_KEYS` (`setup_grid_cli.py`: `binding_df`, `growth_df`, `presplit_df`, `base_growth_df`, `thermo_data`, `library_config`). `tfs-configure-model` reads the originals; the written config is then pointed at the copies wherever it recorded them (`data.*`, `components.thermo_data`/`presplit_df`/`base_growth_df`, `library.source`), and setup fails on any other path in it that names a file outside the grid. The priors/guesses/library CSVs are per-run outputs next to the config and stay as they are. Data paths are read relative to the working directory, so a run is launched from its own directory. Add any new file-valued `configure_model` argument to `_PATH_KEYS`. ### Shared grid utilities -Both grid CLIs import from `tfscreen.util.grid_utils` for run-name generation, Jinja2 environment setup, and config-path rewriting. Add new reusable grid helpers there rather than duplicating them. +Both grid CLIs import from `tfscreen.util.grid_utils` for run-name generation, Jinja2 environment setup and input staging (`InputStager`, `check_no_outside_paths`, `stage_template_vars`, `render_run_template`). Add new reusable grid helpers there rather than duplicating them. ### `examples/` directory @@ -464,6 +615,7 @@ Both grid CLIs import from `tfscreen.util.grid_utils` for run-name generation, J |------|---------| | `simulate/simulate_config.yaml` | Canonical well-commented simulate config reference | | `simulate/simulate_grid.yaml` | Example simulate grid for `tfs-setup-sim-grid` | +| `od600/` | Synthetic `tfs-calibrate-od600` inputs (`replicates.csv`, `plate_counts.csv`, from `make_example_data.py`) and the calibration made from them (`od600_calibration.yaml`, used by the simulate example's `od600` block); not any lab's real calibration | | `simulate/run.sh` | Jinja2 shell template rendered into each simulate grid run subdir | | `simulate-and-analyze/simulate_config.yaml` | Combined simulate + analyze workflow config | | `simulate-empirical/simulate_config.yaml` | Simulate config that resamples phenotypes from a `tfs-build-empirical` model (`phenotype_source: empirical`) | @@ -471,19 +623,24 @@ Both grid CLIs import from `tfscreen.util.grid_utils` for run-name generation, J | `simulate-and-analyze/run.sh` | Jinja2 shell template for simulate-and-analyze runs | | `tfmodel/grid.yaml` | Example tfmodel grid for `tfs-setup-grid` | | `tfmodel/run.srun` | Jinja2 SLURM template rendered into each tfmodel grid run subdir | +| `tfmodel/two_stage.yaml` | Example `tfs-fit-two-stage` configuration (tfs-configure-model's arguments plus the `two_stage` block) | +| `tfmodel/growth_priors_loose.csv` | The loose per-condition k/m priors `two_stage.yaml` names (the 2026 screen's) | +| `tfmodel/two_stage_submit.sh`, `tfmodel/two_stage_step.srun` | Submit `tfs-fit-two-stage` as chained SLURM jobs (jackknife as an array, absolute paths); edit the account/partition in the `.srun`. Documented in `docs/source/screen-analysis.rst`, the real-screen recipe | | `process_raw/library_config.yaml` | Minimal library config for `tfs-process-fastq` and `tfs-configure-model --library_config` (genetics keys + `library_mixture`) | +| `process_raw/` (`make_example_data.py`, `tube_table.csv`, `od600.csv`, `counts/`) | Synthetic tube table, OD600 table and per-tube count files in the formats `tfs-process-counts` reads; run with `../od600/od600_calibration.yaml` | +| `process_raw/corrections.yaml` | Example `tfs-correct-tubes` corrections file (one swap, one relabel, one exclusion) for `tube_table.csv` | ### `process_raw` YAML -`tfs-process-fastq` accepts a `run_config` argument that is passed to `LibraryManager`. It reads only the library genetics keys: `reading_frame`, `wt_seq`, `degen_sites`, `tiles`, `expected_5p`, `expected_3p`, `tile_combos`, `spiked_seqs`. All other keys are ignored. +`tfs-process-fastq` takes the library YAML as its first positional (`library_config`), passed to `LibraryManager`. It reads only the library genetics keys: `reading_frame`, `wt_seq`, `degen_sites`, `tiles`, `expected_5p`, `expected_3p`, `tile_combos`, `spiked_seqs`. All other keys are ignored. -**Required practice**: maintain **one** `run_config.yaml` per experiment and pass that same file to `tfs-simulate`, `tfs-process-fastq` and `tfs-configure-model --library_config`. Nothing in the pipeline checks that the three saw the same file: the genotype-set check in `tfs-configure-model` catches a genetics mismatch, but `library_mixture` (which only `tfs-configure-model` and `tfs-simulate` read) can silently drift. Use `examples/process_raw/library_config.yaml` only when you need a minimal standalone library config (e.g. for processing real data without a matching simulation); note it carries `library_mixture` as well as the genetics keys. +**Required practice**: maintain **one** library YAML per experiment and pass that same file to `tfs-simulate`, `tfs-process-fastq` and `tfs-configure-model --library_config`. Nothing in the pipeline checks that the three saw the same file: the genotype-set check in `tfs-configure-model` catches a genetics mismatch, but `library_mixture` (which only `tfs-configure-model` and `tfs-simulate` read) can silently drift. Use `examples/process_raw/library_config.yaml` only when you need a minimal standalone library config (e.g. for processing real data without a matching simulation); note it carries `library_mixture` as well as the genetics keys. ### Terminology - **Condition**: unique growth setting (marker + selection) — same avg growth rate for same genotype -- **Sample**: one experimental tube (replicate + condition) -- **Timepoint**: one aliquot (replicate + condition + time) +- **Tube = sequenced sample = timepoint**: after `presplit` the culture is split into one tube per condition × titrant × planned timepoint (a grid: rows = timepoints, columns = IPTG); pre-growth and selection both happen in the tubes; a timepoint is taken by pulling every tube in its row and reading its OD600, spinning it down and freezing it; sequencing comes later. Total CFU per tube comes from OD600 through a lab-specific calibration (not in this repo). There are no aliquots from a shared tube (stopping the shaker to pipette disturbed the trajectories more than tube-to-tube noise does). So anything tube-level (growth environment, OD600 reading, PCR) is independent across timepoints, not a trajectory-wide effect. Rationale: `docs/source/process-raw.rst`, "One tube per time-point" +- **Sample** (in `sample_df`/`sample_ln_cfu`): one sequenced tube, i.e. one replicate × condition × titrant × time. Older text sometimes says "sample" for a replicate × condition; read it per this entry - **theta (θ)**: operator occupancy — fraction of operators bound by TF - **A**: per-genotype TF activity (multiplied by theta to scale occupancy) - **dk_geno**: pleiotropic growth effect of mutation independent of TF activity @@ -510,23 +667,31 @@ All scripts use `generalized_main` from `tfscreen.util.cli.generalized_main`. Th - Parameters **without** a default → positional (required) arguments - Parameters **with** a default (including `None`) → `--flag` arguments +- A bool that defaults to False is a `--name` switch; one that defaults to True is turned off with `--no_name` (`--name` is accepted and does nothing). Never invert a flag's meaning. +- A `None` default reads a string unless `manual_arg_types` gives a type (`int`, `float`, or a list type with `manual_arg_nargs`). +- The numpydoc docstring is the `--help`: the text before `Parameters` is the description, each parameter's entry is its help, and defaults are appended unless the entry already says "default". Document every parameter. +- `generalized_main` returns None (console scripts run `sys.exit(main())`; a returned value would exit 1), prints the run's provenance (`util/provenance.py`: version, commit, dirty flag, command) and writes `{out_prefix}_provenance.json` when the function takes `out_prefix`. Configs (`provenance:` block) and fit checkpoints embed `get_provenance()`. +- After changing a CLI signature or docstring, run `python docs/scripts/make_cli_reference.py`; `tests/tfscreen/util/cli/test_cli_reference.py` fails while `docs/source/cli.rst` is stale. + Positional argument order (use only what the script needs): -1. `config_file` — path to YAML config -2. `posterior_file` — path to posteriors `.h5`/`.npz` +1. `config_file` — path to YAML config (`tfs-process-fastq`: the library YAML) +2. `param_file` — a posterior `.h5` or a MAP checkpoint `.pkl` (`tfs-sample-posterior` takes `checkpoint_file`) 3. `data_file` — path to a long-form observable CSV (e.g. `tfs-cat-response` - takes `data_file x_obs y_obs`; `tfs-extract-epistasis` takes `data_file y_obs`) + takes `data_file x_obs` with `--y_obs`; `tfs-extract-epistasis` takes `data_file` with `--y_obs`) + +Shared names: `--seed` for every seed, `--num_workers` (-1 = all CPUs but one), data inputs named as `tfs-configure-model` names them (`growth_df`, `binding_df`, `presplit_df`, `base_growth_df`), the library YAML is always `library_config`. ### `--library_config` -`tfs-configure-model`, `tfs-fit-genotypes` and `tfs-build-empirical` take the library YAML as `--library_config`. It is an exception to the positional-if-required rule: `tfs-configure-model` requires it only when `growth_df` is given, and `growth_df` is itself a flag, so a positional would be wrong for the binding-only call. Validate it in the function body instead. +`tfs-configure-model` and `tfs-build-empirical` take the library YAML as `--library_config`. It is an exception to the positional-if-required rule: `tfs-configure-model` requires it only when `growth_df` is given, and `growth_df` is itself a flag, so a positional would be wrong for the binding-only call. Validate it in the function body instead. The same holds for the data: `tfs-configure-model` takes `--binding_df` and `--growth_df` as flags and requires at least one (both: joint model; binding only: binding-only model; growth only: growth-only model). ### Output flag -Always `--out_prefix` (never `--out_root`, `--out`, or `--output_file`). The function parameter must also be named `out_prefix`. +Always `--out_prefix` (never `--out_root`, `--out`, `--output_file` or `--output_prefix`). The function parameter must also be named `out_prefix`, and outputs are `{out_prefix}.ext` or `{out_prefix}_.ext`. A command whose output is a directory of files takes `--out_dir` instead (`tfs-process-fastq`, `tfs-setup-grid`, `tfs-setup-sim-grid`). ### File-backed list arguments -When a list of genotypes, titrant names, or concentrations is needed, the `_cli` wrapper takes file-path strings (one value per line, `#` comments allowed). Use `manual_arg_types` in `generalized_main` to override the `NoneType` inferred from `default=None`. The shared helper `_read_lines(path)` lives in `tfscreen.util.cli`. +When a list of genotypes, titrant names, or concentrations is needed, the `_cli` wrapper takes file-path strings (one value per line, `#` comments allowed). Use `manual_arg_types` in `generalized_main` to override the `NoneType` inferred from `default=None`. The shared helper `read_lines(path)` lives in `tfscreen.util.cli`. ### `in_training_data` column diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..bbb8584d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,210 @@ +# Contributing to tfscreen + +Thanks for helping. This file covers how to propose a change and what a pull +request needs before it can merge. + +## Before you start + +For anything larger than a bug fix, open an issue first and describe what you +want to change and why. The model is a coupled system, and a change that looks +local often touches the priors, the guides, the simulator and the CLIs at once. +A short discussion up front saves rework. + +Ideas, active plans and the studies behind them live in `planning/`. Check +there before starting work on the model; your idea may already have a plan or +a recorded reason it was set aside. See `planning/README.md`. + +## Workflow + +We develop on forks and merge into `main` through pull requests. Nobody pushes +directly to `main`. + +1. Fork `harmslab/tfscreen` on GitHub and clone your fork. +2. Add the main repository as `upstream`: + + ```bash + git remote add upstream git@github.com:harmslab/tfscreen.git + ``` + +3. Create a branch for each change, named for what it does + (`fix-presplit-offset`, `add-baranyi-guide`). Keep one logical change per + branch. +4. Before opening the pull request, bring your branch up to date with + `upstream/main` and rerun the tests. +5. Open a pull request into `harmslab/tfscreen:main`. The pull request + template asks what changed, why, and how you verified it, and gives a + checklist of everything below. Work through it before asking for review. + +`main` is a protected branch. GitHub enforces the following: + +- Every change goes through a pull request. Direct pushes and force pushes to + `main` are blocked. +- A pull request needs one approving review. +- All CI checks must pass: the test matrix and the coverage job. +- The branch must be up to date with `main` before it can merge. If `main` + moves while your pull request is open, merge or rebase onto it and let CI + run again. +- Pushing new commits dismisses earlier approvals, so a reviewer has to + approve the final version. Finish your changes before asking for a last + review. + +## Development setup + +tfscreen needs Python 3.12 or later. Install it in editable mode with the test +dependencies: + +```bash +pip install -e ".[test]" +``` + +## Tests + +Every pull request needs full unit test coverage of the code it adds or +changes. In practice that means: + +- Every new function, branch and error path has a test that exercises it. + We measure branch coverage, not just line coverage. +- A bug fix comes with a test that fails before the fix and passes after it. +- Tests mirror the source layout: `src/tfscreen/foo/bar.py` is tested in + `tests/tfscreen/foo/test_bar.py`. +- Tests that take more than a few seconds get `@pytest.mark.slow`. They are + skipped by default and run with `--runslow`. +- End-to-end pipeline checks belong in `tests/smoke-tests/`. + +`tests/conftest.py` sets `NUMBA_DISABLE_JIT=1`, so no prefix is needed. Run the +unit tests: + +```bash +pytest tests/tfscreen +``` + +Add `-n auto` to run them in parallel (`pytest-xdist`, in the `test` extra). + +Check coverage, including slow tests. Coverage settings (branch coverage, +source) live in `pyproject.toml`, so the same commands work locally and in CI: + +```bash +pytest -n auto --runslow --cov --cov-report= tests/tfscreen +``` + +```bash +coverage report +``` + +To see what CI will say about the lines your branch changes: + +```bash +coverage xml +``` + +```bash +diff-cover coverage.xml --compare-branch=upstream/main +``` + +Some tests check rules across the whole component registry, for example that +every per-genotype latent is mini-batch safe and that guides follow the +`{site}_loc` / `{site}_scale` naming convention. If one of these fails for a +new component, fix the component rather than the test. + +CI runs the unit tests on Linux and macOS for Python 3.12, 3.13 and 3.14, plus +a lint check for fatal errors. A separate coverage job runs the tests with +`--runslow` and fails if either check misses: + +- Every line a pull request adds or changes must be covered by a test + (`diff-cover --fail-under=100`). diff-cover counts lines, not branches, so + check the branch columns of `coverage report` for your files yourself. +- Total coverage of the package must not drop below the floor set in + `.github/workflows/tests.yml`. Raise the floor when coverage improves. + +Run the same lint locally: + +```bash +flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics +``` + +## Code conventions + +`CLAUDE.md` is the detailed reference for the codebase. It is written for AI +coding tools but serves human contributors equally well. The parts you are +most likely to need: + +- **New model components:** the checklist under "Adding a new model + component", including registration and mini-batch safety. +- **CLI scripts:** the "CLI standards" section. Every `tfs-*` script uses + `generalized_main`, lives in `_cli.py`, writes with `--out_prefix` + and is registered in `pyproject.toml`. Its docstring is its `--help`, one + entry per parameter. After changing any CLI signature or docstring, + regenerate the reference page with `python docs/scripts/make_cli_reference.py`; + `tests/tfscreen/util/cli/test_cli_reference.py` fails until you do. +- **YAML files:** the "YAML standards" section. +- **Quantile outputs:** bare `q` column names, no feature prefix. + +### Docstrings + +Docstrings follow the +[numpydoc](https://numpydoc.readthedocs.io/en/latest/format.html) convention. +Every public function, class and method gets a full docstring: a one-line +summary, an extended description where needed, then `Parameters`, `Returns` +and `Raises` sections as they apply. Give each parameter's type and shape, +for example `numpy.ndarray of shape (num_genotype,)`, and say what units it +is in when that is not obvious. + +```python +def detect_match_keys(dfs, value_columns=(), match_by=None): + """ + Determine the columns that make a row "the same row" across runs. + + Parameters + ---------- + dfs : list of pandas.DataFrame + All tables that must share the key. + value_columns : iterable of str, optional + Columns holding estimates, excluded from the key. + match_by : list of str, optional + Explicit key columns. Overrides auto-detection. + + Returns + ------- + list of str + The match key columns. + + Raises + ------ + ValueError + If the key is not unique within a run. + """ +``` + +Private helpers (leading underscore) may use a one-line docstring when the +behavior is simple. + +### Validation + +Configuration errors should fail fast with a clear message. Do not silently +drop or coerce a value the user supplied. + +### Keeping `CLAUDE.md` current + +If your change alters how the code works, update `CLAUDE.md` in the same pull +request so it stays accurate. + +## Changelog + +Record every user-visible change in `CHANGELOG.md` under `[Unreleased]`, in +the same pull request as the change. The file follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Cite files and +functions rather than commit hashes. + +## Commits + +Write commit messages with an imperative subject line of 72 characters or +fewer ("Add soft-min congression dk rule", not "Added..."). Use the body to say +why the change was made and what it affects. Use American spelling. + +Do not commit data, checkpoints, notebooks of exploratory work or other large +files. `dev/` is untracked scratch space. + +## License + +By contributing, you agree that your contributions are licensed under the +project's license (see `LICENSE`). diff --git a/README.rst b/README.rst index e93afdff..cddd6a11 100644 --- a/README.rst +++ b/README.rst @@ -2,11 +2,15 @@ tfscreen ======== +.. image:: https://github.com/harmslab/tfscreen/actions/workflows/tests.yml/badge.svg?branch=main + :target: https://github.com/harmslab/tfscreen/actions/workflows/tests.yml + :alt: Tests + `tfscreen` is a Python library for simulating and analyzing high-throughput screens of transcription factor (TF) function. It is designed to infer the energetic effects of mutations on conformations -in the TF energy landscape — information that enables interpretable predictive models of how -arbitrary combinations of mutations affect the TF response curve (operator binding versus allosteric -effector). +in the TF energy landscape. Those effects make it possible to build interpretable models that +predict how any combination of mutations changes the TF response curve, operator binding against +allosteric effector. The core strategy is to measure operator occupancy for thousands of double-mutant cycles across multiple allosteric effector concentrations. A single set of energetic effects must explain all @@ -18,14 +22,17 @@ Occupancy is inferred from *E. coli* growth in a dual-marker selection scheme: * resistance, growth enhanced when expressed) and *pheS** (4-chloro-L-phenylalanine sensitivity, growth reduced when expressed) respond in opposite directions to a shift in occupancy. Libraries of TF variants are grown in different effector/selection combinations and variant frequencies are -followed over time by direct sequencing. A hierarchical Bayesian model jointly fits the growth -data and direct binding measurements, learning the relationship between occupancy and growth rate -along the way. +followed over time by direct sequencing. A hierarchical Bayesian model fits the read counts, +alone or jointly with direct binding measurements, and learns the relationship between occupancy +and growth rate along the way. Documentation ------------- -Full documentation is available at https://tfscreen.readthedocs.io. +Full documentation is available at https://tfscreen.readthedocs.io. To analyze a screen, start +with the "Analyzing a screen" page: per-tube read counts to per-genotype response curves with +intervals, through one command, ``tfs-fit-two-stage``. The pipeline page walks from sequencing +reads to a posterior, and the CLI reference lists every ``tfs-*`` command and flag. Installation ------------ @@ -35,3 +42,7 @@ Clone the repository and install in editable mode:: git clone https://github.com/harmslab/tfscreen cd tfscreen pip install -e . + +On a Linux machine with a CUDA 12 GPU, use ``pip install -e ".[cuda12]"`` +(plain ``jax`` from pip runs on the CPU only). tfscreen needs Python 3.12 or +newer. diff --git a/docs/badges/coverage-badge.svg b/docs/badges/coverage-badge.svg deleted file mode 100644 index 7d57d56b..00000000 --- a/docs/badges/coverage-badge.svg +++ /dev/null @@ -1 +0,0 @@ -coverage: 96.16%coverage96.16% \ No newline at end of file diff --git a/docs/badges/tests-badge.svg b/docs/badges/tests-badge.svg deleted file mode 100644 index 934192a8..00000000 --- a/docs/badges/tests-badge.svg +++ /dev/null @@ -1 +0,0 @@ -tests: 4420tests4420 \ No newline at end of file diff --git a/docs/manuscript/2026-04-22_methods-draft.md b/docs/manuscript/2026-04-22_methods-draft.md new file mode 100644 index 00000000..f01c50d9 --- /dev/null +++ b/docs/manuscript/2026-04-22_methods-draft.md @@ -0,0 +1,102 @@ +# Computational Methods (Rough Draft) + +## Overview + +We measured ln colony-forming units (ln CFU) as a function of IPTG concentration under two selective conditions. We aimed to use these data to infer the fractional occupancy (θ) of LacI bound to the operator upstream of both reporter genes. The full model comprised a three-level stack: a biophysical Hill model mapped binding parameters to fractional occupancy; a multiple-transformation correction adjusted that occupancy for plasmid copy number heterogeneity; and a linking function mapped corrected occupancy to observed ln CFU over time. + +## Selective Conditions + +We used two selection schemes to jointly constrain the occupancy inference. The first used *pheS* (4-chlorophenylalanine sensitivity) to measure **repression**: cells grew best when the LacI repressor was bound to the operator, blocking expression of the *pheS* susceptibility gene. The second used kanamycin resistance (*kanR*) to measure **induction**: cells grew best when the repressor was displaced from the operator, permitting KanR expression. Because these two conditions responded in opposite directions to occupancy, fitting them simultaneously allowed robust inference of θ across the dynamic range of repressor function. + +## Data + +We jointly fit two types of data. The first were high-throughput growth measurements: 209,000 mutant cycles covering approximately 210,000 distinct genotypes, with ln CFU measured across a structured set of conditions. Each genotype yielded up to 60 observations per replicate. We observed each genotype at 8 IPTG concentrations between 0 and 1 mM using each of the two reporters (kanR and pheS) under appropriate selection (kanamycin and 4CP). We also included 4 unselected controls (0 mM and 1 mM IPTG paired with each reporter in the absence of selection agent, allowing use to measure pre-selection growth rates). Together, this yielded 20 total conditions, each measured at 3 time points. We performed the full experiment in biological duplicate, yielding 120 observations per genotype. + +The second data type was low-throughput *in vitro* binding measurements: fractional occupancy of selected LacI variants binding to the same operator sequence as we used *in vivo*, measured by fluorescence anisotropy in at least biological triplicate. We used the standard deviation across replicates the uncertainty on each θ measurement. We used these data in the model to anchor the Hill model parameters to physical binding affinities. + +To assess inference quality, we spiked four additional genotypes with known binding properties into the library: the double mutants M42I/H74A, M42I/K84L, and H74A/K84L, and the triple mutant M42I/H74A/K84L. We did not provide their independently measured θ values to the model, allowing us to compare the inferred Hill parameters against ground-truth measurements as a post-hoc validation. + +## Sequencing and CFU Computation + +We derived the high-throughput growth data from deep sequencing of pooled library samples. We performed sequencing at the University of Oregon GC3F facility using 150 bp paired-end reads on the NovaSeq platform, yielding approximately 2.5 billion read pairs per biological replicate. We multiplexed samples using barcoding, and the sequencing facility demultiplexed them prior to analysis. + +We performed read assignment using tfscreen, mapping reads to the *lacI* reference sequence via a direct sequencing strategy. We assigned a base position an "n" if its PHRED quality score was below 15. For positions in the center of the gene with overlapping forward and reverse reads, we called bases hierarchically: if reads agreed, we used their consensus; if both reads disagreed and neither was "n", we assigned "n" to both; if one read had a non-n call and the other was "n", we assigned the non-n call. We matched reads against the known library sequences, allowing up to one base mismatch with an expected sequence, and discarded reads equidistant from two or more library members as ambiguous. + +We converted reads to per-genotype frequencies within each sample (defined by IPTG concentration, selection condition, biological replicate, and time point). We retained a genotype for analysis only if we observed it more than 15 times across all 60 conditions within a biological replicate, ensuring sufficient coverage for reliable frequency estimation. We added a pseudocount of one to all read counts for numerical stability. To convert frequencies to absolute CFU, we estimated the total CFU for each sample from OD$_{600}$ measurements taken in biological triplicate and averaged across replicates. We converted OD$_{600}$ to CFU using a second-order polynomial calibration curve derived from 25 carefully measured samples spanning the relevant density range, obtaining colony counts by serial dilution plating. We computed absolute per-genotype CFU as the product of the genotype frequency and the total CFU for that sample, and log-transformed the result to give ln CFU. + +We propagated uncertainty on ln CFU from two independent sources: counting noise in the sequencing frequencies and variability in the total CFU estimate across triplicates. Because ln CFU = ln(frequency) + ln(sample CFU), the variance on ln CFU is the sum of the relative variances of its two components (delta method): + +$$\sigma^2_{\ln \mathrm{CFU}} = \frac{\sigma^2_f}{f^2} + \frac{\sigma^2_C}{C^2}$$ + +The frequency variance $\sigma^2_f = f(1-f)/N$ followed from the binomial distribution, where $f$ was the estimated frequency and $N$ was the total read count for that sample. The CFU variance $\sigma^2_C$ was the squared standard deviation of the total CFU across the three biological replicates. We propagated this combined $\sigma^2_{\ln \mathrm{CFU}}$ into the likelihood as the scale parameter on the growth model distribution. + +## Biophysical Model + +We modeled fractional occupancy using a scaled Hill equation: + +$$\theta(\mathrm{[IPTG]}) = \theta_\mathrm{min} + \Delta\theta \cdot \frac{[\mathrm{IPTG}]^n}{[\mathrm{IPTG}]^n + K^n}$$ + +where $\Delta\theta = \theta_\mathrm{max} - \theta_\mathrm{min}$ was the change in occupancy from zero to saturating IPTG, $K$ was the binding constant, and $n$ was the Hill coefficient. We inferred the four parameters ($K$, $n$, $\theta_\mathrm{min}$, $\theta_\mathrm{max}$) in the hierarchical Bayesian framework described below. + +## Multiple-Transformation Correction + +Before passing the Hill-model occupancy θ to the linking function, we applied a correction for plasmid copy number heterogeneity. In a high-throughput transformation, most cells take up a single plasmid copy, but a minority acquire two or more. We empirically characterized this distribution and found it followed a zero-truncated Poisson with parameter λ (predominantly one copy, rarely two, vanishingly rarely three). + +Because LacI is a repressor, alleles with higher operator affinity will exhibit a dominant phenotype when present alongside a lower-affinity allele in the same cell. Multiple transformation thus confounded our inference: if a given genotype's frequency changed over the course of selection, it could reflect its own fitness or the fact that it was co-transformed with a dominant plasmid. This effect systematically biased our estimates of low occupancies ($\theta \rightarrow 0$), because the lower the value of $\theta$, the higher the probability that it was masked by a co-transformed variant with higher affinity. + +We applied the correction as follows. At each iteration of the model, we built an empirical distribution of θ values across all genotypes in the current batch. For each genotype's θ, we computed the probability — given λ and this empirical distribution — that an observed cell carried a second, distinct plasmid whose θ masked the genotype's own fitness. Genotypes with high θ had a low probability of being masked; those with low θ were more susceptible. We computed the corrected θ by integrating over this sampling probability. [*Equation to be added.*] + +We parameterized the correction with a single λ, using a prior informed by an independent empirical measurement of co-transformation rates. To measure λ, we transformed bacteria with an equimolar mixture of 8 plasmids that differed slightly in size but shared a single restriction site. We extracted plasmids from 107 individual colonies, digested them with the shared restriction enzyme, and resolved the products by gel electrophoresis; the number of bands per lane directly reported the plasmid copy number for that colony. We fit a zero-truncated Poisson to these counts by maximum likelihood, yielding λ = 0.4 (range 0.1–0.6). Zero-truncation was necessary because cells that failed to take up any plasmid were lethal under selection and therefore unobservable. We used the resulting estimate and its uncertainty as the prior on λ in the full model. + +## Linking Function + +We defined a *linking function* that predicted ln CFU as a function of time given the corrected θ. The core assumption--borne out experimentally--was that growth rate scaled linearly with θ. Cells experienced two sequential growth phases: a pre-selection phase (duration $t_\mathrm{pre}$) followed by a selective phase measured at multiple durations $t_\mathrm{sel}$. The full model was: + +$$\ln \mathrm{CFU} = \ln \mathrm{CFU}_0 + \bigl(\theta \cdot m_\mathrm{pre} + b_\mathrm{pre} + \Delta k_\mathrm{geno}\bigr)\, (t_\mathrm{pre} + \tau) + \bigl(\theta \cdot m_\mathrm{sel} + b_\mathrm{sel} + \Delta k_\mathrm{geno}\bigr)\,(t_\mathrm{sel} - \tau)$$ + +The parameters were: + +- $\ln \mathrm{CFU}_0$: initial colony count for a given genotype +- $\theta \in [0, 1]$: corrected fractional occupancy of the operator (0 = fully unoccupied, 1 = fully occupied) +- $m_\mathrm{pre}$, $b_\mathrm{pre}$: slope and intercept mapping θ to growth rate during pre-selection +- $m_\mathrm{sel}$, $b_\mathrm{sel}$: analogous slope and intercept during selection +- $\Delta k_\mathrm{geno}$: pleiotropic fitness effect of a mutation, independent of its effect on operator occupancy (for example, a destabilizing mutation may impose a metabolic cost from misfolded protein regardless of its DNA-binding phenotype) +- $\tau$: lag parameter describing the delay in growth rate adjustment when transitioning from pre-selection to selective conditions (see below) + +## Lag Model + +We evaluated three functional forms for the lag: an instantaneous transition model (no lag), a model with a sigmoidal transition integrated over time, and a memory model in which the lag depended on current occupancy. We selected the model based on posterior predictive checks, with visual inspection confirming a clear winner. The memory model performed substantially better than the alternatives [*quantitative comparison to be added*]: + +$$\tau = \tau_0 + \frac{k_1}{k_2 + \theta}$$ + +When θ was large (repressor tightly bound, downstream gene expression suppressed), the denominator was large and the lag was shorter — the cell required little time to adjust because the gene product was already scarce. When θ was small (operator largely unoccupied, gene product abundant), the lag was longer because the existing product had to be diluted or turned over before growth under the new condition stabilized. + +## Parameter Inference + +### Likelihood + +We evaluated two likelihood terms simultaneously. We fit the growth data with a Student-t likelihood, which provided robustness to occasional outliers; an equivalent Normal likelihood yielded similar results. We fit the in vitro fluorescence anisotropy binding data with a Normal likelihood. In both cases, we set the observation variance to the experimental uncertainty in the $\ln \mathrm{CFU}$ and $\theta$ values. The tight experimental uncertainties on the in vitro measurements produced a sharp likelihood that effectively pinned the Hill model parameters to physically measured binding affinities without requiring those measurements to function as priors. + +### Prior structure + +Model parameters fell into two categories. Parameters that appeared once across the entire dataset — the linking function growth terms ($m_\mathrm{pre}$, $b_\mathrm{pre}$, $m_\mathrm{sel}$, $b_\mathrm{sel}$), the lag terms ($\tau_0$, $k_1$, $k_2$), and the co-transformation rate (λ) — we assigned simple, weakly informative priors. We additionally informed the prior on λ using the externally measured co-transformation rates described above. + +Parameters that varied across genotypes — $\Delta k_\mathrm{geno}$ and the Hill model parameters ($K$, $n$, $\theta_\mathrm{min}$, $\theta_\mathrm{max}$) — we assigned hierarchical priors, with separate hyperparameter distributions learned for single mutants and double mutants. This distinction reflected the experimental design: we spiked single mutants into the library at higher initial frequencies than double mutants, so the two classes were expected to have different $\ln \mathrm{CFU}_0$ distributions. For all hierarchical parameters, we drew per-genotype values from a Normal distribution whose location and scale were themselves treated as random variables: we placed a Normal hyperprior on the location and a HalfNormal hyperprior on the scale, and learned these hyperparameters jointly from all genotypes within each class. + +We assigned wild type and the four reference genotypes (WT, M42I, H74A, K84L) — spiked in at substantially higher concentrations than the bulk library — individual simple priors on $\ln \mathrm{CFU}_0$ rather than drawing them from either hierarchical class. + +### Fitting procedure + +We performed MAP optimization using NumPyro's AutoDelta guide in two steps: first, a constrained fit in which we optimized only growth parameters (τ and the pre-selection slope) while holding all other parameters at simple initial values; then an unconstrained MAP over all parameters simultaneously. This initialization strategy provided stable starting points for the stochastic phase. + +We then ran SVI using NumPyro's clipped Adam optimizer with an exponentially decaying step size schedule, starting at $10^{-3}$ and decaying to $10^{-6}$. Each SVI step used 2 ELBO particles and operated on a minibatch of 1,024 genotypes: 1,020 drawn from the library and the 4 reference genotypes included in every batch. Because we oversampled the reference genotypes relative to their true library frequency, we used JAX's built-in importance weighting to correct their contribution to the ELBO. + +We declared convergence within a run when the mean ELBO changed by less than $10^{-6}$ over a sliding window of 100 epochs, sustained for at least 10 consecutive windows (patience counter). [*Total epoch counts to be added.*] We assessed convergence across runs by comparing posterior parameter estimates across multiple random seeds. [*Cross-seed convergence thresholds to be added.*] + +### Outputs + +After SVI, we extracted posterior distributions for each model parameter and derived point estimates and 95% credibility intervals. The primary outputs reported per genotype were the Hill model parameters: binding constant $K$, Hill coefficient $n$, unoccupied-state baseline $\theta_\mathrm{min}$, and bound-state baseline $\theta_\mathrm{max}$. + +## Implementation + +We implemented the model in Python using the standard scientific computing stack. We performed probabilistic modeling and inference with NumPyro and ran all calculations on NVIDIA A100 GPUs via the University of Oregon TALAPAS high-performance computing cluster. The implementation is available as the open-source package **tfscreen** at https://github.com/harmslab/tfscreen. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 10.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 10.txt new file mode 100644 index 00000000..330416e2 --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 10.txt @@ -0,0 +1,7 @@ +Actually one more clarification. We used simple priors for log CFU0 for wild type +and for spiked genotypes and then we used hierarchical priors in two +different classes. One set of hierarchical priors for single mutants and +a separate set of hierarchical priors for double mutants. That's +because the single mutants by design had a higher initial frequency in the +library than the double mutants so we wanted to capture that in our main +analysis. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 11.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 11.txt new file mode 100644 index 00000000..8170f3c5 --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 11.txt @@ -0,0 +1,30 @@ +So let's add a few more experimental details. So another detail is that we +spiked in an additional four genotypes that we did not include measured values +of theta four, but we did have the measured theta so we can basically check +the quality of our inference by asking whether or not these values had good +estimates. So those genotypes that we included with measured theta but we +didn't include it in the fit were double mutants of M42I H74A, H74I M40, sorry, H74A M42I, sorry, let's +start with this. So we've got M42I H74A, M42I K84L, and then H74A K84L, and then the +triple mutant with all three. So those were included as spiked samples where +we knew their binding constants or their K values but we didn't actually put them +into the estimate. Another control that we did was making sure that we measured +lambda with some frequency and the way that we did that is we transformed +bacteria with eight different plasmids that differed slightly in size but all +shared a single shared restriction enzyme. We then transformed using exactly the +same transformation protocol that we did by even mixture of these plasmids, +extracted plasmids from single colonies, digested the resulting plasmids using the +restriction enzyme, and then random on a gel allowing us to just count how many +plasmids were present by the number of bands that came out on the gel. We did +this for 107 different transformations allowing us to estimate our lambda. We +did that estimate using a maximum likelihood fit of a truncated Poisson +distribution to these values, these data, allowing us to estimate both the lambda +which ended up being 0.4 and the uncertainty on the lambda. So lambda +basically had ranged from 0.1 up to about 0.6. So we used those as priors, +those values from the experiment as priors on our lambda that we +estimated from the actual experiment itself. We had to use zero truncated +Poisson because we're not able to observe cells that don't pick up plasmids, +right? So if a cell doesn't pick up a plasmid, it would die in our selection. +And further, there's a whole bunch of reasons that a cell would have zero +plasmids. So it's actually two different processes. So all we can see are cells +that pick up one, two, three, etc. or more plasmids. Okay, so that's information +about different genotypes about the lambda. I think that's all I'm going to include in this book. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 3.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 3.txt new file mode 100644 index 00000000..7b808106 --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 3.txt @@ -0,0 +1,50 @@ +So I'm thinking through how to write up the computational methods section for an upcoming +paper we're writing about the lack repressor and lack repressor high throughput screen. +So I'm going to just give some random texts, thinking out loud, blathering, talking, and +then let's turn this into a rough draft of the method. +So the output from our experiment is a collection of measured log CFU as a function of IPTG +concentration and as a function of different selective conditions. +Our two selective conditions are phi s and canamycin. +The phi s condition is scans, measures repression because cells will grow best when the lack +repressor is bound to the operator and the canamycin selector measures induction because +cells will grow best when they're inducing the repressor or they're making the repressor +come off the operator. +The goal is to use this combined data to infer the fractional occupancy of the transcription +factor bound to the operator that's in front of both can and sorry can r and phi s. +The way that we go about this is by defining a linking function. +The linking function maps allows us to predict the fractional sorry the log CFU as a function +of time given the occupancy of the transcription factor. +So the core idea of the linking function is that it consists of a map between growth rate +and fractional occupancy that can then be fit using a hierarchical Bayesian approach. +And so overall the math looks like this, we'll see how good the math transcription is here +for Claude, it's the log CFU goes like the log CFU not so the initial CFU for that genotype +plus parentheses, the fractional occupancy which is theta times the slope plus the intercept +plus delta k geno that whole block is in parentheses times the time for the precondition plus tau +at another term that's going to be theta times the slope under selection plus the intercept +under selection plus delta k geno again and parentheses on that and then parentheses +t selection minus tau and parentheses. +This is our core linking model, you'll notice there are theta terms, those are the fractional +occupancies that map to the growth rate in a linear sense. +So basically the slope times theta is how much does increasing theta between zero and +one change the growth rate and then the intercept terms here are just the growth rate when the +operator is completely unoccupied. +Finally delta k geno is the pleiotropic effect of mutations, so if a mutation occurs how +does that affect growth rate independent of fractional occupancy, this captures the fact +that say a mutation that causes the protein to become destabilized might incur a metabolic +fitness effect just because it's a poorly folded protein. +The tau term is the latency or the lag when we're going from our pre-growth conditions +to our selective conditions, sorry call them our pre-selection conditions to our pre-growth +conditions, well sorry let's try that again. +T pre is the time that our cells spend in the pre-selection conditions, T cell is the +amount of time the protein spent in our selective conditions, we're going to measure the log +CFU at different T cell. +The tau term describes the lag in the growth rate going from the pre-condition to the selective +condition. +We tried a number of different models for tau here, we ended up with the following that +tau, the lag, equals tau naught plus k1 over parenthesis k2 plus theta in parenthesis, +so this tau is basically saying that the amount of time that it takes to lag actually depends +on the occupancy, so if something is very tightly occupied, it is repressing expression, +the change will be relatively rapid because you have a larger number on the bottom, whereas +if tau is very zero, it's going to be as fast as possible because there's a large amount +of the product that was expressed. +So that defines the whole linking function, letting us link theta to log CFU. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 4.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 4.txt new file mode 100644 index 00000000..a8bba46c --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 4.txt @@ -0,0 +1,99 @@ +The next part of the model is the biophysical model for transcription vector binding. +So for this what we're doing is we take the standard hill model with a binding constant, +a cooperativity parameter, a bound baseline parameter, and an unoccupied baseline parameter, +and we fit that to each of the fractional occupancy data. +We do this in a hierarchical Bayesian context where we are inferring the prior distributions +for all four of these parameters from the data together. +The last piece of the mapping for the total function is actually a correction that occurs +between that biophysical fractional occupancy calculated with the hill model and the theta +that we feed into our linking function. +That's actually a correction for multiple transformations. +So one of the challenges in a high-throughbrit screen like this is that when we transform +bacteria, most of them pick up one copy of a plasmid, but some pick up more than one. +We empirically measure this and found that our cells pick up a zero truncated Poisson +distribution, pick up most of the time one, a very small fraction of the time two, and +even a smaller fraction of the time three plasmids. +What this correction parameter does is it actually captures the fact that when we look +at the growth rate of a cell or the frequency of a different genotypes going up in the population, +it could be going up because it, in fact, is higher fitness or because that plasmid was +co-transformed with another plasmid that has high fitness, because the amount of protein +in each cell is higher than the amount of DNA, and because this is a repressor, things +that have a higher binding affinity will always win the exhibit a dominance phenotype. +And so what we need to be able to do is correct for that dominance by asking when a frequency +of a gene is going up, is it going up because it's being dragged out by another gene, and +is it going down because it's being dragged down by another gene. +So to correct for this, we have a sampling from the, we take each theta and then we ask +what's the probability that it's sampling from another theta, and we use that to effectively +correct that theta. +So then if we stack all of these models together, we'll start from the bottom, we have a biophysical +model of transcription factor occupancy with things like binding constants and hill coefficients. +Those are things we want to learn. +We take those values, we run them, and we're sampling hierarchically from their priors. +We then take that theta, that occupancy, and we feed it through a correction that accounts +for multiple transformation. +In this case, it has a single parameter lambda that we learned from all of our data all at +once with a prior that we set from externally measured co-transformation lambda, so it puts +on control parameter. +And then we take this corrected theta and we feed it through our linking function. +We then learn those parameters to describe the whole data. +So that's the entire model of stack. +Let's go through the priors that are on each of these model parameters, and so for the +linking function, the growth parameters, so the slope in the precondition, the intercept +in the precondition, the slope in the selection condition, and the growth in the selection +condition, these terms are all used simple priors. +That is because they only occur once, there's one across the whole data set, so there's +no way to learn a collection, learn a hyper, sorry, no way to learn parameters for some +hyperdistribution from this, right? +It's just a single simple parameter. +Likewise, this is true for the tau terms, tau naught, k1, and k2. +These use simple priors because they are single terms that are used for the entire data set. +If we look at the correction for multiple transformations, again we have a single simple +prior on lambda because there's one lambda for the entire data set, so we set that prior +and then go from there. +But then pretty much everything else uses a hierarchical priors situation where we're +learning the prior distribution from the data. +So we set a broad hyper-prior distribution and then we learn the actual distribution +from the data. +So the terms that are used with this kind of prior approach are the delta k genome, +the log CFU naught, the hill coefficient parameters, so k and then the baseline parameters +for that all use a hierarchical Bayesian model. +One last note on the nature of the priors, there's a special case. +We have several genotypes that we spike in at high concentrations. +These include wild type as well as a set of controls where we know what their binding +measurements are. +These spiked genotypes actually use a standard single prior for log CFU naught, their initial +concentration, because their initial concentrations are much different than the rest of the population +and at the same time have their own values, so they're sampled from their own simple priors. +So what kinds of data do we do our fitting on? +So this is this big complicated model, how do we learn it? +So we feed in two types of data into the model. +The first are the data that I already mentioned, these are growth data that we collect in high +throughput. +For our experiment we have 209 mutant cycles corresponding to, I think we get a number, +but something like 210,000 different genotypes that we measure growth as a function of time +in different IPTG, canamison, and 4CP concentrations. +We then take those data and we complement it with some low throughput data which we used +to measure bindings. +We measured the fractional occupancy of different genotypes of the lacropressor to the same +operator in vitro using fluorescence and isotropy. +We then put both of those data into the hierarchical model and then attempt to find parameters or +sample parameters that line up with this. +So how do we actually do that sampling process? +We use a two-stage process. +We first start with a simple maximum aposterior I run, where we run a quick optimization to +get good guesses for the initial parameters. +We do this actually in two stages. +We do one fix, fit where we fit our growth parameters, things like tau and the pre-selection +slope. +So we do a map run on those, pinning every other parameter and fitting a very simple +binding model to everything else, sorry, a simple model, and the idea there is to really +get good initial guesses for those terms. +We then do unconstrained map on all parameters at once to move away from whatever our starting +guesses are, to much better starting guesses. +And then we follow that up with stochastic variance, sorry, stochastic variational inference +SVI, which we then run until we see convergence for multiple things. +We do this multiple times with different seeds to make sure that we see convergent results +on our final outputs. +But at the end of the day, the key thing that we're pulling out are these hill coefficients +K, N, and then the baselines, how much is bound in the absence of IPTG, how much is +bound in the presence of IPTG. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 5.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 5.txt new file mode 100644 index 00000000..44b6978b --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 5.txt @@ -0,0 +1,10 @@ +I'm now going to talk about some of the computational architecture that we used for this. +So we implemented our model inside of a Python framework using the standard scientific computing +stack. +The model itself was implemented in Jack's NumPyro with all of the tools that come with +that. +We ran all of our calculations on NVIDIA A100 GPUs that were available in the University +of Oregon TALAPAS computing cluster. +I think that pretty much covers sort of the Python sort of computational architecture +there. +Yeah, I guess that's actually about it. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 6.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 6.txt new file mode 100644 index 00000000..dbfa26a7 --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 6.txt @@ -0,0 +1,27 @@ +Some other thoughts about what to mention in here. +We actually ended up measuring 60 observations in our, sorry, in our experimental pipeline. +We get 60 different observations for each genotype. +So that is because we're covering eight different IPTG concentrations with selection, with canomycin +or with 4CP. +So that's 16. +We then measure four different controls. +So zero millimolar IPTG and one millimolar IPTG with the fee S, excuse me, with the fee +S or the can R in the absence of selection. +So those are basically getting the pre-growth rates or the growth rates without actually +adding selection. +So that brings us to 20. +And then we measure three different time points for each one. +So as the cells progressed over time, we get three observations. +So that gives us 60 different condition observations. +We then pulled out genotype frequency information for each of those. +So each genotype has 60 possible observations. +We then did the entire experiment in biological duplicate. +So now we have two sets of independent measurements, giving us 120 observations per genotype. +That we are trying to ultimately use to constrain four parameters, the hill coefficient, K, +N, and then the baseline betas. +So that's what going is experiment. +Let's also talk about how we pulled parameters out. +So after we do the SVI runs, what we do is we then extract posterior distributions for +each of the different parameters. +Pull from those posterior distributions to get credibility intervals for each of these. +And so that gives us a 95% credibility region for the binding constant for each of those. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 7.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 7.txt new file mode 100644 index 00000000..c402f131 --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 7.txt @@ -0,0 +1,4 @@ +I guess we should also note that this implementation of our library, TF screen, that is the letter +T, the letter F, S-C-R-E-E-N, lowercase, we implemented TF screen and it is available +on github at HTTPS github.com slash harm's lab slash TF screen so anybody can use it. +I think that's all that's important with that. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 8.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 8.txt new file mode 100644 index 00000000..90f783ca --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 8.txt @@ -0,0 +1,105 @@ +All right, so let's go through these questions that we need to add. +So what is the Hill model equation? +The Hill model equation is the theta with zero IPTG plus delta theta, that's the change +in theta upon binding, upon going to fully saturating IPTG. +That's multiplied by the IPTG concentration raised to the N over the IPTG concentration +raised to the N plus the equilibrium constant raised to the N, where N is the hill coefficient. +The multiple transformation correction math, I'm going to skip for now, but I'll get to +that in a second. +I'm going to add a block that just says there's some math to be done here, let me know. +For the likelihood and noise model, we actually use the student T probability distribution +to account for some possible outliers. +We also actually tried it with normal distribution and saw no huge difference between the two. +We did not actually add specifically, in that we estimated the variance on the distribution +directly. +We did not add any specific noise. +We have our deterministic model of binding, which we then use to attempt to deterministically +reproduce the binding and the growth data without actually adding a specific noise model +to that. +We just infer the variance on the probability distribution as our noise model. +For the lag model selection, we basically did posterior predictive checks formally, +but visually it was very clear that this was the best of the models that we tried. +The models that we tried were an instantaneous transition model, so assuming no lag, a baryony +transition model, which basically has a integrated sigmoidal curve that gives rise to the amount +of time that it takes for a transition. +Then the memory model that we described, that was the best one by far. +I can add a note, actually, to the text to say we're going to add some numbers to support +that, but for now we're just going to say we did the posterior predictive and visual +inspection checks. +How are the logs, if you can compute it from sign sequencing data, yes, let's add that +to the top. +The way that this is done is we did noviceek high throughput sequencing, which gave us +approximately 2.5 billion reads per bioreplicate using barcoding to be able to pool samples +into single reads. +We demultiplexed those reads using standard, actually I'll describe the reads really quickly, +so we did 150 base pair paired end reads on noviceek, so that's 2.5 billion paired end +reads per run. +We demultiplexed from our sequencing core itself, all sequencing was done at the UOGC3F, +that is University of Oregon Genome, actually I don't know what I was saying before, but +it's GC3F Gamma Charlie 3 Foxtrot, the UOGC3F facility, so they ran our sequencing for us +and then demultiplexed. +Once we had the demultiplexed reads, we then assigned reads using the TF screen library. +We used a direct sequencing strategy here, so we were asking could we map our reads back +onto our reference gene, the lacropressor gene, and also ask if we could do base calling. +We did the base calling in a sort of multi-level strategy. +We basically took every base in the read and we looked at its Q score and a Q score less +than 15 was assigned to an N at that position. +We then asked whether this was compatible with a unique gene in our library, so we specified +what was in our library to begin with and asked could we match it to that library to +do our read calling, and we basically allowed no more than one mismatch with the existing +library and if there was ambiguity, meaning that it was one base away from more than one +library, remember we threw the read out, so that was our primary quality control. +We then took this collection of reads and on a per genotype basis asked if we saw that +genotype more than 15 times within a bio-replicant across all of the different read conditions. +This is basically asking whether we truly saw this genotype somewhere in the library. +Once we had that, that gave us a raw count per position, for a raw count per genotype +per condition. +We then converted these counts into frequencies, so within a given population, so a given IPDG +concentration, bio-replicant, pre-selection condition, and post-selection vision that +we could uniquely define each tube that it came from effectively. +We then calculated the frequency of each gene using a pseudo-count strategy. +We added a pseudo-count of one for numerical stability. +We then took these frequencies and multiplied the frequencies by the overall CFU port for +that condition. +We got the CFU for that condition by actually doing the experiment in a biological triplicate +and measuring the OD600 at each bioreplicant and then taking the average across the bioreplicates +to give us an estimate for how the OD600 changed as a function of time in these samples. +We converted the OD600 into a log CFU using a careful plate calibration and polynomial +fitting strategy where we measured OD600 very closely for 25 different conditions, and then +we did serial dilution plating to measure how that mapped to colony forming units. +That allowed us to define a second-order polynomial that allowed us to map from the OD600 to the +CFUs. +Now we have the total CFUs for each tube, and we have the frequency of each genotype +in that. +We multiply those numbers together to get us the absolute CFU for that genotype under +those conditions. +Finally, the last step is to do log conversion to convert those into log CFU as a function +of time. +Then with the In vivo binding data, how did they enter the likelihood? +The way that this worked is we actually had two observations, so we had two likelihood +terms. +One likelihood term using a student T for the growth data. +Our second likelihood term is a normal distribution for the binding curves, both go in at the +same time. +We did not use these to set priors. +The priors were previously existing. +We did have those. +We had very tight uncertainties on these values experimentally, so the likelihood function +was quite sharp around the correct theta for each of these in vitro binding data. +Another feature of this is that we made sure, since we're using SVI, we're having to sample, +we used a batch size of 1024. +We found empirically that that gave us the best outcomes. +We're looking at 1,024 genotypes per run. +We also included the genotypes which measured theta in every single batch. +Actually, I'll say what those genotypes are. +The genotypes were wild type, M42I, H74A, K84L. +We included those in every single batch, those four genotypes. +We had 1,020 library genotypes and these four in every single genotype. +Then we used JAX's internal tools to weight them correctly so that we got a correct likelihood +weight. +We were basically oversampling on these genotypes so we make sure they have a lower weight in +the final analysis. +Then I'll get to the SVI conversion criteria later, but at some level the primary convergence +criteria was an elbow plateau and then making sure that we saw parameter stability across +different parameter seeds. +Those were the two major mechanisms we used. diff --git a/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 9.txt b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 9.txt new file mode 100644 index 00000000..45ef45c9 --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-22_University of Oregon 9.txt @@ -0,0 +1,31 @@ +All right some quick notes on the transformation correction which I kind of +left as vague before. What we did is we empirically built a distribution of +Thetas at every iteration of the model and then integrated this set of theta +values. We then sampled from that distribution and so we could say all +right for a given theta that we have what's the probability given the lambda +and this empirical distribution what's the probability that my particular theta +is being masked by another one. So if theta is very very high for example the +probability of being masked is very low. If theta is very very low the +probability of being masked is higher. Conditioned of course on that lambda the +Poisson parameter there we go brain is falling apart here. Okay so that's how we +did that there I'm actually gonna have to pull up the equation later which I +don't have so again put a pin in that saying we have to add that that data to +that. The other thing I wanted to add into this sort of extra bit is some of +the parameters that we used in the actual SVI run so we use the atom +optimizer which is built into +sorry we used the clipped atom optimizer for SVI that is built into NumPyro and +we use the auto delta optimizer for our map estimates. When we did our run we +used a exponential decay in our step sizes we started with an atom step size +of one times ten to the minus three and reached to a one time ten to the minus +six step size. We did this with two elbow particles and did this for some +number of epochs. I actually don't have the number of epochs on me. We used a +convergence criteria for having the elbow score changed by less than ten to +the minus six over a sliding window of a hundred epochs so we could pair the +first hundred epochs the average elbow and the first hundred epochs the average +elbow and the next set of hundred epochs. We then asked did they both or they +both was the difference between them less than ten to the minus six and then +we did this actually with a patience counterbase. We had to see that for at +least ten times in a row before we decided that the system was converged so +that was looking for convergence during each run. We then looked for convergence +the level of the posterior parameters and I'm actually gonna have to give you the +numbers for that later so we're gonna leave that now off that now. diff --git a/docs/manuscript/verbal-transcripts/2026-04-23_predict-efffecs-mutation-protein.txt b/docs/manuscript/verbal-transcripts/2026-04-23_predict-efffecs-mutation-protein.txt new file mode 100644 index 00000000..9e68a892 --- /dev/null +++ b/docs/manuscript/verbal-transcripts/2026-04-23_predict-efffecs-mutation-protein.txt @@ -0,0 +1,5 @@ +So what would it take to predict the effects of mutations on protein function? There are many tools that exist that can predict, say, a mutation that disrupts protein function in their tools that exist that, say, give me sequences that are compatible with a given protein structure. And there are even tools that, for example, can try to map binding to particular interaction. can predict how mutation changes a particular binding interaction. But when you think about protein function, it's a broader phenomenon. A good example of this would be a transcription factor, that binds to DNA and either recruits or inhibits the recruitment of transcriptional machinery, in response to chemical signals. So it has to we have to talk about its function, we have to integrate minimally how good is it at bonding to the small molecule that affects its binding activity to the DNA? And how old does it bind to the DNA itself? Um, if we think about tools that exist to do that, uh, we might imagine uh, training some kind of machine learning model on a bunch of uh, geentific variants of a protein. And what that might allow us to do is make some predictions. If I introduce a mutation at this position, it changes the function in this way. But there are a couple of real problems with that. Uh, the 2 major ones are that often the output of a hydrobit screen is a single number for a protein. So what we're training on isn't a function exactly. We're training on some proxy for function, so maybe, uh, fitness and bacterium or protein stability. And the second problem is that we can never build sequence libraries that are big enough to actually do the sampling. The sequence libraries are really stuck with single mutations and a smattering sample of combined mutations. If mutations just added up, could some additively to give function without any epistasis, this wouldn't be a problem. But of course, epistasis is everywhere. The effect of one mutation changes the effect of another. So, even if we train an amazing model that allows us to predict the effects of single mutations, based on some high rubid data, as soon as we start combining mutations, the ability of that model to function is going to decay. We think we're in a really good place to take a next step towards actualizing the dream of building a model that protects protein function. And we can do that by leveraging existing powerful tools. And specifically, we can do this for transcription factors. So, the reason transcription factors are an attractive starting point is, first of all, that their function can be described in terms of protein stability. What do I mean by that? Well, if we think about a transcription factor as a collection of different confirmations that are all exchanging with one another, and let's make this specific. Let's imagine that this is a repressor that binds to DNA in the absence of its ligand. You have a collection of different confirmations. You've got the bound to DNA form in the repress state. The off of DNA in unrepressed state without liggin bound. And then maybe you have an effector bound confirmation. It's not bound to DNA, and you could even formally have the other combinations, right? There's an innumerable set of protein confirmations that you could use to describe the function of this protein. Or to model the function. And then when you think about the function itself, let's say that's the occupancy of the transcription factor binding site on the genome, the operator, as a function of its allosteric effector. You can calculate that, right? If you know the structures and you know the equilibrium constants, sorry, I guess you don't need instructions. But if you know the equilibrium constants for the system and you know the concentrations of the components, you can then calculate the function, given that information. So, what we could put this down to is what we really are trying to learn for a transcription factor system is what are the equilibrium constants describing the system? How do mutations alter those equilibrium constants? Now, we could just think about these as terms to learn, just trying to learn the equilibrium constants. Um, and, uh, when we start adding up mutations, we're adding a plug, equilibrium constants, uh, maybe we have to add some epistasis in there, but we're trying to think about it at that term. But I would argue that that's not the right level of granularity thing about this problem. And the reason for that is twofold. The 1st is that an equilibrium constant folds together multiple things. I'll talk about in a second. And the 2nd is if we just have a naked equilibrium constant, it's difficult to bring to bear other information on that, right? Um, Let's talk about, to get to both of those. Let's think about what an equilibrium constant actually is. At the end of the day, an equilibrium constant could be written as E to the minus delta G not prime over RT. And what's the delta in that delta G? It's the difference in the free energy of 2 confirmations at standard state. Right? And so if we think about this in terms of, say, a binding reaction, how we often think about this, this would just be what's the difference in the energy of the unbound, the protein and the ligand unbound versus them bound together. That's a structural question, right? A structural, and there's entry involved, so you write entry points, but it's like, the, the microstate ensemble for the unbalanced state versus the balance state, what are the differences between those two? Um, If we think about a reaction, it doesn't involve binding, say, a protein is transitioning between, is, say, a high affinity DNA form and a low DNA affinity form. Now we're just talking about the literal difference in those 2 energies. That's cool, because what that means, if equilibrium constants are what we need to learn how the function, how the function changes, and those can be related to structure on one hand and function on another, we can then think about a combined experiment where we bring together, uh, information about function on one side, and we bring in information about structure on the other, to then ultimately learn these equilibrium constants to describe the system with high precision. And so let's make this concrete for the lack of pressor. So the lack of pressor exists in two different forms, a high affinity form, and a low affinity form for DNA, HD. And this can be approximated. People have shown with experimental things. We're just talking about the equilibrium population of the species, this can be approximated as basically H form, and H form bound to DNA, L form, and L form bound to its effector, allosteric vector. In this case, IPDG. This means that we have four pieces of structural information, four different confirmations we have to worry about, that we can then use to learn the output infrastructure. + +So if the task then is to connect structure to equilibrium constant in the equilibrium constant function, to function, what do we need on the structure side? So to do that, there are fantastic new machine learning tools that allow you to ask, basically, the question, is a sequence compatible with a given structure? So, thermal, MPNN, ligand, NPN are, uh, sorry, protein, MPNN, and ligand, MPNN are 2 really good examples of this. You can feed in a structure in a sequence, and get out a series of features that at some level map the difference between those two. And when we were thinking about a confirmational ensemble, like the lack of pressor or an energy landscape, we have, say, 4 different conformations that each must be compatible with the same sequence to define the function. The other thing that's been shown can be successfully done there is transfer learning, where you take the output of one of these neural networks, say, protein and BNN. And then I reweight it, put a neural network in between it and you're observable to reweight it to reproduce, say, binding free energy. So, what we have here is a clear route forward to take sequences as input and structures, sorry, structures is inputs, and map them towards equilibrium constants, right? Calculate the compatibility of a sequence with a structure, turn that into an energy for that structure, take the differences in those energies for the different structures. Those give you the equilibrium constants, within allow you to calculate the function. The question, then, is, what is the training data that goes on the other side? And you think about the problem here. If we just imagine that we measure for individual point mutes, the output of an occupancy versus binding curves, right? So on the X axis, you have IPG, and the Y axis you have occupancy. This is already a challenging measurement to do experimentally, but in principle, it can be done, and it's been done for 100s of thousands of variants, not millions, but a few thousands of variants. If we have that information for individual mutants, the question is, do we have enough information to train equilibrium poses? From a formal number of parameters perspective, maybe, right? We have a binding curve, let's say, you've got 10 data points in that curve. For each curve, you're trying to learn 4 parameters for each of the Delta Gs, which turns into 3 equilibrium constants, maybe even describe the system that way. Problem, though, is that's really actually a pretty challenging statistical problem to do. Um, because there's a degeneracy here, right? You introduce mutation and it shifts the equilibrium to the left. You don't know if it shifted the equilibrium to the left because it's stabilized to the balance state or it destabilized the unbound state, right? So a shift in equilibrium, you don't know what structural change it has. That is where a recent insight that we had actually comes into play, unless you fill in this hole in the middle. And the thing that we discovered was, we reason about, was that you can get interesting coupling between the effects of mutations, or epistasis, due to redistribution of the conformational ensemble. And so the basic idea is that if you have the observation that led to it, if you have two mutations that have no effect on their own, but together, they give a completely different effect. one explanation for that is they're having different effects on different confirmations. Really good example would be destabilizing mutations, right? Mutation one can be destabilizing, it's destabilizing the native state, but not so much that the native state isn't civil one you see. Mutation 2 could be destabilizing. It's destabilizing the native state, but not so much that you can't see it. When you put them down together, they're both destabilized native states extensively, and you reveal the unfolded state as a new ground state in the system. So, what you're seeing there is a rerating the ensemble between the folded and the unfolded state. But really, this could happen between any collection of confirmations, right? We could replace the unfolded state with a new functional form, right? So this new functional form comes about when it happens, and it's actually something we've observed experimentally over in evolution. The question, though, then, that we have is, what is it? What does that have to do with our inference? Well, if mutations can lead to coupling by redistributing the ensemble, and that leads to apostasis, sorry, if mutations that redistribute the ensemble lead to apostasis, maybe we could use apostasis to learn about the conformational ensemble, and thus learn about these equilibrium constants. And what we found, actually, we can show this in a pretty straightforward way, that there's, if you just think about the types of coupling that might happen in a system like this, there's a few different possibilities. mutations have some sort of through space, through structure coupling, that happens in one structure. That will be reflected as epistasis in the baselines when you're basically all one structure and the other. Um, Uh, you could also have epistasis that changes as a function of, uh, allosteric vector. That reflects ensemble redistribution. And so, what our premise is, is that if we measure mutant cycles as a function of IPTG, and measure fractional occupancy, we're going to be getting the data that we need to actually train the model to learn the equilibrium constants describing the system. So that, uh, we can then, the input to this would be a finally, a trained model where you put in a sequence on the one side. It runs through structural information, tracks, train tracks, and structural information to give you equilibrium constants. Those equilibrium constants can then be used to calculate function. So sequence goes to function. And what we're trying to do then is demonstrate that's true. So what are some preliminary data that we have? The preliminated it that we have is twofold. The 1st is we've developed this massive experiment, where we have measured 60 different data points for 209, sorry, 120 data points. So 2 bio replicates of 60 for 209,000 different mutant cycles inside the lack of pressor. So these are, we know that wild type effect, the effects of the individual mutations and the pairways effect. And we've measured this at 8 different IPG concentrations, under 2 selective markers, so a positive conditional selective marker, and a negative conditional selective marker. And we have a sophisticated linking function that links from bacterial growth to binding. So, we could think about this too, like, pretty good approximation. We have measured the binding profile as a function of IPDG for 209,000 variants of lack of pressure. So that's a really great piece of data, because we can look at this and we can actually pull out things like, oh, look, there's apostasis as a function of concentration. Uh, that reflects redistribution of the ensemble, for we can see things like contacts. Um, the, uh, so that's a really great piece of input. Uh, the other preliminary data that we have is, uh, we decided to, we're going to, what we're going to do in this, uh, calculation is actually adapt, uh, an approach that was pioneered by thermal MPNN, which does transfer learning to take, uh, protein of PNN and train it on thermodynamic values. We're going to do the same thing to, instead of learning equilibrium that, you're going to learn equilibrium constants as a hidden variable that's reproducing our binding measurements. But before we even did that, we said, well, let's just, this is effectively an energetic confirmational stability thing. introduce mutations, change the stability of the different confirmations, and that's from 5 there. So we ran that model and showed that we can actually reproduce our data, quantitative. It's not perfect, but with a very simple model, we can reproduce binding. So with these two pieces of information, we're now positioned to bolt this whole system together, and use transfer learning approach to reweight each of the energies here, and to take the outputs from multiple runs of ligand and PNN and actually predict function. + +So let's actually go through the computational pipeline that we're envisioning. We take each of the 4 structures in the lack repressor ensemble, the high affinity form, the high affinity form bound to DNA, the low affinity form, and the low affinity form bound to IPTG. We take those 4 structures and we feed them individually through ligand MPNN. This generates a fixed collection of features that are then going to be the input to our inputs. And we then label those sequences, those things with protein sequences. I need to actually think through how to do this. Yeah, overall idea. I'm actually sure how to do the sequence mapping, so we'll come back to that. But the idea would be we can use feature one, our reference structure, as an input. We're going to hold that. And then the other structures will just take as the difference. So, structure 2 features minus structure one features. Structure 3 features minus structure one features. So what we have is a collection of features, major features, and then perturbations to those features depending on different structures. Um. For each training sequence, we then label those features according to their sequence identity. That's the part that I'm actually realizing I'm a little fuzzy on, so I've got to think through. Um, and run that through a, all of those features, feature one, Delta feature 2, Delta feature 3, et cetera, through a light attention block. And what that will do is take this, these high dimensional features and their differences and reduce it down to a manageable, small number of values. A scalar vector or a vector of scalars. Um, and then we take those features and we run them through a multi-layer perceptron, another small neural network. And the idea with that is it will reduce these to a collection of free energies. And so, uh, the free energies that we will get out of this are what is the pre-energy of the, um, 1st structure. And then what are the changes in the free energy for each of the individual structures? So what's the difference in energy between, uh, one, that's different to energy between, uh, 2 and one, cetera? We can then take those free energies that we calculate. Um, and we can then uh, take the differences in those energies to give us equilibrium constants. So, for example, if structure one is the high affinity form, uh, not bound to DNA, and structure 2 is a high energy, uh, energy affinity form bound to DNA. Uh, the, uh, We would get the energy change, the energy for the wild type structure, H form, and then the difference in the energy when you bring them together into the HD form. of a balance form. That, Delta G, not prime, can then be converted into an equilibrium concept. What is the binding constant for the DNA protein interaction, which can then be fed forward? diff --git a/docs/scripts/make_cli_reference.py b/docs/scripts/make_cli_reference.py new file mode 100644 index 00000000..303a8e2f --- /dev/null +++ b/docs/scripts/make_cli_reference.py @@ -0,0 +1,116 @@ +""" +Write docs/source/cli.rst: the --help of every tfs-* command. + +The page is generated from the same parsers the commands build +(``generalized_main``), so the flags it lists are the flags the code takes. +Rerun after changing any CLI signature or docstring: + + python docs/scripts/make_cli_reference.py + +``tests/tfscreen/util/cli/test_cli_reference.py`` fails when the page is stale. +""" + +import importlib +import os +import sys +import tomllib + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +OUT = os.path.join(ROOT, "docs", "source", "cli.rst") + +HEADER = """\ +.. This file is generated by docs/scripts/make_cli_reference.py. Do not edit. + +============= +CLI reference +============= + +Every ``tfs-*`` command, as its ``--help`` prints it. Arguments without a +leading ``--`` are positional and required. A switch that is on by default is +turned off with ``--no_``. Every command prints its provenance (tfscreen +version, git commit, command line) when it starts and, if it takes +``--out_prefix``, writes it to ``{out_prefix}_provenance.json``. + +The commands in pipeline order are described in :doc:`pipeline`. + +.. contents:: + :local: + :depth: 1 + +""" + +# Pipeline order; any command not listed here goes at the end, alphabetically. +ORDER = [ + "tfs-process-fastq", + "tfs-calibrate-od600", + "tfs-process-counts", + "tfs-configure-model", + "tfs-prefit-calibration", + "tfs-fit-model", + "tfs-sample-posterior", + "tfs-extract-params", + "tfs-predict-theta", + "tfs-predict-growth", + "tfs-predict-epistasis", + "tfs-extract-epistasis", + "tfs-cat-response", + "tfs-compare-runs", + "tfs-summarize-fit", + "tfs-fit-genotypes", + "tfs-sample-prior", + "tfs-simulate", + "tfs-build-empirical", + "tfs-setup-sim-grid", + "tfs-setup-grid", + "tfs-summarize-grid", + "tfs-summarize-calibration", +] + + +def entry_points(): + with open(os.path.join(ROOT, "pyproject.toml"), "rb") as fh: + scripts = tomllib.load(fh)["project"]["scripts"] + names = [n for n in ORDER if n in scripts] + names += sorted(n for n in scripts if n not in ORDER) + return [(n, scripts[n]) for n in names] + + +def help_text(name, target): + from tfscreen.util.cli.generalized_main import capture_parser + module, func = target.split(":") + main = getattr(importlib.import_module(module), func) + parser = capture_parser(main) + parser.prog = name + return parser.format_help() + + +def render(): + # fixed width so the page does not depend on the terminal + os.environ["COLUMNS"] = "88" + parts = [HEADER] + for name, target in entry_points(): + text = help_text(name, target) + parts.append(f"{name}\n{'-' * len(name)}\n\n.. code-block:: text\n\n") + parts.append("".join(f" {line}".rstrip() + "\n" + for line in text.splitlines())) + parts.append("\n") + return "".join(parts).rstrip() + "\n" + + +def main(): + text = render() + check = "--check" in sys.argv + if check: + with open(OUT) as fh: + if fh.read() != text: + sys.exit("docs/source/cli.rst is stale; run " + "python docs/scripts/make_cli_reference.py") + return + with open(OUT, "w") as fh: + fh.write(text) + print(f"Wrote {OUT}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/analysis.rst b/docs/source/analysis.rst deleted file mode 100644 index 1d8b32e4..00000000 --- a/docs/source/analysis.rst +++ /dev/null @@ -1,463 +0,0 @@ -======== -Analysis -======== - -The ``tfscreen.tfmodel`` module provides a hierarchical Bayesian model that -infers per-genotype and operator occupancy -(*θ*) from bacterial growth data and direct binding measurements. - -Standard Workflow -================= - -A complete analysis run consists of the following steps. The example commands -assume all input files are in the current working directory and use the -default output-file naming convention (``tfs_configure_*``, ``tfs_fit_model_*``, -etc.). The :download:`example run.srun <../../examples/tfmodel/run.srun>` -shows a complete Slurm script for a cluster run. - -Step 1: Configure Model (``tfs-configure-model``) --------------------------------------------------- - -Validates the input data, maps categorical labels to numerical indices, selects -model components, and writes the configuration files: - -* ``{out_prefix}_config.yaml`` — main configuration read by all downstream steps -* ``{out_prefix}_priors.csv`` — prior distribution settings for all parameters -* ``{out_prefix}_guesses.csv`` — initial-value guesses for array parameters -* ``{out_prefix}_library.csv`` — per-genotype library composition resolved from - ``--library_config`` (growth models only) - -``binding_df`` (direct binding measurements) is the only required argument. -``growth_df`` is optional; omitting it configures a binding-only model. See -:doc:`model-inputs` for what each input contributes to the fit, how much -data is useful, and the exact file format for ``binding_df``, ``growth_df``, -``base_growth_df``, ``presplit_df``, and ``transformation_lambda``. - -.. code-block:: bash - - tfs-configure-model binding.csv \ - --growth_df growth.csv \ - --library_config run_config.yaml \ - --out_prefix tfs_configure - -``--library_config`` is the library YAML describing the screened library — the -**same file** passed to ``tfs-process-fastq``. It is required whenever -``--growth_df`` is given. It tells the model which genotypes came in as -monoclonal spikes (and so are congression-free), replacing the older -hand-written ``--spiked`` list; because a spiked control is usually also -encoded in the bulk sub-libraries, the resolved table records each genotype's -expected ``pool_fraction`` and ``bulk_fraction`` rather than a yes/no flag. -``tfs-configure-model`` fails if any genotype in the data is absent from the -library, which catches a residue-numbering or wt-sequence mismatch between the -library description and the reads it was called against. - -Key component flags (see :ref:`model-components` for the full list): - -* ``--condition_growth_model`` — how growth rate depends on TF occupancy (default: ``linear``) -* ``--growth_transition_model`` — pre-to-selection phase lag model (default: ``instant``) -* ``--activity_model`` — per-genotype TF activity prior (default: ``horseshoe_geno``) -* ``--theta_model`` — operator occupancy parameterisation (default: ``hill_geno``) -* ``--dk_geno_model`` — pleiotropic growth-effect prior (default: ``hierarchical_geno``) -* ``--transformation_model`` — multi-plasmid congression correction (default: ``single``, i.e. no correction) - -Step 2: Pre-fit Calibration (``tfs-prefit-calibration``) ---------------------------------------------------------- - -Runs a fast MAP fit on a simplified version of the model to calibrate the priors -for the ``condition_growth`` and ``growth_transition`` components. The -calibration fit is restricted to the intersection of genotypes and titrant -conditions present in both the growth and binding data. - -After convergence the script updates the production ``{out_prefix}_priors.csv`` -and ``{out_prefix}_guesses.csv`` files in place (a ``.bak`` backup is written -first), giving the full production fit a warm start. - -Diagnostic artefacts from the calibration MAP run are also written (default -prefix ``tfs_prefit``): - -* ``tfs_prefit_params.npz`` — MAP point estimates in constrained space. -* ``tfs_prefit_checkpoint.pkl`` — optimizer checkpoint (can be passed to - ``--checkpoint_file`` to resume an interrupted calibration run). -* ``tfs_prefit_losses.txt`` — per-epoch loss history. - -.. code-block:: bash - - tfs-prefit-calibration tfs_configure_config.yaml \ - --seed 42 \ - --convergence_tolerance 0.00001 - -Step 3: Fit Model (``tfs-fit-model``) --------------------------------------- - -Performs the main parameter estimation. Three inference methods are available -via ``--analysis_method``: - -* **map** — Maximum A Posteriori optimisation (Adam). Fast; produces a point - estimate. Use ``tfs-sample-posterior`` afterwards to obtain uncertainty - estimates via a Laplace approximation. -* **svi** (default) — Stochastic Variational Inference. Automatically runs a - short MAP pre-pass (``--pre_map_num_epoch``) before the full variational fit. - Produces a full approximate posterior; posterior samples are drawn after - convergence. -* **nuts** — No-U-Turn Sampler (exact MCMC). Slowest; most accurate. - -The example below matches the MAP configuration used in the example ``run.srun``: - -.. code-block:: bash - - tfs-fit-model \ - tfs_configure_config.yaml \ - --seed 42 \ - --analysis_method map \ - --adam_step_size 1e-6 \ - --convergence_check_interval 100 \ - --convergence_window 50 \ - --checkpoint_interval 100 \ - --max_num_epochs 100000000 \ - --pre_map_num_epoch 100000 \ - --convergence_tolerance 0.0005 \ - --patience 5 - -Key outputs (with default ``--out_prefix tfs_fit_model``): - -* ``tfs_fit_model_checkpoint.pkl`` — optimizer checkpoint; resume with ``--checkpoint_file`` -* ``tfs_fit_model_params.npz`` — MAP/SVI parameter point estimates - -Step 4: Sample Posterior (``tfs-sample-posterior``) ----------------------------------------------------- - -Draws posterior samples from a checkpoint produced by ``tfs-fit-model``. The -checkpoint type is detected automatically: - -* **MAP checkpoint** — constructs a Laplace (Hessian-based) Gaussian - approximation at the MAP point, then draws samples. -* **SVI checkpoint** — draws directly from the fitted variational distribution - (resumes with 0 additional optimisation epochs). -* **NUTS checkpoint** — reconstructs posteriors from the saved MCMC samples. - -.. code-block:: bash - - tfs-sample-posterior \ - tfs_configure_config.yaml \ - tfs_fit_model_checkpoint.pkl \ - --num_posterior_samples 1000 \ - --sampling_batch_size 10 \ - --seed 42 - -Output (default ``--out_prefix tfs_posterior``): - -* ``tfs_posterior.h5`` — posterior samples for all latent variables; passed to - the prediction steps below. - -Step 5: Extract Parameters (``tfs-extract-params``) ----------------------------------------------------- - -Extracts interpretable parameter summaries from a posterior ``.h5`` file and -writes one CSV per parameter group. - -.. code-block:: bash - - tfs-extract-params \ - tfs_configure_config.yaml \ - tfs_posterior.h5 - -Outputs (default ``--out_prefix tfs_params``): - -* ``tfs_params_log_hill_K.csv`` — per-genotype log₁₀(*Kd*) with posterior quantiles. -* ``tfs_params_theta_low.csv``, ``tfs_params_theta_high.csv`` — per-genotype - lower and upper occupancy plateaux. -* ``tfs_params_dk_geno.csv`` — per-genotype pleiotropic growth effect. -* Additional CSVs depending on the components selected during configuration - (e.g. ``tfs_params_d_log_hill_K.csv`` for per-mutation effects when using - ``hill_mut``; ``tfs_params_epi_*.csv`` for epistasis terms when - ``--epistasis`` is set). - -**Quantile columns** - -Each CSV contains metadata columns (typically ``genotype`` and -``titrant_name``) followed by 17 quantile columns named ``q{level}``: - -.. code-block:: text - - q0.001 q0.005 q0.01 q0.025 q0.05 q0.1 q0.159 q0.25 - q0.5 q0.75 q0.841 q0.9 q0.95 q0.975 q0.99 q0.995 q0.999 - -``q0.5`` is the posterior median (the recommended point estimate). ``q0.025`` -and ``q0.975`` span the 95% credible interval; ``q0.159`` and ``q0.841`` -span the ±1 σ interval under a normal approximation. - -When ``tfs-extract-params`` is given a MAP checkpoint (``.pkl``) instead of -a posterior file, only a single ``q0.5`` column is written (the MAP point -estimate, not a true posterior quantile). - -Step 6: Predict Growth (``tfs-predict-growth``) ------------------------------------------------- - -Predicts ln(CFU) from the fitted model. By default, predictions are produced at -every (genotype, replicate, condition, titrant_name, titrant_conc, time) -combination present in the training data. - -.. code-block:: bash - - tfs-predict-growth \ - tfs_configure_config.yaml \ - tfs_posterior.h5 - -Output (default ``--out_prefix tfs_growth_pred``): - -* ``tfs_growth_pred.csv`` — one row per prediction point; quantile columns - (``median``, ``lower_95``, ``upper_95``, etc.) plus ``in_training_data``. - -To add predictions at novel genotypes or concentrations, use -``--genotypes_file`` or ``--titrant_concs_file`` (plain-text files, one value -per line). Pass ``--only_files`` to skip training-data combinations and predict -only at the file-specified inputs. - -Step 7: Predict Theta (``tfs-predict-theta``) ----------------------------------------------- - -Predicts operator occupancy *θ* as a function of titrant concentration. - -.. code-block:: bash - - tfs-predict-theta \ - tfs_configure_config.yaml \ - tfs_posterior.h5 \ - --genotypes_file predict_genotypes.txt - -Output (default ``--out_prefix tfs_theta_pred``): - -* ``tfs_theta_pred.csv`` — one row per (genotype, titrant_name, titrant_conc) - with posterior quantile columns and an ``in_training_data`` flag. - -``predict_genotypes.txt`` is a plain-text file with one genotype per line -(e.g. ``M42I/K84L`` or ``wt``). Genotypes not seen during training can be -predicted using the mutation-additivity model when the chosen ``theta_model`` -supports it (e.g. ``hill_mut``). - -Step 8: Categorise Response (``tfs-cat-response``) ---------------------------------------------------- - -Fits categorical response curve models to a ``y_obs``-vs-``x_obs`` curve and -selects the best-fitting model per group by AIC weight. Groups are defined by -the ``genotype`` column plus any ``--group_by`` columns (mirroring -``tfs-extract-epistasis``). For the *θ*-vs-titrant output of -``tfs-predict-theta``, pass the concentration column as ``x_obs`` and a theta -column (e.g. ``q0.5``) as ``y_obs``. - -.. code-block:: bash - - tfs-cat-response \ - tfs_theta_pred.csv \ - titrant_conc \ - q0.5 \ - --group_by titrant_name \ - --num_workers -1 - -If ``--y_std`` is omitted and the ``q0.841``/``q0.159`` quantile columns are -present, the per-row sigma is taken as ``(q0.841 - q0.159) / 2``; otherwise the -fit is unweighted. - -The per-group fits are embarrassingly parallel. ``--num_workers`` defaults to -``-1`` (use ``os.cpu_count() - 1`` processes); pass ``1`` to run serially or a -positive integer to pin the worker count. - -Output (default ``--out_prefix tfs_cat_response``): - -* ``tfs_cat_response.csv`` — one row per group with ``best_model``, AIC - weights, and fitted parameters for every model. -* ``tfs_cat_response_.csv`` — one file per model with that model's - parameter table and per-group fit statistics. -* ``tfs_cat_response_predictions.csv`` — predicted curves for every group and - model. - -Step 9: Summarise Fit (``tfs-summarize-fit``) ----------------------------------------------- - -Collects the outputs of the completed run, computes prediction quality -statistics, and writes diagnostic plots and tables to a ``summary/`` -subdirectory. It can be run after Steps 5–7 are complete and does not -require the full posterior file to be retained. - -.. code-block:: bash - - tfs-summarize-fit out/ - -For a full guide to every output file — including how to read the theta -correlation plots, calibration curves, and parameter-recovery scatter -plots — see :doc:`summarize-fit`. - ---- - -.. _model-components: - -Model Components -================ - -``tfs-configure-model`` accepts ``--_model`` flags to select from a -registry of pluggable sub-models for each aspect of the generative process. - -Condition Growth (``--condition_growth_model``) ------------------------------------------------ - -Maps operator occupancy to per-condition growth rates: *k = b + A·m·θ*. - -* **linear** (default) — shared hierarchical prior for *m* and *b* across conditions -* **power** — power-law relationship; incompatible with ``--theta_rescale_model logit`` -* **saturation** — saturating (Michaelis-Menten-like) relationship; incompatible with ``logit`` - -Growth Transition (``--growth_transition_model``) --------------------------------------------------- - -Models the lag phase when bacteria switch from pre-selection to selection medium. - -* **instant** (default) — no lag; genotypes immediately adopt the new growth rate -* **memory** — occupancy-dependent lag time -* **baranyi** — Baranyi–Roberts lag model -* **baranyi_k** — Baranyi model parameterised through growth rate *k* -* **baranyi_tau** — Baranyi model parameterised through lag time *τ* -* **two_pop** — two-population lag model - -Initial Population (``--ln_cfu0_model``) ------------------------------------------ - -Models the starting genotype frequencies in each replicate. - -* **hierarchical** (default) — shared global prior on ln(CFU\ :sub:`0`) -* **hierarchical_factored** — factored hierarchical prior - -Pleiotropic Growth Effect (``--dk_geno_model``) ------------------------------------------------- - -Models the growth-rate offset attributable to each genotype independent of TF -occupancy (*dk_geno*). - -* **hierarchical_geno** (default) — per-genotype effects drawn from a global prior -* **fixed** — no genotype-specific pleiotropic effects -* **pinned** — genotype-specific effects pinned to externally supplied values - -Activity (``--activity_model``) ---------------------------------- - -Models the per-genotype scalar *A* that multiplies the occupancy contribution -to growth. - -* **horseshoe_geno** (default) — sparse horseshoe prior over genotypes -* **hierarchical_geno** — standard hierarchical prior over genotypes -* **horseshoe_mut** — sparse horseshoe prior, decomposed by mutation -* **hierarchical_mut** — hierarchical prior, decomposed by mutation -* **fixed** — all genotypes share the wildtype activity (*A* = 1) - -Occupancy (``--theta_model``) ------------------------------- - -Parameterises fractional operator occupancy *θ* as a function of titrant -concentration. - -* **hill_geno** (default) — Hill equation with per-genotype *Kd* and *n* -* **categorical_geno** — independent *θ* at each titrant concentration -* **hill_mut** — Hill equation with per-mutation additive effects on *Kd* and *n* -* **thermo.\*** — thermodynamic partition-function models (see - ``configure_model_cli.py`` docstring for the full set of registry keys) - -Transformation Correction (``--transformation_model``) -------------------------------------------------------- - -Corrects for congression (multiple plasmids entering one cell during -transformation). - -* **empirical** (default) — empirical correction curve -* **logit_norm** — logit-normal correction -* **single** — no correction (assumes exactly one plasmid per cell) - -Theta Rescale (``--theta_rescale_model``) ------------------------------------------- - -Rescales *θ* before it enters the condition-growth model. - -* **passthrough** (default) — identity; *θ* ∈ [0, 1] -* **logit** — maps *θ* → log(*θ*/(1−*θ*)); expands dynamic range at extremes. - Incompatible with the ``power`` and ``saturation`` growth models. - -Noise Models ------------- - -Additional noise components (default ``zero`` for all): - -* ``--theta_growth_noise_model``: ``zero`` (default), ``beta``, ``logit_normal`` -* ``--theta_binding_noise_model``: ``zero`` (default), ``beta`` -* ``--growth_noise_model``: ``zero`` (default), ``normal_kt`` (learns a global - growth-rate noise term *σ_k* in quadrature with *ln_cfu_std*) - ---- - -.. _model-naming: - -Model Naming Conventions -======================== - -Model component names follow a set of conventions that encode what each component does. - -Level of Parameterisation --------------------------- - -Models that infer one parameter per *genotype* carry a ``_geno`` suffix; models that -decompose effects at the *mutation* level carry a ``_mut`` suffix. Models with no -natural per-mutation alternative (e.g. ``fixed``) have no suffix. - -Examples: ``hierarchical_geno``, ``horseshoe_mut``, ``hill_geno``, ``hill_mut``. - -Thermodynamic Theta Models ---------------------------- - -Operator-occupancy (*θ*) models derived from an explicit partition function use a -three-part dot-separated name:: - - thermo.{MODEL}.{PRIOR} - -*MODEL* encodes the partition-function topology with four underscore-separated fields: - -.. list-table:: - :header-rows: 1 - :widths: 10 60 - - * - Field - - Meaning - * - ``O`` - - Oligomeric state (e.g. ``O2`` = homodimer) - * - ``C`` - - Number of conformational states - * - ``K`` - - Number of independent equilibrium constants - * - ``U`` - - Unfolded state: ``U0`` = folded only, ``U1`` = folding equilibrium - -A trailing letter (``a``, ``b``, …) disambiguates topologically distinct models -that share the same O/C/K/U counts. These letters carry no ordering; ``a`` simply -means "first registered variant." - -Currently implemented models: - -* ``O2_C4_K3_U0_a`` — four-state lac-repressor homodimer (no unfolding) -* ``O2_C4_K3_U1_a`` — same with an explicit folding/unfolding equilibrium -* ``O2_C12_K5_U0_a`` — full MWC two-state homodimer (no unfolding) -* ``O2_C12_K5_U1_a`` — same with an explicit folding/unfolding equilibrium - -*PRIOR* describes how the equilibrium constants are parameterised: - -.. list-table:: - :header-rows: 1 - :widths: 10 60 - - * - Name - - Description - * - ``PK`` - - Independent normal prior on each log-*K* - * - ``PddG`` - - Priors informed by estimated ΔΔG values - * - ``PnnC`` - - Neural network predicting per-conformation ΔΔG values - * - ``PnnK`` - - Neural network predicting log-*K* values directly (planned) - -Full example names: ``thermo.O2_C4_K3_U0_a.PK``, ``thermo.O2_C12_K5_U1_a.PnnC``. diff --git a/docs/source/cli.rst b/docs/source/cli.rst new file mode 100644 index 00000000..91442d28 --- /dev/null +++ b/docs/source/cli.rst @@ -0,0 +1,2223 @@ +.. This file is generated by docs/scripts/make_cli_reference.py. Do not edit. + +============= +CLI reference +============= + +Every ``tfs-*`` command, as its ``--help`` prints it. Arguments without a +leading ``--`` are positional and required. A switch that is on by default is +turned off with ``--no_``. Every command prints its provenance (tfscreen +version, git commit, command line) when it starts and, if it takes +``--out_prefix``, writes it to ``{out_prefix}_provenance.json``. + +The commands in pipeline order are described in :doc:`pipeline`. + +.. contents:: + :local: + :depth: 1 + +tfs-process-fastq +----------------- + +.. code-block:: text + + usage: tfs-process-fastq [-h] [--out_dir OUT_DIR] [--phred_cutoff PHRED_CUTOFF] + [--min_read_length MIN_READ_LENGTH] + [--allowed_num_flank_diffs ALLOWED_NUM_FLANK_DIFFS] + [--allowed_diff_from_expected ALLOWED_DIFF_FROM_EXPECTED] + [--print_raw_seq] [--max_num_reads MAX_NUM_READS] + [--chunk_size CHUNK_SIZE] [--num_workers NUM_WORKERS] + library_config f1_fastq f2_fastq + + Count the protein genotypes observed in a pair of fastq files. + + Writes counts_.csv and stats_.csv into + out_dir. tfs-process-counts finds each tube's counts file by its + sample name, so name the fastq files after the tubes. + + positional arguments: + library_config The library YAML (the same file later passed to tfs-configure- + model --library_config), a dict of its keys, or a + LibraryManager. Only the genetics keys are read; see + LibraryManager. + f1_fastq forward-read fastq/fastq.gz file + f2_fastq reverse-read fastq/fastq.gz file of the same tube + + options: + -h, --help show this help message and exit + --out_dir OUT_DIR output directory, created if absent (default: 'counts') + --phred_cutoff PHRED_CUTOFF + assign any base with phred < cutoff to "N" (default: 10) + --min_read_length MIN_READ_LENGTH + toss reads where the final readable sequence is less than this + long (default: 50) + --allowed_num_flank_diffs ALLOWED_NUM_FLANK_DIFFS + allow up to allowed_num_diffs differences between the read and + expected_5p and expected_3p when looking for them. (default: + 1) + --allowed_diff_from_expected ALLOWED_DIFF_FROM_EXPECTED + allow up to allowed_diff_from_expected differences between the + read and the expected library sequences when calling the + sequence (default: 2) + --print_raw_seq if True, print the foward and reverse sequences to standard + out. These are written with random 10 letter prefixes in the + order they are encountered. This only writes out correctly + oriented sequences, but does not alter the sequences in any + way. Setting this to True with a high number of + allowed_num_flank_diffs reveals sequences that are somewhat + close to expected and can be used for troubleshooting. The + sequences include the 5' and 3' flank regions. + --max_num_reads MAX_NUM_READS + only read the first max_num_reads. if None, read whole files. + (default: None) + --chunk_size CHUNK_SIZE + break the reads into chunks of chunk_size reads for processing + by their own worker (default: 10000) + --num_workers NUM_WORKERS + number of worker processes; -1 (or None) uses os.cpu_count() - + 1 (default: -1) + +tfs-calibrate-od600 +------------------- + +.. code-block:: text + + usage: tfs-calibrate-od600 [-h] [--out_prefix OUT_PREFIX] [--degree DEGREE] + [--pipette_rel_error PIPETTE_REL_ERROR] + [--detection_threshold DETECTION_THRESHOLD] + replicate_file plate_count_file + + Fit an OD600-to-CFU/mL calibration and write it for the pipeline. + + A calibration depends on the plate reader, plate, volume and strain, so + each lab makes its own from two small experiments done with the same + handling as production samples. + + positional arguments: + replicate_file CSV/TSV/Excel of repeated OD600 readings of a dilution series, + one row per reading: dilution (relative to the undiluted + culture), od600. Gives the reading noise (the largest relative + SD across dilutions) and the detection threshold (midway + between the mean readings of the two most dilute samples). + plate_count_file CSV/TSV/Excel of plate counts of cultures whose OD600 was + read, one row per plate: od600, colonies, dilution (total + dilution factor before plating), plated_volume_mL, + num_dilutions and plating_steps (for the pipetting error). + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Writes {out_prefix}.yaml (the calibration, read by the rest of + the pipeline), {out_prefix}_replicates.csv (reading noise by + dilution), {out_prefix}_plate_counts.csv (CFU/mL, its SD, the + fitted curve and the standardized residual) and + {out_prefix}.pdf. (default: 'tfs_od600') + --degree DEGREE Degree of the polynomial of CFU/mL in OD600 (default 2). + --pipette_rel_error PIPETTE_REL_ERROR + Relative error of one pipetting step (default 0.02). + --detection_threshold DETECTION_THRESHOLD + Override the detection threshold estimated from the dilution + series. (default: None) + +tfs-process-counts +------------------ + +.. code-block:: text + + usage: tfs-process-counts [-h] [--out_prefix OUT_PREFIX] [--od600_file OD600_FILE] + [--od600_calibration_file OD600_CALIBRATION_FILE] + [--tube_volume_mL TUBE_VOLUME_ML] [--presplit] + [--counts_glob_prefix COUNTS_GLOB_PREFIX] + [--min_genotype_obs MIN_GENOTYPE_OBS] + [--pseudocount PSEUDOCOUNT] [--no_verbose] + sample_file counts_dir + + Turn per-tube genotype counts into the table tfs-configure-model reads. + + The tube table has one row per sequenced tube: a unique sample name, + its library, and the design columns (replicate, condition_pre, + t_pre, condition_sel, t_sel, titrant_name, + titrant_conc), which are carried through to the output. Every library + is processed in one call; genotypes are filtered and filled in within + each library. + + Each tube needs its total CFU. Give its OD600 (--od600_file, or an + od600 column in the tube table) with --od600_calibration_file and + --tube_volume_mL, and the totals and their SDs are computed here: + sample_cfu_curve_std is the calibration curve's error, shared by every + tube, and sample_cfu_reading_std is the reading's own. Or supply them + in the tube table as sample_cfu and sample_cfu_std. + + Writes {out_prefix}.csv. Without --presplit it is the growth table + for tfs-configure-model --growth_df: one row per genotype and tube + with counts, sample_reads, frequency, ln_cfu and + ln_cfu_var. With --presplit it is the presplit table for + --presplit_df (library, replicate, condition_pre, + genotype, ln_cfu, ln_cfu_std). + + positional arguments: + sample_file Tube table (CSV, TSV or Excel), one row per sequenced tube. + counts_dir Directory of tfs-process-fastq count files. Each tube's file + must be the only one matching + {counts_glob_prefix}*{sample}*.csv. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Output prefix. Default tfs_growth, or tfs_presplit with + --presplit. + --od600_file OD600_FILE + Table of one OD600 reading per tube (columns sample, od600). + (default: None) + --od600_calibration_file OD600_CALIBRATION_FILE + OD600-to-CFU/mL calibration written by tfs-calibrate-od600. + (default: None) + --tube_volume_mL TUBE_VOLUME_ML + Culture volume of one tube in mL. Required with OD600. + (default: None) + --presplit Write the presplit table instead of the growth table. The tube + table then needs replicate and condition_pre. + --counts_glob_prefix COUNTS_GLOB_PREFIX + File-name prefix of the count files. (default: 'counts') + --min_genotype_obs MIN_GENOTYPE_OBS + Drop genotypes with fewer total reads than this within a + library. (default: 10) + --pseudocount PSEUDOCOUNT + Added to every genotype's count before taking frequencies. + (default: 1) + --no_verbose Turn off (on by default): Print which count file was matched + to each tube. + +tfs-configure-model +------------------- + +.. code-block:: text + + usage: tfs-configure-model [-h] [--binding_df BINDING_DF] [--growth_df GROWTH_DF] + [--presplit_df PRESPLIT_DF] + [--base_growth_df BASE_GROWTH_DF] [--out_prefix OUT_PREFIX] + [--condition_growth_model CONDITION_GROWTH_MODEL] + [--growth_transition_model GROWTH_TRANSITION_MODEL] + [--ln_cfu0_model LN_CFU0_MODEL] + [--dk_geno_model DK_GENO_MODEL] + [--activity_model ACTIVITY_MODEL] + [--theta_model THETA_MODEL] + [--transformation_model TRANSFORMATION_MODEL] + [--transformation_lambda TRANSFORMATION_LAMBDA TRANSFORMATION_LAMBDA] + [--congression_theta_rule CONGRESSION_THETA_RULE] + [--congression_dk_rule CONGRESSION_DK_RULE] + [--congression_dk_alpha CONGRESSION_DK_ALPHA] + [--congression_sets CONGRESSION_SETS [CONGRESSION_SETS ...]] + [--theta_rescale_model THETA_RESCALE_MODEL] + [--theta_growth_noise_model THETA_GROWTH_NOISE_MODEL] + [--theta_binding_noise_model THETA_BINDING_NOISE_MODEL] + [--growth_noise_model GROWTH_NOISE_MODEL] + [--sample_offset_model SAMPLE_OFFSET_MODEL] + [--growth_likelihood GROWTH_LIKELIHOOD] + [--theta_gauge_conc THETA_GAUGE_CONC THETA_GAUGE_CONC] + [--library_config LIBRARY_CONFIG] + [--growth_shares_replicates] [--epistasis] + [--thermo_data THERMO_DATA] + [--thermo_ddG_units THERMO_DDG_UNITS] + [--thermo_temperature_K THERMO_TEMPERATURE_K] + [--thermo_column_map THERMO_COLUMN_MAP] + [--thermo_residue_offset THERMO_RESIDUE_OFFSET] + [--thermo_max_missing_fraction THERMO_MAX_MISSING_FRACTION] + [--batch_size BATCH_SIZE] [--binding_weight BINDING_WEIGHT] + [--set_priors SET_PRIORS [SET_PRIORS ...]] + [--growth_priors GROWTH_PRIORS] + [--growth_priors_wt_rates GROWTH_PRIORS_WT_RATES] + [--growth_priors_sd_floor GROWTH_PRIORS_SD_FLOOR] + [--allow_missing_spikes] [--skip_model_stats] + + Build and write the YAML configuration files needed by tfs-fit-model. + + Constructs a ModelOrchestrator from the supplied data and model-component choices, + then writes {out_prefix}_config.yaml (the main configuration), + {out_prefix}_priors.csv (prior distributions for all parameters), + {out_prefix}_guesses.csv (initial-value guesses for array parameters) and, + for a growth model, {out_prefix}_library.csv (the per-genotype library + composition resolved from library_config). + + At least one of growth_df and binding_df is required. With both, the + joint growth + binding model is configured. With only binding_df, a + binding-only model infers theta directly from the binding measurements. + With only growth_df, a growth-only model infers theta from growth alone + (no binding likelihood, so theta_binding_noise_model must stay 'zero' + and binding_weight unset; tfs-prefit-calibration needs binding and + refuses such a config). + + options: + -h, --help show this help message and exit + --binding_df BINDING_DF + Path to the binding data CSV file (theta vs. titrant + measurements per genotype). When omitted, a growth-only model + is configured. (default: None) + --growth_df GROWTH_DF + Path to the growth data CSV file (ln_cfu measurements per + genotype, replicate, and timepoint). When omitted, a binding- + only model is configured. (default: None) + --presplit_df PRESPLIT_DF + Path to the pre-split (t = -t_pre) sequencing-observation CSV + file. Provides a direct constraint on ln_cfu0 for genotypes it + covers. See data_class.PreSplitData. (default: None) + --base_growth_df BASE_GROWTH_DF + Path to a CSV of direct, reference-condition growth-rate + measurements (columns: genotype, rate, rate_std) for a subset + of genotypes (wt at minimum). Anchors the new k_ref latent + scalar to dk_geno via rate_obs ~ Normal(k_ref + dk_geno, + rate_std), resolving an identifiability confound between + condition_growth's k/m and dk_geno's hierarchical + hyperparameters. See model_orchestrator._read_base_growth_df + and generative/model.py's base_growth_obs block. (default: + None) + --out_prefix OUT_PREFIX + Prefix for the output files ({out_prefix}_config.yaml, + {out_prefix}_priors.csv, {out_prefix}_guesses.csv and, for a + growth model, {out_prefix}_library.csv). Default + 'tfs_configure'. + --condition_growth_model CONDITION_GROWTH_MODEL + Model to use to describe growth under different conditions + (e.g., pheS+4CP). Allowed values are 'linear' (default), + 'power', or 'saturation'. + --growth_transition_model GROWTH_TRANSITION_MODEL + Model to use to describe the transition between the pre- + selection and selection phases. Allowed values are 'instant' + (default), 'memory', 'baranyi', 'baranyi_k', 'baranyi_tau', or + 'two_pop'. + --ln_cfu0_model LN_CFU0_MODEL + Model to use to describe ln_cfu0, the initial populations of + genotypes in each replicate. Allowed values are 'hierarchical' + (default) or 'hierarchical_factored'. The latter shares each + genotype's baseline across pre-conditions and is refused when + they come from different libraries (kanR and pheS are grown up + separately). + --dk_geno_model DK_GENO_MODEL + Model to use to describe dk_geno, the pleiotropic effect of a + genotype on growth, independent of occupancy. Allowed values + are 'hierarchical_geno' (default), 'fixed', or 'pinned' + (dk_geno fixed to externally supplied per-genotype values for + a subset of genotypes). + --activity_model ACTIVITY_MODEL + Model to use to describe activity, a scalar multiplied against + occupancy that defines how strongly a genotype alters + transcription given its occupancy. Allowed values are 'fixed' + (default; activity 1 for every genotype), 'hierarchical_geno', + 'horseshoe_geno', 'hierarchical_mut', or 'horseshoe_mut'. + --theta_model THETA_MODEL + Model to use to describe theta, the fractional occupancy of a + genotype on the transcription factor binding site. Allowed + values are 'hill_geno' (default), 'categorical_geno', + 'hill_mut', and the thermodynamic partition-function models + (pass the exact registry key, e.g. 'thermo.O2_C4_K3_U0_a.PK', + 'thermo.O2_C4_K3_U0_a.PnnC', 'thermo.O2_C4_K3_U0_a.PddG', + 'thermo.O2_C12_K5_U0_a.PK', 'thermo.O2_C12_K5_U0_a.PnnC', + 'thermo.O2_C12_K5_U0_a.PddG', and their O2_C4_K3_U1_a / + O2_C12_K5_U1_a unfolded equivalents). 'hill_relative' fits + growth alone on a wt-relative scale X instead of theta (wt's X + is 1 at the low and 0 at the high gauge concentration, + theta_gauge_conc). It takes no binding_df, and requires + activity 'fixed', theta_rescale 'passthrough', linear + condition growth, theta_growth_noise 'zero' and, with the + 'mixture' transformation, congression_theta_rule 'max'. + --transformation_model TRANSFORMATION_MODEL + Model for congression. Allowed values are 'single' (default; + one plasmid per cell) or 'mixture' (clean and congressed cells + mixed at the observable level). 'empirical' and 'logit_norm' + were removed. + --transformation_lambda TRANSFORMATION_LAMBDA TRANSFORMATION_LAMBDA + (mean, std) -- the experimentally measured congression lambda, + in linear space (e.g. (0.36, 0.05)). Required when + transformation_model is 'mixture'; forbidden when it is + 'single'. Used to moment-match a LogNormal prior for the + transformation's lambda parameter, replacing the manual step + of hand-editing the priors/guesses CSVs with rescaled log- + space values. (default: None) + --congression_theta_rule CONGRESSION_THETA_RULE + How the 'mixture' transformation builds a congressed cell's + theta from its plasmids' thetas: 'homodimer' (default), + 'heterodimer' or 'max'. The homodimer and heterodimer rules + require activity_model 'fixed'. Ignored by 'single'. + --congression_dk_rule CONGRESSION_DK_RULE + How the 'mixture' transformation builds a congressed cell's + dk_geno from its plasmids': 'dilution' (default; the share- + weighted mean), 'softmin' (with congression_dk_alpha) or 'min' + (the worst variant sets the cost). Ignored by 'single'. + --congression_dk_alpha CONGRESSION_DK_ALPHA + alpha for congression_dk_rule 'softmin' (> 0, in units of + 1/dk): dk_cell = -(1/alpha) log sum_g x_g exp(-alpha dk_g). + Required by 'softmin', refused by the other rules. (default: + None) + --congression_sets CONGRESSION_SETS [CONGRESSION_SETS ...] + With the 'mixture' transformation, the number of fixed co- + resident sets drawn per genotype, by co-resident count: entry + n-1 is the number of sets with n co-resident plasmids (the + mixture's quadrature over a congressed cell's partners). None + keeps the orchestrator's default, 12 3 1 (16 sets). Each set + is one cell class grown through the whole model, so the cost + of a fit grows with the total. Written to the config. + --theta_rescale_model THETA_RESCALE_MODEL + Rescaling applied to theta before it enters the growth model. + Allowed values are 'passthrough' (default, identity) or + 'logit' (maps theta to log(theta/(1-theta)), expanding the + dynamic range at both extremes). + --theta_growth_noise_model THETA_GROWTH_NOISE_MODEL + Model to use for stochastic experimental noise in theta + measured by bacterial growth. Allowed values are 'zero' + (default), 'beta', or 'logit_normal'. + --theta_binding_noise_model THETA_BINDING_NOISE_MODEL + Model to use for stochastic experimental noise in theta + measured by binding. Allowed values are 'zero' (default), + 'beta' or 'genotype_offset' (one offset per binding genotype, + the same at every titrant concentration, with a learned scale: + an in vivo/in vitro difference or a curve's day-normalization + error). + --growth_noise_model GROWTH_NOISE_MODEL + Model for additive growth-rate noise. 'zero' (default) adds no + noise; 'normal_kt' learns a global sigma_k that inflates the + observation scale in quadrature with ln_cfu_std, capturing + biological variability in growth rates not explained by theta + or dk_geno. + --sample_offset_model SAMPLE_OFFSET_MODEL + Per-tube offset shared by every genotype in a tube. 'level' + (default) adds one ln_cfu offset per tube with a learned, + constant SD (the tube's composition offset and any error in + its supplied total, including the tube's own growth noise); + 'zero' adds none; 'normal' adds a per-tube growth-rate offset + scaled by elapsed time. With growth_likelihood 'counts', an + error in a tube's supplied total otherwise moves every + genotype in it. + --growth_likelihood GROWTH_LIKELIHOOD + How growth data are observed: 'counts' (default; negative + binomial on the read counts, with a learned dispersion whose + variance has a part proportional to the mean and a quadratic + part; no pseudocount) or 'lncfu' (Student-t on ln_cfu). + 'counts' needs a 'counts' column, each tube's total reads + ('sample_reads', or 'adjusted_counts' and 'frequency' to + derive it) and total cells ('sample_ln_cfu', or 'sample_cfu' + with its uncertainty), and growth_noise_model 'zero'. It cut + theta RMSE by 35-60% against 'lncfu' on simulations + (planning/studies/count-likelihood/). + --theta_gauge_conc THETA_GAUGE_CONC THETA_GAUGE_CONC + c_lo c_hi: the titrant concentrations at which theta_model + 'hill_relative' pins wt's X to 1 and 0. Default: the lowest + and highest concentration in growth_df. Refused by other theta + models. + --library_config LIBRARY_CONFIG + Path to the library YAML describing the screened library -- + the same file handed to tfs-process-fastq. Required whenever + growth_df is given (a binding-only model has no congression + correction and no ln_cfu0 latents, so it does not need one). + Read keys: reading_frame, first_amplicon_residue, wt_seq, + degen_sites, tiles, tile_combos, spiked_seqs and + library_mixture; any other keys (a full simulate config, say) + are ignored. Genotypes encoded by a spiked sequence become the + spiked set, replacing the old hand-supplied --spiked list. The + resolved per-genotype table is written to + {out_prefix}_library.csv and is what tfs-fit-model reads. + (default: None) + --growth_shares_replicates + Whether replicates should share the same parameters for the + growth and growth transition models. Default is False. + --epistasis Whether to model pairwise epistatic interactions between + mutations in any mutation-level model (hill_mut, + hierarchical_mut). When True, each pair of mutations present + in the same genotype gets an independent epistasis term. When + False (default), effects are purely additive at the mutation + level. + --thermo_data THERMO_DATA + Path to the structural/thermodynamic data file. Required when + theta_model is a thermo-based model; ignored otherwise. For + PnnC models this must be the HDF5 file produced by + scripts/generate_struct_ensemble.py. For PddG models this must + be a CSV file with a mut column and one column per structure + containing pre-computed ΔΔG prior means; the required + structure columns depend on the topology -- O2_C4_* models use + (H, HD, L, LE2) while O2_C12_* models use (H, HO, L, LO, HE2, + LE2). Mutation or mutation may name the mutation column, an + unnamed index column is dropped and other columns are ignored. + ΔΔG_X shifts state X's free energy (positive destabilises X). + The loader prints how many library mutations the file covers + and fails on zero coverage or on wt residues that disagree + with the library. (default: None) + --thermo_ddG_units THERMO_DDG_UNITS + Unit of the PddG CSV's ΔΔG values: kT (default; used as-is), + kcal/mol or kJ/mol (divided by RT at load). + --thermo_temperature_K THERMO_TEMPERATURE_K + Temperature for that conversion. Default 298.15. + --thermo_column_map THERMO_COLUMN_MAP + model_name=csv_column pairs, comma-separated (e.g. HD=HO), + naming the CSV column that supplies a model structure. + (default: None) + --thermo_residue_offset THERMO_RESIDUE_OFFSET + Added to the file's residue numbers (CSV labels or HDF5 + residue_nums) to give the library numbering, e.g. 1 for a + 0-based file. Default 0. + --thermo_max_missing_fraction THERMO_MAX_MISSING_FRACTION + Warn when more than this fraction of library mutations has no + ΔΔG in the CSV. Default 0.1. These five are stored in the + config as components.thermo_options. + --batch_size BATCH_SIZE + Mini-batch size for SVI. Defaults to 1024. Set to None to use + the full dataset as a single batch. + --binding_weight BINDING_WEIGHT + Multiplicative scale applied to the binding log-likelihood. + None (default) is 1: each binding observation counts once, the + likelihood of the data as measured. Until 2026-10-16 the + default was N_growth_rows / N_binding_rows, which made the + binding data count as much as the whole growth dataset; pass + that ratio to reproduce an older configuration. The resolved + value (never None) is saved in the YAML, so a config written + before the change keeps its weight. + --set_priors SET_PRIORS [SET_PRIORS ...] + name=value pairs that set scalar priors in + {out_prefix}_priors.csv, so the file is never edited by hand. + A name is a full row name + (theta.theta_log_hill_n_hyper_scale_fixed) or a unique dotted + suffix of one (theta_log_hill_n_hyper_scale_fixed, + sample_offset.sigma_fixed). An unknown or ambiguous name is an + error. Common uses: sigma_fixed=0.17 holds the level tube + offsets' SD; theta_log_hill_n_hyper_scale_fixed=0.5 holds the + population SD of log(hill_n) (and, for hill_relative, + theta_X_low_hyper_scale_fixed, + theta_X_delta_hyper_scale_fixed, + theta_log_hill_K_hyper_scale_fixed); for the PddG/PnnC thermo + models, theta_ddG_sigma_fixed=1.69 holds every structure's ΔΔG + scale σ_s at 1.69 kT. (default: None) + --growth_priors GROWTH_PRIORS + CSV of per-condition growth priors for linear condition + growth: a condition_rep column (and replicate, if conditions + are not shared across replicates) plus any of k_loc, k_scale, + m_loc, m_scale, and the 0/1 flags k_pinned, m_pinned, which + hold that condition's k or m at its loc instead of fitting it + (say, one marker's k and m held at an independent estimate + while the other's are fitted). Conditions or values it leaves + out keep the defaults. Use it where tfs-prefit-calibration + does not run (a growth-only model); the pre-fit overwrites + k_loc and m_loc. + --growth_priors_wt_rates GROWTH_PRIORS_WT_RATES + CSV of wt monoculture growth rates (condition_sel, + titrant_conc, rate_mean, rate_sd, num_replicates), for + theta_model 'hill_relative' only. For each listed condition, k + = the rate at the high gauge concentration and m = the rate at + the low one minus k, each with the standard error of the mean + as SD (floored at growth_priors_sd_floor). List only the + conditions where the monoculture stands for the library's wt. + Overrides growth_priors for the conditions it lists. (default: + None) + --growth_priors_sd_floor GROWTH_PRIORS_SD_FLOOR + Smallest k and m prior SD (per minute) derived from + growth_priors_wt_rates. Default 0.002. + --allow_missing_spikes + By default, a spiked genotype named in library_config with no + growth data is an error: it usually means the counts were + called with a different library config. Pass this to only + report it (a spike can drop out of a real experiment). + --skip_model_stats Skip the pre-fit parameter/observation accounting. By default + a summary is printed to stdout and written to + {out_prefix}_model_stats.csv (one row per latent) and + {out_prefix}_model_stats.json (headline counts, coverage, + anchors, warnings). See tfmodel.model_stats. The accounting + traces the model abstractly, so it costs no meaningful time or + memory; skip it only if a component's trace misbehaves. + +tfs-prefit-calibration +---------------------- + +.. code-block:: text + + usage: tfs-prefit-calibration [-h] [--seed SEED] [--checkpoint_file CHECKPOINT_FILE] + [--out_prefix OUT_PREFIX] + [--adam_step_size ADAM_STEP_SIZE] + [--adam_final_step_size ADAM_FINAL_STEP_SIZE] + [--adam_step_size_cut ADAM_STEP_SIZE_CUT] + [--adam_clip_norm ADAM_CLIP_NORM] + [--elbo_num_particles ELBO_NUM_PARTICLES] + [--convergence_window_steps CONVERGENCE_WINDOW_STEPS] + [--patience PATIENCE] [--convergence_z CONVERGENCE_Z] + [--loss_rtol LOSS_RTOL] + [--param_tolerance PARAM_TOLERANCE] + [--checkpoint_interval CHECKPOINT_INTERVAL] + [--max_num_epochs MAX_NUM_EPOCHS] + [--epoch_checkpoint_interval EPOCH_CHECKPOINT_INTERVAL] + [--k_scale_floor K_SCALE_FLOOR] + [--m_scale_floor M_SCALE_FLOOR] + [--k_scale_ceiling K_SCALE_CEILING] + [--m_scale_ceiling M_SCALE_CEILING] + [--hessian_chunk_size HESSIAN_CHUNK_SIZE] [--pin_m] + config_file + + Run the calibration pre-fit and update the production priors / guesses + CSVs in place. + + The script: + + 1. Reads the production config_file to obtain the production + ModelOrchestrator (so we know which priors / guesses files to + update later) and the production data paths. + 2. Filters the growth and binding inputs to their shared + (genotype, titrant_name, titrant_conc) cells. + 3. Builds an in-process calibration ModelOrchestrator whose + condition_growth and growth_transition components match + the production choices but whose other components are pinned / + collapsed (see module docstring). + 4. Runs MAP, then computes per-site Hessian sigmas at the MAP + point. + 5. Writes .bak copies of the production priors and guesses CSVs + and overwrites only the rows that belong to condition_growth + or growth_transition. + + positional arguments: + config_file Path to the *production* YAML configuration file. The pre-fit + does not write a calibration YAML — it only updates the + existing production CSVs. + + options: + -h, --help show this help message and exit + --seed SEED Random seed for reproducibility. Required unless resuming from + a checkpoint. (default: None) + --checkpoint_file CHECKPOINT_FILE + Path to a previously written pre-fit checkpoint to resume + from. (default: None) + --out_prefix OUT_PREFIX + Prefix for calibration MAP artefacts ({out_prefix}_params.npz, + {out_prefix}_checkpoint.pkl, etc.). Defaults to "tfs_prefit". + These are diagnostic outputs; the user-facing artefact is the + in-place update of the production CSVs. + --adam_step_size ADAM_STEP_SIZE + Starting step size for the Adam optimizer (default 1e-3). + --adam_final_step_size ADAM_FINAL_STEP_SIZE + Smallest step size for the Adam optimizer (default 1e-6). The + step size is cut by adam_step_size_cut each time the fit + plateaus, and the run converges at the first plateau at this + size. + --adam_step_size_cut ADAM_STEP_SIZE_CUT + Factor applied to the step size at each cut (default 0.1). + --adam_clip_norm ADAM_CLIP_NORM + Clip each gradient element to +/- this value (numpyro + ClippedAdam). None (default) disables clipping: when gradients + are much larger than the clip, an elementwise clip makes Adam + follow the gradient's sign, and its fixed point is biased by + rare large ELBO penalties. + --elbo_num_particles ELBO_NUM_PARTICLES + Number of particles for ELBO estimation (default 2). + --convergence_window_steps CONVERGENCE_WINDOW_STEPS + Optimizer steps per convergence window (default 2000; at least + 10 epochs). + --patience PATIENCE Consecutive windows without loss improvement required for a + step-size cut, and (at the final step size) without loss + improvement or parameter movement required for convergence + (default 3). + --convergence_z CONVERGENCE_Z + Standard errors of change attributed to noise (default 3). + --loss_rtol LOSS_RTOL + Loss changes per window below this fraction of the loss count + as no change (default 1e-6). + --param_tolerance PARAM_TOLERANCE + Parameter movement per window, beyond noise, still counted as + no change (default 0.05), in prior SDs (or log/logit units for + positive or bounded parameters). + --checkpoint_interval CHECKPOINT_INTERVAL + Frequency (in epochs) between checkpoint writes (default 10). + --max_num_epochs MAX_NUM_EPOCHS + Maximum number of MAP optimization epochs (default 100000); a + cap only. + --epoch_checkpoint_interval EPOCH_CHECKPOINT_INTERVAL + Frequency (in epochs) to write numbered epoch checkpoints to a + checkpoints/ subdirectory (default 0). Set to 0 or None to + disable. + --k_scale_floor K_SCALE_FLOOR + Minimum prior scale applied to the baseline growth-rate (k/b) + parameters after Hessian estimation (default + _DEFAULT_K_SCALE_FLOOR = 0.002). Represents the irreducible + day-to-day variability in growth rates that the Hessian cannot + capture. For real data, estimate this by running the prefit on + multiple independent calibration replicates and measuring the + spread of MAP estimates. + --m_scale_floor M_SCALE_FLOOR + Minimum prior scale applied to the theta-slope (m) parameters + after Hessian estimation (default _DEFAULT_M_SCALE_FLOOR = + 0.001). + --k_scale_ceiling K_SCALE_CEILING + Maximum prior scale for k after Hessian estimation (default + _DEFAULT_K_SCALE_CEILING = 0.1, matching the linear component + default). Prevents degenerate Hessian directions — which arise + from the b / ln_cfu0 near-degeneracy in the prefit calibration + — from producing a scale larger than the original production + prior and thereby *loosening* the constraint. + --m_scale_ceiling M_SCALE_CEILING + Maximum prior scale for m_scale_plus after Hessian estimation + (default _DEFAULT_M_SCALE_CEILING = 0.01, matching the linear + default). + --hessian_chunk_size HESSIAN_CHUNK_SIZE + Number of Hessian rows computed per device batch (default 64). + Reduce if device runs out of memory during Hessian + computation. + --pin_m When set, hard-clamp the production model's per-condition + slope m to its calibration MAP loc by writing + condition_growth.m_pinned into the priors CSV (default False). + Use this instead of an ultra-tight m_scale when you want m + held exactly at the calibration value — a soft prior can be + overridden by the growth likelihood. Only affects m; k keeps + its floored soft prior. + +tfs-fit-model +------------- + +.. code-block:: text + + usage: tfs-fit-model [-h] [--seed SEED] [--checkpoint_file CHECKPOINT_FILE] + [--analysis_method ANALYSIS_METHOD] [--guide_type GUIDE_TYPE] + [--guide_rank GUIDE_RANK] [--guide_init_scale GUIDE_INIT_SCALE] + [--out_prefix OUT_PREFIX] [--adam_step_size ADAM_STEP_SIZE] + [--adam_final_step_size ADAM_FINAL_STEP_SIZE] + [--adam_step_size_cut ADAM_STEP_SIZE_CUT] + [--adam_clip_norm ADAM_CLIP_NORM] + [--elbo_num_particles ELBO_NUM_PARTICLES] + [--convergence_window_steps CONVERGENCE_WINDOW_STEPS] + [--patience PATIENCE] [--convergence_z CONVERGENCE_Z] + [--loss_rtol LOSS_RTOL] [--param_tolerance PARAM_TOLERANCE] + [--checkpoint_interval CHECKPOINT_INTERVAL] + [--max_num_epochs MAX_NUM_EPOCHS] + [--forward_batch_size FORWARD_BATCH_SIZE] + [--pre_map_num_epoch PRE_MAP_NUM_EPOCH] + [--init_param_jitter INIT_PARAM_JITTER] + [--nuts_num_warmup NUTS_NUM_WARMUP] + [--nuts_num_samples NUTS_NUM_SAMPLES] + [--nuts_num_chains NUTS_NUM_CHAINS] + [--nuts_target_accept_prob NUTS_TARGET_ACCEPT_PROB] + [--nuts_dense_mass] + [--epoch_checkpoint_interval EPOCH_CHECKPOINT_INTERVAL] + [--init_from INIT_FROM] [--stage_offsets STAGE_OFFSETS] + [--staged_step_size STAGED_STEP_SIZE] + config_file + + Fit the joint hierarchical model using a previously generated configuration file. + + This function extracts estimates of transcription factor fractional occupancy (theta) + and other latent parameters using Stochastic Variational Inference (SVI) or maximum a + posteriori (MAP) approaches based on the config. + + Optimization (SVI, MAP and the pre-MAP warm-up) runs in windows of + convergence_window_steps optimizer steps. After patience + consecutive windows without a significant improvement in the loss + (against its own noise), the step size is cut by adam_step_size_cut. + Once it has reached adam_final_step_size, the run stops after + patience consecutive windows in which the loss has not improved and no + parameter has moved (against its posterior or prior width). Each window + is recorded in {out_prefix}_convergence.csv. + + positional arguments: + config_file Path to a YAML configuration file to load settings from. + + options: + -h, --help show this help message and exit + --seed SEED Random seed for reproducibility. Must be provided if not + loading from a checkpoint. (default: None) + --checkpoint_file CHECKPOINT_FILE + Path to a checkpoint file to resume SVI from, or None to start + fresh. A resumed run continues at the checkpoint's step size + and convergence stage. (default: None) + --analysis_method ANALYSIS_METHOD + Method for inference. Allowed values are 'svi' (default), + 'map', or 'nuts'. Case-insensitive. Posterior sampling is not + performed; call tfs-sample-posterior after fitting. + --guide_type GUIDE_TYPE + Variational family for analysis_method 'svi' (default + 'component', the guide assembled from the model components). + Also accepts the numpyro autoguides 'auto_normal', + 'auto_diagonal_normal', 'auto_multivariate_normal', + 'auto_low_rank_multivariate_normal' and 'delta'; numpyro class + names such as 'AutoNormal' work too. Every guide's location + starts at the pre-MAP solution when pre_map_num_epoch > 0, + otherwise at the configured guesses. A resumed checkpoint must + use the guide it was written with. + --guide_rank GUIDE_RANK + Covariance rank for 'auto_low_rank_multivariate_normal' + (numpyro's default when omitted). + --guide_init_scale GUIDE_INIT_SCALE + Initial scale of the variational distribution, in each site's + unconstrained units (default 1e-4, DEFAULT_GUIDE_INIT_SCALE). + For 'component' it caps every guide scale at the start; for an + autoguide it is numpyro's init_scale for a fresh fit (a + resumed fit keeps its checkpoint's). SVI widens the scales + itself; a large start throws the pre-MAP point away. Not + accepted by 'delta'. + --out_prefix OUT_PREFIX + Prefix for all output files (default 'tfs_fit_model'): + {out_prefix}_checkpoint.pkl, {out_prefix}_params.npz, + {out_prefix}_losses.txt, {out_prefix}_convergence.csv, the + pre-MAP's {out_prefix}_premap_* files, and, for NUTS only, the + posterior HDF5. + --adam_step_size ADAM_STEP_SIZE + Starting step size for the Adam optimizer (default 1e-3). + --adam_final_step_size ADAM_FINAL_STEP_SIZE + Smallest step size (default 1e-6). Set equal to adam_step_size + for a constant step size. + --adam_step_size_cut ADAM_STEP_SIZE_CUT + Factor applied to the step size at each cut (default 0.1). + --adam_clip_norm ADAM_CLIP_NORM + Clip each gradient element to +/- this value (numpyro + ClippedAdam). None (default) disables clipping: when gradients + are much larger than the clip, an elementwise clip makes Adam + follow the gradient's sign, and its fixed point is biased by + rare large ELBO penalties. + --elbo_num_particles ELBO_NUM_PARTICLES + Number of particles for ELBO estimation (default 2). + --convergence_window_steps CONVERGENCE_WINDOW_STEPS + Optimizer steps per convergence window (default 2000; at least + 10 epochs, so with mini-batching every genotype is visited + several times per window). + --patience PATIENCE Consecutive windows without loss improvement required for a + step-size cut, and (at the final step size) without loss + improvement or parameter movement required for stopping + (default 3). + --convergence_z CONVERGENCE_Z + Standard errors of change attributed to noise (default 3). + --loss_rtol LOSS_RTOL + Loss changes per window below this fraction of the loss count + as no change (default 1e-6); matters only for + (near-)deterministic losses. + --param_tolerance PARAM_TOLERANCE + Parameter movement per window, beyond noise, still counted as + no change (default 0.05), in posterior SDs (SVI), prior SDs + (MAP), or log/logit units (positive or bounded parameters). + --checkpoint_interval CHECKPOINT_INTERVAL + Frequency (in epochs) between checkpoints (default 10). + --max_num_epochs MAX_NUM_EPOCHS + Maximum number of SVI or MAP epochs (default 100000); a cap + only. + --forward_batch_size FORWARD_BATCH_SIZE + When getting NUTS posteriors, calculate forward predictions in + batches of this size (default 512). + --pre_map_num_epoch PRE_MAP_NUM_EPOCH + Maximum number of epochs of the MAP warm-up run before SVI or + NUTS (default 10000; 0 skips it). The warm-up stops earlier + when it converges. SVI's guide and NUTS's chains start at its + point. Not used by 'map'. + --init_param_jitter INIT_PARAM_JITTER + Multiplicative jitter on the component guide's starting + parameters (default 0, none). Not used by autoguides or MAP. + Leave it off when starting from a pre-MAP: it scales with each + value, so 0.1 moves an ln_cfu0 location near 15 by about 1.5 + ln units. + --nuts_num_warmup NUTS_NUM_WARMUP + Number of NUTS warmup steps (default 500). Only used if + analysis_method is 'nuts'. + --nuts_num_samples NUTS_NUM_SAMPLES + Number of NUTS posterior samples to draw (default 500). Only + used if analysis_method is 'nuts'. + --nuts_num_chains NUTS_NUM_CHAINS + Number of MCMC chains (default 1). Only used if + analysis_method is 'nuts'. + --nuts_target_accept_prob NUTS_TARGET_ACCEPT_PROB + Target acceptance probability for NUTS step-size adaptation + (default 0.9). Only used if analysis_method is 'nuts'. + --nuts_dense_mass Adapt a dense mass matrix for NUTS instead of a diagonal one + (default False). Worth it for correlated posteriors of up to a + few thousand latents; with a diagonal one the relative fit ran + at NUTS's maximum tree depth. Only used if analysis_method is + 'nuts'. + --epoch_checkpoint_interval EPOCH_CHECKPOINT_INTERVAL + Frequency (in epochs) to write numbered epoch checkpoints to a + checkpoints/ subdirectory alongside out_prefix (default 1000). + Files are named {epoch:07d}_checkpoint.pkl. Set to 0 or None + to disable. Raises FileExistsError if a target file already + exists. + --init_from INIT_FROM + A *_params.npz written by a MAP fit ({site}_auto_loc arrays) + to start from instead of the configured guesses, wherever it + names a site of this model; other sites start at their + guesses. The fit may be of a different model: starting a + sample_offset: level fit from a zero fit's point, say (the npz + can carry sample_offset_offset_auto_loc to set the offsets + too). Refused with checkpoint_file, which sets the start + itself. (default: None) + --stage_offsets STAGE_OFFSETS + 'auto' (default), 'on' or 'off'. A MAP with level tube offsets + (sample_offset: level) started cold can settle in a mode where + the offsets carry the population's growth. Staged, it runs + three MAPs: the offsets held at 0 ({out_prefix}_stage1_*), the + offsets alone with everything else held there + ({out_prefix}_stage2_*), and the joint MAP from that point at + staged_step_size ({out_prefix}_*). A finished stage is reused + when the command is rerun; an interrupted one resumes. 'auto' + stages a fresh MAP of a level-offset model (no + checkpoint_file, no init_from); 'on' requires one; 'off' fits + it in one go. + --staged_step_size STAGED_STEP_SIZE + Starting step size of the staged MAP's joint stage (default + 1e-4): Adam moves every parameter by about a step size per + step, and a larger step can carry the start back into the + offset mode. + +tfs-sample-posterior +-------------------- + +.. code-block:: text + + usage: tfs-sample-posterior [-h] [--out_prefix OUT_PREFIX] [--seed SEED] + [--num_posterior_samples NUM_POSTERIOR_SAMPLES] + [--sampling_batch_size SAMPLING_BATCH_SIZE] + [--forward_batch_size FORWARD_BATCH_SIZE] + [--hessian_chunk_size HESSIAN_CHUNK_SIZE] + [--laplace LAPLACE] + [--laplace_max_params LAPLACE_MAX_PARAMS] + [--skip_growth_observations] + [--genotype_chunk_size GENOTYPE_CHUNK_SIZE] + [--allow_flat_laplace_floor] + config_file checkpoint_file + + Draw posterior samples from an existing MAP, SVI, or NUTS checkpoint. + + Three checkpoint types are handled automatically: + + 1. NUTS checkpoint: runs the forward model over the saved MCMC samples and + writes posterior predictives. + 2. MAP checkpoint (AutoDelta guide): forms a Laplace (Gaussian) approximation + via the Hessian of the log-joint at the MAP point, then draws samples. + 3. SVI checkpoint (component guide): restores the fitted variational state + and draws posterior samples directly from the guide. + + The .h5 file written by this command can be passed directly to + tfs-predict-growth, tfs-predict-theta, and tfs-extract-params to obtain + full posterior uncertainty (quantile columns) on any quantity of interest. + This is the recommended path for obtaining uncertainty estimates from a + MAP-fitted model: fit with tfs-fit-model, sample here, then predict. + + positional arguments: + config_file Path to the YAML configuration file used when fitting the + model. + checkpoint_file Path to the checkpoint .pkl file produced by tfs-fit-model or + tfs-prefit-calibration. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for the posterior output file (default + 'tfs_posterior'). Posterior samples are written to + {out_prefix}.h5. + --seed SEED Random seed used when constructing the RunInference object + (default 0). For SVI and MAP checkpoints the PRNG state is + restored from the checkpoint, so this value has no effect on + the posterior samples. + --num_posterior_samples NUM_POSTERIOR_SAMPLES + Number of posterior samples to draw (default 10000). Not used + for NUTS checkpoints (all MCMC samples are used directly). + --sampling_batch_size SAMPLING_BATCH_SIZE + Parameter sampling batch size (default 100). Not used for + NUTS. + --forward_batch_size FORWARD_BATCH_SIZE + Forward-model batch size for posterior predictives (default + 512). + --hessian_chunk_size HESSIAN_CHUNK_SIZE + Number of Hessian rows computed per device batch for MAP + checkpoints (default 64). Reduce if the Hessian computation + hits device OOM. + --laplace LAPLACE How a MAP checkpoint becomes a posterior (ignored for SVI and + NUTS): - auto (default): full up to laplace_max_params + parameters, arrowhead above. - full: the Laplace from the + dense Hessian, O(D^2) memory. Out of reach on a full library + (millions of parameters). - arrowhead: the full Laplace + computed per genotype. The Hessian is an arrowhead (per- + genotype blocks, their coupling to the shared parameters, the + shared block); the shared parameters (growth k and m, + hyperparameters, tube offsets) are drawn from their marginal + and each genotype from its conditional. Runs on a full + library. A negative direction of the shared block's Schur + complement is held at the MAP; those directions are written to + {out_prefix}_held_directions.csv, since they have no interval. + - blocks: per-genotype blocks only, the shared parameters held + at the MAP. Leaves out the k/m uncertainty. - point: the MAP + point itself, one sample, no Hessian. + --laplace_max_params LAPLACE_MAX_PARAMS + Largest number of MAP parameters for which auto picks the full + Laplace (default 20000). + --skip_growth_observations + Leave the per-observation growth sites (growth_pred, + growth_obs) out of the file (default False). They are most of + its size (94% on a 300-genotype subset; about 100 GB per site + at 500 samples on a 200,000-genotype library), and tfs- + predict-growth recomputes growth from the parameter samples + rather than reading them. + --genotype_chunk_size GENOTYPE_CHUNK_SIZE + Genotypes per Hessian-vector-product pass for the arrowhead + and blocks Laplace (default None, all at once). Lower it on + device OOM. + --allow_flat_laplace_floor + If the prior SDs that floor the Laplace's Hessian eigenvalues + cannot be computed, warn and floor every eigenvalue at a flat + 1e-3 instead of stopping with an error (default False). The + flat floor turns a negative eigenvalue into a direction with + SD ~30 in unconstrained units, so use it only to get a + posterior out of a model whose prior SDs fail for a reason you + understand. + +tfs-extract-params +------------------ + +.. code-block:: text + + usage: tfs-extract-params [-h] [--out_prefix OUT_PREFIX] config_file param_file + + Extract parameters from a checkpoint or posterior file and write to CSV. + + If param_file ends with .pkl it is treated as a MAP (AutoDelta) + checkpoint: parameters are converted to constrained space and written with + a single q0.5 column. + + Otherwise it is treated as a posterior samples .h5 or .npz file + produced by tfs-sample-posterior, and 17 quantile columns are written: + q0.001, q0.005, q0.01, q0.025, q0.05, q0.1, + q0.159, q0.25, q0.5, q0.75, q0.841, q0.9, + q0.95, q0.975, q0.99, q0.995, q0.999. + + Writes one CSV per parameter group named {out_prefix}_{param_name}.csv + (e.g. tfs_params_activity.csv, tfs_params_theta.csv; thermo PddG/PnnC + also give one ddG_{structure} and ddG_prior_mean_{structure} + table per structure, both deterministic sites that need a posterior + file, plus one ddG_sigma table of the per-structure SD σ_s keyed by + structure; the ΔΔG is computed from the fitted constants, + thermo/prior.py). A model with its own learned + pyro.param sites (the thermo PnnC NN weights) also gets + {out_prefix}_model_params.csv: one row per element (parameter, + index, value). + + positional arguments: + config_file Path to the YAML configuration file used when fitting the + model. + param_file Path to a MAP checkpoint .pkl file, or a posterior .h5/.npz + file produced by tfs-sample-posterior. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for output CSV files. Default 'tfs_params'. + +tfs-predict-theta +----------------- + +.. code-block:: text + + usage: tfs-predict-theta [-h] [--out_prefix OUT_PREFIX] + [--genotypes_file GENOTYPES_FILE] + [--titrant_names_file TITRANT_NAMES_FILE] + [--titrant_concs_file TITRANT_CONCS_FILE] [--only_files] + [--num_samples NUM_SAMPLES] + [--genotype_batch_size GENOTYPE_BATCH_SIZE] + [--shared_shift SHARED_SHIFT] [--shift_seed SHIFT_SEED] + config_file param_file + + Predict operator occupancy (theta) from a fitted hierarchical model. + + By default predicts theta at all (genotype, titrant_name, titrant_conc) + combinations present in the training data, unioned with any genotypes or + titrant (name, conc) pairs supplied via file arguments. Pass --only_files + to predict exclusively at the file-specified inputs and skip training-data + combinations. + + titrant_names_file and titrant_concs_file together define a paired + (name, conc) grid that is unioned with the training titrant grid (or + replaces it when --only_files is set). They must always be provided + together or both omitted. + + When any requested genotype was not seen during training, + extract_theta_unmeasured is used. + + A boolean column 'in_training_data' is added to the output: 1 if the + (genotype, titrant_name, titrant_conc) triple was in the training data, + 0 otherwise. + + A relative theta component (hill_relative) predicts the wt-relative + growth variable X, not theta; its output gets a column theta_scale + with the value X so nothing downstream takes it for an occupancy. + + positional arguments: + config_file Path to the YAML configuration file. + param_file Path to a posterior .h5 file produced by tfs-sample-posterior, + or a MAP checkpoint .pkl file produced by tfs-fit-model. When + a .pkl file is supplied the output has a single q0.5 column + (the MAP point) and no uncertainty. To obtain uncertainty + estimates from a MAP fit, first run tfs-sample-posterior on + the .pkl checkpoint; it builds a Laplace approximation and + writes a .h5 file. Passing that .h5 here gives the full ladder + of bare q quantile columns (q0.025, q0.5, q0.975, ...). + NUTS and SVI checkpoints are not supported directly; run tfs- + sample-posterior first. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for the output CSV file. Written to {out_prefix}.csv. + Default 'tfs_pred_theta'. + --genotypes_file GENOTYPES_FILE + Plain-text file with one genotype per line (slash-separated + mutations, e.g. 'M42I/K84L', or 'wt'). These genotypes are + unioned with all training genotypes unless --only_files is + set. May include genotypes not seen during training when the + theta component supports predict_unmeasured. Default None. + --titrant_names_file TITRANT_NAMES_FILE + Plain-text file with one titrant name per line. Must be + provided together with titrant_concs_file (or both omitted). + The resulting (name, conc) pairs are unioned with the training + titrant grid unless --only_files is set. A single name is + broadcast across all concentrations in titrant_concs_file. + Default None. + --titrant_concs_file TITRANT_CONCS_FILE + Plain-text file with one concentration per line. Must be + provided together with titrant_names_file (or both omitted). + Default None. + --only_files If True, predict only at the genotypes and titrant (name, + conc) pairs supplied via file arguments, ignoring training- + data combinations. Default False. + --num_samples NUM_SAMPLES + Number of joint posterior samples to include as sample_0 … + sample_N-1 columns alongside the quantile columns. 0 (default) + writes quantiles only. + --genotype_batch_size GENOTYPE_BATCH_SIZE + When predicting unmeasured genotypes, process this many at a + time to cap the memory used by the epistasis pair-indicator + matrix (batch_size × N_pair × 4 bytes). Smaller values reduce + peak memory at the cost of more iterations. Default 2000. + --shared_shift SHARED_SHIFT + CSV of a shift shared by every non-wt genotype (columns + genotype, titrant_name, titrant_conc, shift), written by the + two-stage fit: the X change from one standard deviation of + stage 1's k. Each posterior draw gets one standard-normal + multiple of it, the same for every row, before quantiles are + taken, so each genotype's interval carries k's uncertainty + while differences between genotypes keep only what does not + cancel. wt (fixed by the X gauge) does not move. Needs a + posterior file with draws (not a MAP checkpoint) and training + genotypes only. Default None. + --shift_seed SHIFT_SEED + Seed for the shared per-draw multiples. Default 0. + +tfs-predict-growth +------------------ + +.. code-block:: text + + usage: tfs-predict-growth [-h] [--out_prefix OUT_PREFIX] + [--genotypes_file GENOTYPES_FILE] + [--titrant_names_file TITRANT_NAMES_FILE] + [--titrant_concs_file TITRANT_CONCS_FILE] [--only_files] + [--num_samples NUM_SAMPLES] + [--num_marginal_samples NUM_MARGINAL_SAMPLES] + [--genotype_batch_size GENOTYPE_BATCH_SIZE] + [--subset_genotypes] [--seed SEED] + config_file param_file + + Predict growth signal (ln_cfu) from a fitted hierarchical model. + + By default predicts at all (genotype, titrant_name, titrant_conc, + replicate, t_pre, t_sel) combinations present in the training data, unioned + with any genotypes or concentrations supplied via file arguments. Pass + --only_files to predict exclusively at the file-specified inputs and skip + training-data combinations. + + titrant_names_file is a post-prediction row filter (restrict-only): it + narrows which titrant names appear in the output but does not affect which + concentrations are predicted. It is not subject to union semantics. + + A boolean column 'in_training_data' is added to the output: 1 if the + (genotype, titrant_name, titrant_conc) triple was in the training data, + 0 otherwise. + + positional arguments: + config_file Path to the YAML configuration file. + param_file Path to a posterior .h5 file produced by tfs-sample-posterior, + or a MAP checkpoint .pkl file produced by tfs-fit-model. When + a .pkl file is supplied the output has a single q0.5 column + (the MAP point) and no uncertainty. To obtain uncertainty + estimates from a MAP fit, first run tfs-sample-posterior on + the .pkl checkpoint; it builds a Laplace approximation and + writes a .h5 file. Passing that .h5 here gives the full ladder + of bare q quantile columns (q0.025, q0.5, q0.975, ...). + NUTS and SVI checkpoints are not supported directly; run tfs- + sample-posterior first. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for the output CSV file. Written to {out_prefix}.csv. + Default 'tfs_pred_growth'. + --genotypes_file GENOTYPES_FILE + Plain-text file with one genotype per line (slash-separated + mutations, e.g. 'M42I/K84L', or 'wt'). These genotypes are + unioned with all training genotypes unless --only_files is + set. Default None. + --titrant_names_file TITRANT_NAMES_FILE + Plain-text file with one titrant name per line. Filters output + rows to the named titrants (restrict-only; does not affect + which concentrations are predicted). If None, all titrant + names are included. (default: None) + --titrant_concs_file TITRANT_CONCS_FILE + Plain-text file with one concentration per line. These + concentrations are unioned with all training concentrations + unless --only_files is set. Default None. + --only_files If True, predict only at the genotypes and concentrations + supplied via file arguments, ignoring training-data + combinations. Default False. + --num_samples NUM_SAMPLES + Number of joint posterior samples to include as sample_0 … + sample_N-1 columns alongside the quantile columns. 0 (default) + writes quantiles only. + --num_marginal_samples NUM_MARGINAL_SAMPLES + Number of posterior samples to run through the model when + computing quantiles. If None, all available samples are used. + (default: None) + --genotype_batch_size GENOTYPE_BATCH_SIZE + Maximum number of genotypes per predict() call. The genotype + list is split into chunks of this size, each chunk is + predicted separately, and the results are concatenated. + Reduces peak memory at the cost of one JAX re-compilation per + batch. If None (default), a batch size is estimated + automatically from the available device memory and the per- + genotype tensor cost; pass an explicit value to override. + --subset_genotypes If True, predict only a single memory-fit block of genotypes + instead of every genotype. The block size is the auto-sized + (or explicit) genotype_batch_size; the block always includes + the binding genotypes, the spiked genotypes, and any genotypes + supplied via genotypes_file, with the remainder of the block + filled by a random sample of the other genotypes. Intended for + quickly assessing the input/output ln_cfu correlation without + paying for a full prediction sweep. Default False. + --seed SEED Seed for the random draw used by subset_genotypes, making the + sampled block reproducible. Ignored unless subset_genotypes is + True. Default None (non-deterministic draw). + +tfs-predict-epistasis +--------------------- + +.. code-block:: text + + usage: tfs-predict-epistasis [-h] [--out_prefix OUT_PREFIX] + [--genotypes_file GENOTYPES_FILE] + [--titrant_names_file TITRANT_NAMES_FILE] + [--titrant_concs_file TITRANT_CONCS_FILE] [--only_files] + [--scale SCALE] [--scale_constant SCALE_CONSTANT] + [--regime_eps REGIME_EPS] [--regime_ci REGIME_CI] + [--shared_shift SHARED_SHIFT] [--shift_seed SHIFT_SEED] + config_file param_file + + Predict second-order epistasis from the joint posterior of a fitted model. + + For every double mutant, epistasis is calculated on its mutant cycle (the + double, its two single-mutant parents, and the wildtype). In contrast to + running tfs-extract-epistasis on a tfs-predict-theta table -- which uses the + per-genotype marginal theta estimates and assumes the four corners are + independent -- this command draws theta for all genotypes from the same + posterior sample, computes epistasis within each draw, and then reports + quantiles across draws. The uncertainty therefore reflects the true + posterior covariance between the corners of each cycle. + + Epistasis is computed independently at each condition (titrant_name, + titrant_conc). Output columns are 'genotype', 'titrant_name', + 'titrant_conc', one 'q' column per quantile (e.g. 'q0.5', 'q0.025', + 'q0.975' -- the library-wide quantile-output convention), and a trailing + 'in_regime' flag (int 0/1), written to {out_prefix}.csv. + + 'in_regime' marks whether the estimate is backed by data or by model + extrapolation. It is 1 only when all four corners of the mutant cycle (wt, + both singles, double) have their theta posterior (central 95% interval by + default) inside the resolvable band [regime_eps, 1 - regime_eps]. When a + corner's theta is near saturation (0 or 1), logit(theta) is compressed and + the growth data constrain it weakly, so the epistasis there leans on the + theta-model's functional form and cross-genotype posterior covariance -- + treat in_regime == 0 rows as model-conditional. (This checks posterior mass + only; it does not separately test whether the growth signal exceeds the + growth noise.) + + Only genotypes seen during training are supported (the joint sample matrix + is built from the training theta curves); requesting an out-of-training + genotype is an error. + + positional arguments: + config_file Path to the YAML configuration file. + param_file Path to a posterior .h5 file produced by tfs-sample-posterior, + or a MAP checkpoint .pkl file produced by tfs-fit-model. A + .pkl provides a single point estimate, so every q + column collapses to that value (no uncertainty); run tfs- + sample-posterior first to obtain a Laplace posterior and real + quantiles. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for the output CSV file. Written to {out_prefix}.csv. + Default 'tfs_pred_epistasis'. + --genotypes_file GENOTYPES_FILE + Plain-text file with one genotype per line (slash-separated + mutations, e.g. 'M42I/K84L', or 'wt'). Restricts the analysis + to cycles whose genotypes appear here (unioned with all + training genotypes unless --only_files is set). All genotypes + must have been seen during training. Default None. + --titrant_names_file TITRANT_NAMES_FILE + Plain-text file with one titrant name per line. Must be + provided together with titrant_concs_file (or both omitted). + The resulting (name, conc) pairs are unioned with the training + titrant grid unless --only_files is set. A single name is + broadcast across all concentrations in titrant_concs_file. + Default None. + --titrant_concs_file TITRANT_CONCS_FILE + Plain-text file with one concentration per line. Must be + provided together with titrant_names_file (or both omitted). + Default None. + --only_files If True, use only the genotypes and titrant (name, conc) pairs + supplied via file arguments, ignoring training-data + combinations. Default False. + --scale SCALE Epistatic scale. 'logit' (default) computes additive epistasis + of logit(theta) -- the natural scale for an occupancy in [0, + 1]. 'add': (Y11 - Y10) - (Y01 - Y00). 'mult': (Y11 / Y10) / + (Y01 / Y00). A relative fit (theta 'hill_relative') gives X, + not theta: it takes only 'add', and its output has no + 'in_regime' column. + --scale_constant SCALE_CONSTANT + Constant applied to the transform before the difference-of- + differences; multiplies the reported epistasis. Default 1.0. + Mainly a unit conversion for scale='logit': since logit(theta) + = -dG/RT, passing -RT (e.g. -0.6159 for kcal/mol at 310.15 K) + reports epistasis as an interaction free energy. Has no effect + on (and is rejected for) scale='mult'. + --regime_eps REGIME_EPS + Theta resolution floor for the 'in_regime' flag. A cycle + corner counts as resolvable when its theta posterior sits in + [regime_eps, 1 - regime_eps]. Default 0.01 (matching a 1%-in- + theta resolution, whose logit band is +/- ~4.6). Must be in + [0, 0.5). + --regime_ci REGIME_CI + Central posterior-mass fraction required inside the band for a + corner to count as in-regime. Default 0.95 (checks the theta + 2.5-97.5% interval). Must be in (0, 1). + --shared_shift SHARED_SHIFT + CSV of a shift shared by every non-wt genotype (genotype, + titrant_name, titrant_conc, shift), written by the two-stage + fit: the X change from one standard deviation of stage 1's k. + Each joint draw gets one standard-normal multiple of it before + epistasis is computed. The cycle's wt corner does not move, so + an epistasis on X (X_ab - X_a - X_b + X_wt) keeps one net copy + of the shift. Needs a posterior file with draws. Default None. + --shift_seed SHIFT_SEED + Seed for the shared per-draw multiples. Default 0. + +tfs-extract-epistasis +--------------------- + +.. code-block:: text + + usage: tfs-extract-epistasis [-h] [--y_obs Y_OBS] [--out_prefix OUT_PREFIX] + [--y_std Y_STD] [--group_by GROUP_BY [GROUP_BY ...]] + [--scale SCALE] [--scale_constant SCALE_CONSTANT] + [--keep_extra] [--logit_eps LOGIT_EPS] + data_file + + Calculate second-order epistasis for pairs of mutations. + + Reads a long-form CSV (one row per genotype, with genotypes in the format + 'MUT1/MUT2') and, for every double mutant, builds a mutant cycle from its + two single-mutant parents and the wildtype. Epistasis is calculated on the + requested observable and written, one row per double mutant, to + {out_prefix}.csv. + + positional arguments: + data_file Path to the input CSV. Must contain a 'genotype' column, the + column named by y_obs, and (if given) the columns named by + y_std and group_by. + + options: + -h, --help show this help message and exit + --y_obs Y_OBS Name of the column holding the observable for which epistasis + is calculated (e.g. 'fitness', 'dG'). If None (default) and + the input has a 'q0.5' column (as written by tfs-predict- + theta), 'q0.5' is used. + --out_prefix OUT_PREFIX + Prefix for the output CSV file, written to {out_prefix}.csv. + Default 'tfs_epistasis'. + --y_std Y_STD Name of the column holding the standard error of y_obs. If + given, the epistasis error (ep_std) is propagated and written. + If None (default) and both 'q0.841' and 'q0.159' are present, + the standard error is taken as (q0.841 - q0.159) / 2. + --group_by GROUP_BY [GROUP_BY ...] + One or more column names that define a unique experimental + condition. Epistasis is calculated independently within each + condition. If omitted, the whole table is treated as a single + condition. (default: None) + --scale SCALE Epistatic scale. "add" (default): (Y11 - Y10) - (Y01 - Y00). + "mult": (Y11 / Y10) / (Y01 / Y00). "logit": additive epistasis + of logit(Y) — requires an observable in (0, 1), such as + theta/occupancy. + --scale_constant SCALE_CONSTANT + Constant applied to the transformed observable before + epistasis is taken; multiplies ep_obs (and abs() multiplies + ep_std). Default 1.0. Mainly a unit conversion for + scale="logit": since logit(theta) = -dG/RT, passing -RT (e.g. + -0.6159 for kcal/mol at 310.15 K) reports epistasis as an + interaction free energy. The caller owns the sign, + temperature, and gas constant/units. Has no effect on (and is + rejected for) scale="mult". + --keep_extra If True, retain all columns from the input CSV in the output. + If False (default), keep only the identifier columns and the + calculated epistasis values. + --logit_eps LOGIT_EPS + Only used when scale="logit". Observations are clamped to + [logit_eps, 1 - logit_eps] before the transform. Default 1e-9. + +tfs-cat-response +---------------- + +.. code-block:: text + + usage: tfs-cat-response [-h] [--y_obs Y_OBS] [--out_prefix OUT_PREFIX] [--y_std Y_STD] + [--group_by GROUP_BY [GROUP_BY ...]] + [--models MODELS [MODELS ...]] [--alpha ALPHA] + [--select_by SELECT_BY] [--adequacy_alpha ADEQUACY_ALPHA] + [--curvy_cutoff CURVY_CUTOFF] [--rope_cutoff ROPE_CUTOFF] + [--rope_multiplier ROPE_MULTIPLIER] [--write_all_predictions] + [--num_workers NUM_WORKERS] + data_file x_obs + + Classify each group's response curve using categorical response models. + + Reads a long-form CSV and fits one or more response models to every group. + Groups are defined by the 'genotype' column plus any --group_by columns. For + each group the best model is selected per --select_by (default the 'shape' + classifier), then graded against zero on the observed data (per-point + sig_nonzero + a per-curve data-based nonzero test with a Benjamini-Hochberg + FDR correction) to assign the 'fittable' bool. + + Writes: + - {out_prefix}.csv one row per group; all models' weights and + parameter estimates, plus the assessment + rollups (nonzero_p/q, omnibus_p/q, + n_nonzero, all_equiv_zero, fittable). + - {out_prefix}_{model}.csv one file per model; that model's parameter + table and per-group fit statistics. + - {out_prefix}_predictions.csv best-model predicted curves (all models + when --write_all_predictions). + - {out_prefix}_assessment.csv per (group, x) best-model assessment. + + positional arguments: + data_file Path to the input CSV. Must contain a 'genotype' column, + x_obs, y_obs, and (if given) y_std and every column in + group_by. + x_obs Name of the column holding the independent variable (e.g. + 'titrant_conc'). + + options: + -h, --help show this help message and exit + --y_obs Y_OBS Name of the column holding the observable (e.g. 'q0.5'). If + None (default) and the input has a 'q0.5' column (as written + by tfs-predict-theta), 'q0.5' is used. + --out_prefix OUT_PREFIX + Prefix for the output CSV files. Default 'tfs_cat_response'. + --y_std Y_STD Name of the column holding per-row uncertainty (standard + deviation). If None (default) and both 'q0.841' and 'q0.159' + are present, sigma is computed as (q0.841 - q0.159) / 2; + otherwise the fit is unweighted. + --group_by GROUP_BY [GROUP_BY ...] + Additional column(s) that, together with 'genotype', define a + group. If omitted, groups are defined by 'genotype' alone. + (default: None) + --models MODELS [MODELS ...] + Response models to fit. If None (default), the curated + DEFAULT_MODELS set is used (flat, linear_log, repressor, + inducer, bell_peak_log, bell_dip_log). Pass explicit names to + reach any model in MODEL_LIBRARY, including the raw-x and + biphasic variants. + --alpha ALPHA Significance level with two roles: the per-point sig_nonzero + test, and the nonzero_q threshold used to call a curve 'real'. + Default 0.05. + --select_by SELECT_BY + Model-selection strategy. "shape" (default) is the liberal + shape classifier: it gates flat-vs-curvy on the flat fit's + residual autocorrelation and names the curvy shape by best R2 + (defaults to the physical SHAPE_MODELS vocabulary -- no + linear, includes biphasic). "aicc" selects the lowest-AICc + model. "adequacy" keeps the AICc pick unless flagged, then + escalates to a no-simpler adequate model (never demotes). + --adequacy_alpha ADEQUACY_ALPHA + Runs-test threshold for the shape_status diagnostic and, when + select_by="adequacy", for escalation. Default 0.05. + --curvy_cutoff CURVY_CUTOFF + Only used when select_by="shape": flat-vs-curvy gate on the + flat fit's residual-autocorrelation p-value (larger = more + curves called curvy). Sweep it and visually inspect. Default + 0.1. + --rope_cutoff ROPE_CUTOFF + ROPE half-width separating confident_zero from indeterminate. + If None (default), auto-derived as rope_multiplier * + median(observed y_std) -- a detectability threshold that + rarely fires confident_zero; pass a fixed, biologically- + meaningful value to make it fire. + --rope_multiplier ROPE_MULTIPLIER + Multiplier used when rope_cutoff is auto-derived. Default 2.0. + --write_all_predictions + If True, write every fit model's predicted curve rather than + only the best model's. Default False. + --num_workers NUM_WORKERS + Number of parallel worker processes. 1 runs serially; -1 (the + default) uses os.cpu_count() - 1; N uses N processes. + +tfs-compare-runs +---------------- + +.. code-block:: text + + usage: tfs-compare-runs [-h] [--out_prefix OUT_PREFIX] [--reference REFERENCE] + [--index_by INDEX_BY] [--group_by GROUP_BY [GROUP_BY ...]] + [--match_by MATCH_BY [MATCH_BY ...]] [--y_obs Y_OBS] + [--y_std Y_STD] [--point_quantile POINT_QUANTILE] + [--sigma_quantiles SIGMA_QUANTILES SIGMA_QUANTILES] + [--no_aggregate] + estimates [estimates ...] + + Measure cross-run agreement of an estimate across N independent runs. + + Reads N estimate CSVs of the same quantity -- predicted features + (tfs-predict-theta, tfs-predict-growth, tfs-predict-epistasis) or fitted + parameters (tfs-extract-params) alike -- and reports, for every entity, how + much the runs disagree (reproducibility, in native units) and whether that + disagreement is explained by each run's reported uncertainty + (self-consistency, unit-free). + + No thresholds are applied and no grades are assigned: every quantity is a + number, so filter downstream (e.g. `df["overdispersion"] > 2`) where the + cutline is recorded alongside the analysis that used it. + + Writes {out_prefix}.csv (one row per entity, or per entity x group_by), + {out_prefix}_aggregate.csv (unless --no_aggregate), and + {out_prefix}_metadata.json (every resolved setting, so the run is + reconstructable). + + positional arguments: + estimates Either two or more estimate CSV paths (e.g. `rep1.csv + rep2.csv`), or a single path to a manifest file listing one + CSV path per line ('#' comments allowed). A single argument + ending in '.csv' is treated as a direct path, not a manifest. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for the output files. Default 'tfs_compare_runs'. + --reference REFERENCE + Path to a reference-run CSV. If given, switches to reference + mode: each estimate run is scored by its deviation from this + run (e.g. k-fold dropouts vs. a full-data fit). If omitted, + runs are compared to their symmetric cross-run mean. (default: + None) + --index_by INDEX_BY The entity column being scored. Default None: auto-detect + 'genotype', else 'parameter', else the sole match-key column. + --group_by GROUP_BY [GROUP_BY ...] + Extra columns to break each entity out by (e.g. `titrant_name + titrant_conc`). Must be part of the match key. This is a + statistical zoom: the finer the grouping, the fewer degrees of + freedom behind every statistic. Check the n_rows / n_eff + columns. (default: None) + --match_by MATCH_BY [MATCH_BY ...] + The columns that make a row 'the same row' across runs. + Default None: auto-detect every shared non-value, non- + bookkeeping column. + --y_obs Y_OBS Point-estimate column. Default None, which uses the + --point_quantile column (q0.5). + --y_std Y_STD Uncertainty column. Default None, which uses the symmetric + quantile half-width (q0.841 - q0.159)/2. If neither is + available the self-consistency axis is reported as NaN. + --point_quantile POINT_QUANTILE + Quantile used as the point estimate. Default 0.5. + --sigma_quantiles SIGMA_QUANTILES SIGMA_QUANTILES + Two ascending quantiles bracketing one sigma. Default (0.159, + 0.841). + --no_aggregate Suppress {out_prefix}_aggregate.csv, the long-form table + combining the N runs as an equal-weight mixture of their per- + run posteriors (folding in both per-run posterior width and + run-to-run spread; it does not shrink with N). Only the N + estimate runs are mixed -- a reference run is never included. + Aggregation needs at least two shared q columns and is + skipped with a warning when they are absent. It is also the + slowest step on a large library, so this is the escape hatch. + +tfs-summarize-fit +----------------- + +.. code-block:: text + + usage: tfs-summarize-fit [-h] [--ref_theta_file REF_THETA_FILE] + [--out_prefix OUT_PREFIX] + run_dir + + Evaluate theta prediction quality for a tfs-fit-model run directory. + + Scans run_dir for a *_config.yaml, *_pred_theta.csv, *_pred_growth.csv + (optional), and *_losses.txt (optional). Computes prediction statistics + and writes output files to the summary directory. + + **CSV files are the authoritative outputs.** Every plot written by this + function has a matched CSV containing the exact data used to generate it. + The PDFs are human-readable summaries; the CSVs are the record of truth + for downstream quantitative analysis. + + Output files written: + + - {out_prefix}_fit_summary.json — nested statistics with top-level + keys metadata, theta (training / test sub-keys), and growth + (training sub-key). + - {out_prefix}_theta_corr_training.csv — joined (ref, predicted) + theta pairs for training genotypes; ref column holds the training + observation used as ground truth. + - {out_prefix}_theta_corr_test.csv — same for test genotypes (only + written when a ref theta file is resolved and has matching rows). + - {out_prefix}_theta_corr.pdf — two-panel correlation plot for theta + (training left, test right). + - {out_prefix}_growth_corr.csv — a copy of the *_pred_growth.csv in + run_dir with ln_cfu/ln_cfu_std renamed ref/ref_std + (only written when that file is present). + - {out_prefix}_growth_corr.pdf — correlation plot for ln_cfu (only + written when *_pred_growth.csv is present). + - {out_prefix}_{genotype}_theta_fits.csv / .pdf — per-genotype + joined observation + prediction data and fit plot (one pair per + genotype present in both binding and prediction CSVs). + - {out_prefix}_{genotype}_trajectory.csv / .pdf — per-genotype + predicted growth trajectory data and plot (only when growth data is + configured and a posterior/params file is found). + - {out_prefix}_losses.pdf — training loss curve (only written when + *_losses.txt is present). + - {out_prefix}_tube_offsets.csv / _tube_offset_trends.csv / + _tube_offsets.pdf — the per-tube sample offsets in prior SDs and, + per condition, their Spearman trend with titrant and with time (only + when tfs-extract-params wrote *_sample_offset_offset.csv). A + condition whose trend has BH q < 0.05 is flagged structured: the + offsets are carrying growth (the offset mode) rather than tube noise. + The JSON's tube_offsets key holds the summary. + + positional arguments: + run_dir Directory containing model fit outputs. + + options: + -h, --help show this help message and exit + --ref_theta_file REF_THETA_FILE + CSV with columns genotype, titrant_name, titrant_conc, and a + theta column containing known theta values for evaluating out- + of-sample prediction. When omitted the function looks for a + *_sim_genotype_theta.csv file in *run_dir* automatically (the + file produced by tfs-simulate). The theta column is resolved + in order: theta_obs (preferred) → theta (fallback); it is + renamed to ref in the output CSV. If neither column is present + a warning is issued and test statistics are skipped. Pass this + argument explicitly to use a theta reference from outside + *run_dir* or with a non-standard name. (default: None) + --out_prefix OUT_PREFIX + Prefix for output files. Defaults to + {run_dir}/summary/tfs_summarize. + +tfs-fit-genotypes +----------------- + +.. code-block:: text + + usage: tfs-fit-genotypes [-h] [--out_prefix OUT_PREFIX] + [--intercept_cols INTERCEPT_COLS] + [--dk_geno_prior_sd DK_GENO_PRIOR_SD] [--min_obs MIN_OBS] + [--num_workers NUM_WORKERS] + growth_df growth_calibration_file + + Fit the growth model to each genotype independently (frozen calibration). + + positional arguments: + growth_df Processed growth CSV (tfs-process-counts output). + growth_calibration_file + Frozen per-condition growth calibration: a tfs-prefit- + calibration priors CSV, or a wide + condition_rep,growth_k,growth_m CSV. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Output prefix (see the module docstring for the files + written). (default: 'tfs_fit_genotypes') + --intercept_cols INTERCEPT_COLS + Comma-separated columns whose unique combinations each get a + nuisance ln_cfu0 (default "replicate"; empty string -> single + intercept). + --dk_geno_prior_sd DK_GENO_PRIOR_SD + Std of the weak Normal prior on dk_geno (<=0 disables it). + (default: 1.0) + --min_obs MIN_OBS Skip genotypes with fewer than this many usable observations. + (default: None) + --num_workers NUM_WORKERS + Parallelize the per-genotype fits over a process pool: 1 + (default) serial; -1 uses os.cpu_count() - 1; N uses N. + +tfs-sample-prior +---------------- + +.. code-block:: text + + usage: tfs-sample-prior [-h] [--out_prefix OUT_PREFIX] [--num_datasets NUM_DATASETS] + [--no_noise] [--seed SEED] + config_file + + Draw synthetic datasets from the model prior. + + For each dataset, two files are written: + + _NNN_growth.csv + Synthetic growth DataFrame with ln_cfu replaced by prior + predictions. Observation noise drawn from Normal(0, ln_cfu_std) + is added unless --no_noise is given. Pass it to + tfs-configure-model --growth_df to fit it. + + _NNN_ground_truth.h5 + The latent parameters used to generate that dataset, in the same + HDF5 format as a tfs-sample-posterior output. Pass this to + tfs-predict-growth or tfs-extract-params to inspect the + ground-truth values, or compare it to the fitted posterior for + parameter recovery checks. + + When --num_datasets 1 (the default) the _NNN index is still + included in the filenames for consistency. + + positional arguments: + config_file Path to the YAML configuration file (same one used for tfs- + fit-model). + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for all output files. Default 'tfs_prior'. + --num_datasets NUM_DATASETS + Number of independent synthetic datasets to generate. Default + 1. Set to a larger value (e.g. 100) for Simulation-Based + Calibration. + --no_noise Turn off (on by default): If True (default), add Normal(0, + ln_cfu_std) noise to each synthetic observation. Set to False + to get the deterministic prior predictions. + --seed SEED Base random seed. Dataset i uses seed + i to ensure + independence. Default 0. + +tfs-simulate +------------ + +.. code-block:: text + + usage: tfs-simulate [-h] [--out_prefix OUT_PREFIX] [--num_replicates NUM_REPLICATES] + [--seed SEED] [--no_write_raw] [--no_write_growth] + config_file + + Simulate a TF selection experiment from a YAML configuration file. + + Runs library_prediction once to establish ground-truth phenotypes, then + simulates num_replicates independent experimental replicates using + selection_experiment. Writes library, parameters, genotype_theta (long-form: + genotype/titrant_name/titrant_conc/theta), growth, growth_parameters + (per-condition condition_growth ground truth; see + tfscreen.simulate.growth_parameters_output.generate_growth_parameters_df), + and transformation_lam (single-row congression Poisson-rate ground truth; + see tfscreen.simulate.transformation_lam_output.generate_transformation_lam_df) + CSV files. A mutation-decomposed theta model drawn from its prior (the + thermodynamic models) also writes mutation_parameters (one row per + mutation: the true d_* deltas and, for PddG/PnnC, ddG_{structure}) and, + when the library has multi-mutants, pair_parameters (one row per pair: + the true epi_* terms); see + tfscreen.simulate.thermo_to_growth._mutation_truth_tables. PddG/PnnC + also write ddG_sigma (one row per structure: the true SD of ΔΔG about + the structural prior mean, set by the config's ddG_truth_sd, 1 kT by + default; see _ddG_sigma_truth_table). If the config + contains a 'binding_data' block, also writes a + simulated binding curve CSV (see + tfscreen.simulate.binding_data.generate_binding_df). If it contains a + 'base_growth_data' block, also writes a simulated direct growth-rate + calibration CSV and a single-row k_ref ground-truth CSV (see + tfscreen.simulate.base_growth_data.generate_base_growth_df and + .generate_k_ref_df). If it contains a 'presplit_data' block, also writes + a simulated presplit CSV (see + tfscreen.simulate.presplit_data.generate_presplit_df). If it contains an + 'od600' block, also writes one OD600 reading per tube for every + replicate, sequenced or not ('od600'; see tfscreen.simulate.od600), and + simulates od600.num_od_only_replicates extra replicates that get OD600 + but no reads. With shared_transformation, every replicate (OD-only ones + included) draws its cells from one library assembly and transformation. + + With library_genotypes and/or library_subsample the library is + restricted to a subset of the described genotypes before anything is + simulated ({out_prefix}_library.csv lists the kept ones); with the + default library_subsample_depth: full_library each kept sequence + keeps the pool share, transformants, reads and PCR templates it has in + the full library. See tfscreen.genetics.library_subsample. + + positional arguments: + config_file Path to the YAML run configuration file. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Output prefix: files are written as {out_prefix}_{name}.csv. A + directory part (sim/tfs_sim) is created if absent. (default: + 'tfs_sim') + --num_replicates NUM_REPLICATES + Number of independent experimental replicates to simulate. + Default 2. + --seed SEED Random seed. Overrides seed in the config file when provided. + (default: None) + --no_write_raw Turn off (on by default): Also write the experiment in the raw + formats real data come in (one count file per tube in + {out_prefix}_counts/, the tube table {out_prefix}_tubes.csv + and, with an od600 block, the OD600 table and the + calibration), so tfs-process-counts processes it exactly as it + would a lab's data. The command is printed. See + tfscreen.simulate.raw_output. + --no_write_growth Turn off (on by default): Also write {out_prefix}_growth.csv, + the growth table built directly from the simulated counts + (with each row's true values). tfs-process-counts builds the + same table from the raw files, so a full-size simulation can + skip it: on a real library's size it is several GB. With an + od600 block, tubes read below the calibration's detection + threshold are left out of it, as they are out of the raw tube + table (tfs-process-counts refuses them). + +tfs-build-empirical +------------------- + +.. code-block:: text + + usage: tfs-build-empirical [-h] [--binding_df BINDING_DF] [--out_prefix OUT_PREFIX] + [--seed SEED] + [--growth_calibration_file GROWTH_CALIBRATION_FILE] + [--library_config LIBRARY_CONFIG] + [--base_growth_df BASE_GROWTH_DF] + [--thermo_data THERMO_DATA] + [--intercept_cols INTERCEPT_COLS] + [--dk_geno_prior_sd DK_GENO_PRIOR_SD] [--min_obs MIN_OBS] + [--no_drop_railed] [--num_workers NUM_WORKERS] + growth_df + + Fit real screen data to an empirical phenotype generating distribution. + + positional arguments: + growth_df Processed growth CSV (tfs-process-counts output) for real + data. + + options: + -h, --help show this help message and exit + --binding_df BINDING_DF + Real binding CSV. Required unless growth_calibration_file is + given; it identifies the growth slope m from data (the in- + library anchors). (default: None) + --out_prefix OUT_PREFIX + Output prefix. Writes _phenotype_model.json (the + generating distribution) and _stage1_fits.csv (the + per-genotype fits that feed Stage 2). Unless + growth_calibration_file is given, also the + _configure_* / _prefit_* intermediates. + (default: 'tfs_empirical') + --seed SEED Random seed for the pre-fit MAP calibration. Required unless + growth_calibration_file is given. (default: None) + --growth_calibration_file GROWTH_CALIBRATION_FILE + Skip the configure+prefit step and use this calibration + directly — a prefit priors CSV or a wide + condition_rep,growth_k,growth_m CSV. Use this to iterate on + the Stage-1/2 knobs without re-running the MAP. (default: + None) + --library_config LIBRARY_CONFIG + Library YAML describing the screened library (the same file + handed to tfs-process-fastq), forwarded to configure_model. + Required unless growth_calibration_file is given. (default: + None) + --base_growth_df BASE_GROWTH_DF + Direct growth-rate calibration CSV forwarded to + configure_model. (default: None) + --thermo_data THERMO_DATA + Thermodynamic data path forwarded to configure_model (unused + by the default hill_geno theta). + --intercept_cols INTERCEPT_COLS + Comma-separated columns whose unique combinations each get a + nuisance ln_cfu0 (default "replicate"; empty string -> single + intercept). + --dk_geno_prior_sd DK_GENO_PRIOR_SD + Std of the weak Normal prior on dk_geno (<=0 disables it). + (default: 1.0) + --min_obs MIN_OBS Skip genotypes with fewer than this many usable observations. + (default: None) + --no_drop_railed Turn off (on by default): Drop theta-logit-railed Stage-1 fits + before the Stage-2 fit. + --num_workers NUM_WORKERS + Parallelize the Stage-1 per-genotype fits over a process pool: + 1 (default) serial; -1 uses os.cpu_count() - 1; N uses N. + Recommended for large libraries (the fits are embarrassingly + parallel). + +tfs-setup-sim-grid +------------------ + +.. code-block:: text + + usage: tfs-setup-sim-grid [-h] [--out_dir OUT_DIR] grid_yaml + + Set up a directory grid for simulation runs. + + Reads a grid YAML file, loads a base simulate config, expands all simulate + and template blocks into their Cartesian product, and for each combination: + + 1. Creates a subdirectory under *out_dir*. + 2. Merges the simulate-block overrides into the base config and writes + tfs_sim_config.yaml inside that subdirectory. + 3. Renders the Jinja2 template (output_file) with the template + variables and writes the result to the subdirectory. + 4. Writes combo.json recording the variable assignments for that run. + + A grid_summary.json is written to *out_dir* listing all created runs. + Input files named by the configs or template variables are copied into + /inputs/ and referenced as ../inputs/, so the + *out_dir* directory can be moved as a unit. Every run is validated + before anything is written. + + positional arguments: + grid_yaml Path to the grid YAML file. + + options: + -h, --help show this help message and exit + --out_dir OUT_DIR Root directory for the grid. Created if it does not exist. + Default "sim_grid". + +tfs-setup-grid +-------------- + +.. code-block:: text + + usage: tfs-setup-grid [-h] [--out_dir OUT_DIR] grid_yaml + + Set up a directory grid for model runs. + + Reads a grid YAML file, expands all configure_model and template blocks + into their Cartesian product, and for each combination: + + 1. Creates a subdirectory under *out_dir*. + 2. Calls tfs-configure-model (via its Python API) to generate + tfs_configure_config.yaml, tfs_configure_priors.csv, and + tfs_configure_guesses.csv inside that subdirectory. Incompatible + combinations are skipped and logged. + 3. Renders the Jinja2 template (output_file) with the template + variables and writes the result to the subdirectory. + 4. Writes combo.json recording the variable assignments for that run. + + A grid_summary.json is written to *out_dir* listing all created + runs and any skipped combinations. Input files named by the + configure_model or template variables are copied into + /inputs/ and referenced as ../inputs/, so the + *out_dir* directory can be moved as a unit. Every combination's inputs + and template are validated before anything is written. + + positional arguments: + grid_yaml Path to the grid YAML file. + + options: + -h, --help show this help message and exit + --out_dir OUT_DIR Root directory for the grid. Created if it does not exist. + Default "grid". + +tfs-summarize-grid +------------------ + +.. code-block:: text + + usage: tfs-summarize-grid [-h] [--out_prefix OUT_PREFIX] grid_dir + + Summarize a model grid created by tfs-setup-grid. + + Scans every immediate subdirectory of *grid_dir* for combo.json and + collects the per-run variable assignments (configure_model, simulate and + template variables) into a summary CSV, with the statistics of the run's + *_fit_summary.json (in the run directory or its summary/, where + tfs-summarize-fit writes by default) when there is one. + + positional arguments: + grid_dir Path to the grid root directory created by tfs-setup-grid. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for the output CSV; {out_prefix}.csv is written. + Defaults to {grid_dir}/grid_summary. + +tfs-summarize-calibration +------------------------- + +.. code-block:: text + + usage: tfs-summarize-calibration [-h] [--out_prefix OUT_PREFIX] + [--summary_subdir SUMMARY_SUBDIR] + [--replicate_keys REPLICATE_KEYS [REPLICATE_KEYS ...]] + [--baseline BASELINE [BASELINE ...]] + [--facet_by FACET_BY [FACET_BY ...]] + [--plot_quantity PLOT_QUANTITY] + [--regime_eps REGIME_EPS] + grid_dir + + Summarize posterior calibration across a simulation grid. + + positional arguments: + grid_dir Directory of run subdirectories written by tfs-setup-sim-grid + (each with combo.json and, when finished, summary/). + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Prefix for all outputs (default tfs_calibration). + --summary_subdir SUMMARY_SUBDIR + Subdirectory holding the tfs-summarize-fit outputs (default + summary). + --replicate_keys REPLICATE_KEYS [REPLICATE_KEYS ...] + Grid variables that index replicates rather than arms (default + seed and fit_seed). Keys absent from the grid are ignored. + --baseline BASELINE [BASELINE ...] + key=value pairs selecting the baseline arm, e.g. + guide_type=component guide_rank=None. When given, every other + run is paired with the baseline run sharing all remaining grid + variables (same simulated data, same fit seed). (default: + None) + --facet_by FACET_BY [FACET_BY ...] + Grid variables that split the calibration-curve plot into + panels. (default: None) + --plot_quantity PLOT_QUANTITY + Quantity plotted (default theta_test). + --regime_eps REGIME_EPS + θ within [regime_eps, 1 - regime_eps] is resolvable (default + 0.01). + +tfs-check-tubes +--------------- + +.. code-block:: text + + usage: tfs-check-tubes [-h] [--out_prefix OUT_PREFIX] [--min_counts MIN_COUNTS] + [--min_genotypes MIN_GENOTYPES] [--flag_z FLAG_Z] + growth_df + + Score every tube's apparent titrant from its genotype composition and + flag tubes that are out of order within their row. + + Within each library x condition_pre x condition_sel x titrant_name + panel, each tube's log-ratio composition has its row (replicate x t_pre + x t_sel) effect removed by a robust fit (medians, with a per-row scale + since the titrant's effect grows with time) and is projected on the + axis from the lowest concentration's composition (score 0) to the + highest's (score 1). Tube totals and read depth cancel, so OD600 and depth errors + do not enter. A tube is flagged when its score is out of order with the + rest of its row (leave-one-out isotonic fit) or far from its label's + expected score, by more than flag_z noise SDs; rows of fewer than + three tubes are not flagged. Rows missing some concentrations are + handled without the shift of centering a row on itself. + + Writes {out_prefix}.csv, one row per tube (score, expected, + order_z, deviation_z, flagged and more), and prints each + panel's scores by row and concentration and every tube of the rows + with a flagged tube. Once a row is down to two unflagged tubes neither + is flagged, so read the whole row: a large order_z left there is + also suspect. + + positional arguments: + growth_df Growth table from tfs-process-counts (any long-form table with + genotype, counts, replicate and titrant_conc). + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Output prefix. (default: 'tfs_tube_qc') + --min_counts MIN_COUNTS + Use a genotype only if it has at least this many reads in + every tube of its panel. (default: 1) + --min_genotypes MIN_GENOTYPES + Fewest usable genotypes a panel may have. (default: 10) + --flag_z FLAG_Z Flag a tube whose order violation or deviation from its + expected score exceeds this many noise SDs. (default: 4.0) + +tfs-correct-tubes +----------------- + +.. code-block:: text + + usage: tfs-correct-tubes [-h] [--out_prefix OUT_PREFIX] + tube_table_file corrections_file + + Apply documented corrections to a tube table, leaving the original as is. + + A correction is an entry in a YAML corrections file kept with the + experiment's data: an id, an action, the tubes it names (by + the tube table's sample), a reason, who decided it (by) and + when (date), and optional evidence such as a QC table. Actions: + swap (two tubes of one library exchange their design columns), + relabel (one tube gets the design values in set) and exclude + (tubes left out). The design columns are replicate, + condition_pre, t_pre, condition_sel, t_sel, + titrant_name and titrant_conc. Every other column (the sample + name and so its count file, OD600, tube totals, barcodes) stays with the + physical sample it was measured on, which is right whether the tubes + were swapped at the bench or the sample sheet was. library never + changes, and a swap across libraries is refused. + + Writes {out_prefix}.csv (the corrected tube table, with a + correction_id column, for tfs-process-counts), + {out_prefix}_changes.csv (one row per changed value or excluded + tube, with its original and corrected value) and + {out_prefix}_provenance.json (the corrections and the SHA-256 of + the inputs, the evidence files and the output). + + Fails on an unknown tube, a tube named in two corrections, a swap + across libraries or one that changes nothing, a relabel of a column that + is not a design column, a correction that leaves two tubes with the same + library and design, a missing or empty reason, by or date, or an output + that would overwrite an input. + + positional arguments: + tube_table_file The original tube table (CSV, TSV or Excel), in tfs-process- + counts format. + corrections_file YAML corrections file. + + options: + -h, --help show this help message and exit + --out_prefix OUT_PREFIX + Output prefix. (default: 'tfs_tubes_corrected') + +tfs-fit-two-stage +----------------- + +.. code-block:: text + + usage: tfs-fit-two-stage [-h] [--out_dir OUT_DIR] [--step STEP] + [--jackknife_task JACKKNIFE_TASK] [--no_consistency] + config_file + + Fit a growth-only relative model in two stages. + + The full-library MAP makes the growth slope m too steep (each double + mutant pulls it a little and the pulls add), and congression flattens a + marker's m in any stage 1 large enough to pin X's scale + (planning/studies/m-bias-factorial/). This command runs, in out_dir: + stage1 (wt, spikes and singles with the congression mixture, max rule, + lambda fitted with two_stage.transformation_lambda as its prior: every + condition's k and m); stage2 (the full library with every condition's + k and m pinned at stage 1; MAP, Laplace, X); predict (stage 2's X, and + with two_stage.predict_epistasis its epistasis, predict/); consistency + (stage 2 on one library's tubes alone against the full stage 2, per + genotype z-scores, consistency/consistency.csv); summary + (out_dir/tfs_x.csv: per genotype and concentration, X's quantiles, the + genotype's reads in total and per library, its number of mutations, + in_training_data and the consistency z and flag; the table to read + results from). + + X's intervals leave out k's uncertainty, shared by every genotype; on + simulations the deep genotypes' 95% intervals covered 0.88-0.90 up to + 1e5 reads and 0.76 above (planning/studies/two-stage-validation/). + two_stage.kshift (experimental, off by default) adds it: jackknife + (two_stage.jackknife_subsamples refits of stage 1, each dropping + two_stage.jackknife_drop_frac of the tubes, stratified by condition and + IPTG: k's SE and main direction), then kshift (two stage 2 refits at k + plus and minus one jackknife SD, from stage 2's MAP at + two_stage.kshift_step_size with the dk_geno population held there: + kshift/shift.csv, added to predict's draws as one shared multiple; with + two_stage.kshift_control a k-unchanged control refit and + kshift/drift.csv). Validation run 2 found that shift a third or less of + the true one on simulations. + + Every step but summary skips work whose outputs exist, so a run resumes + and a step reruns by deleting its directory; summary is rewritten each + time. --step runs one step, the k-shift steps included whatever + two_stage.kshift says. + + positional arguments: + config_file YAML whose top-level keys are tfs-configure-model's arguments + (growth_df and library_config required; theta_model + hill_relative, growth_shares_replicates true; file paths + relative to the YAML), plus a two_stage block: + transformation_lambda (mean, std; required), seed (1), + num_posterior_samples (500), consistency (true), + consistency_library (default the first library + alphabetically), predict_epistasis (false), kshift (false; + experimental, see above), jackknife_subsamples (30), + jackknife_drop_frac (0.2), kshift_step_size (1e-5, the k-shift + refits' starting step size), kshift_control (true, the + k-unchanged control refit), fit (extra tfs-fit-model + arguments) and posterior (extra tfs-sample-posterior + arguments). + + options: + -h, --help show this help message and exit + --out_dir OUT_DIR Directory for every step's outputs. Default 'two_stage'. + --step STEP 'all' (default) or one of stage1, jackknife, stage2, kshift, + predict, consistency, summary. + --jackknife_task JACKKNIFE_TASK + Run only this jackknife subsample (1 to jackknife_subsamples), + for a cluster array; needs stage 1 done. The kshift step + collects them. (default: None) + --no_consistency Turn off (on by default): Run the consistency step with 'all' + (default True; --no_consistency skips it, as does consistency: + false in the YAML). + +tfs-process-binding +------------------- + +.. code-block:: text + + usage: tfs-process-binding [-h] [--protein_conc PROTEIN_CONC] + [--protein_conc_units PROTEIN_CONC_UNITS] + [--genotype_col GENOTYPE_COL] [--protein_col PROTEIN_COL] + [--titrant_col TITRANT_COL] [--theta_col THETA_COL] + [--conc_units CONC_UNITS] [--replicate_cols REPLICATE_COLS] + [--titrant_name TITRANT_NAME] [--theta_std THETA_STD] + [--sig_digits SIG_DIGITS] [--out_prefix OUT_PREFIX] + binding_file + + Average a lab binding table into a binding_df at one protein + concentration. + + The table has one row per measurement (genotype, protein and titrant + concentrations, theta, and the columns naming its replicate). Rows + without theta are dropped; concentrations are rounded to sig_digits + significant digits so replicates from different days line up; + measurements within a replicate are averaged, then replicates per + genotype and titrant concentration (theta_obs). theta_std is the + standard error of that mean from one replicate SD pooled over every + replicated point at this protein concentration, or from --theta_std. + The titrant is written in mM. + + Writes {out_prefix}.csv (genotype, titrant_name, titrant_conc, + theta_obs, theta_std; for tfs-configure-model --binding_df) and + {out_prefix}_summary.csv (replicate and measurement counts and each + point's own replicate SD). + + positional arguments: + binding_file The lab's binding table (CSV, TSV or Excel). + + options: + -h, --help show this help message and exit + --protein_conc PROTEIN_CONC + Protein concentration to keep, in protein_conc_units. + (default: 2.0) + --protein_conc_units PROTEIN_CONC_UNITS + Unit of protein_conc: M, mM, uM or nM. (default: 'uM') + --genotype_col GENOTYPE_COL + Genotype column. (default: 'genotype') + --protein_col PROTEIN_COL + Protein concentration column. (default: + 'protein_dimer_conc_molar') + --titrant_col TITRANT_COL + Titrant concentration column. (default: 'iptg_conc_molar') + --theta_col THETA_COL + Measured theta column. (default: 'theta') + --conc_units CONC_UNITS + Unit of the protein and titrant columns: M, mM, uM or nM. + (default: 'M') + --replicate_cols REPLICATE_COLS + Comma-separated columns that identify one replicate; empty for + one replicate per row. (default: 'biorep,date') + --titrant_name TITRANT_NAME + titrant_name for every row; must match the growth table's. + (default: 'iptg') + --theta_std THETA_STD + Fixed per-replicate SD instead of the pooled one. (default: + None) + --sig_digits SIG_DIGITS + Significant digits the concentrations are rounded to. + (default: 6) + --out_prefix OUT_PREFIX + Output prefix. (default: 'tfs_binding') diff --git a/docs/source/downstream.rst b/docs/source/downstream.rst new file mode 100644 index 00000000..cceb222d --- /dev/null +++ b/docs/source/downstream.rst @@ -0,0 +1,413 @@ +=================== +Downstream analysis +=================== + +This page covers everything after a posterior exists: parameter tables, +predicted occupancy and growth, epistasis, response-shape classification and +cross-run comparison. Every flag and default is listed in :doc:`cli`. The +steps that produce the posterior are in :doc:`fitting`, and the diagnostic +summary of a finished run is in :doc:`summarize-fit`. + +The examples assume the default file names from the earlier steps: +``tfs_configure_config.yaml`` from ``tfs-configure-model``, +``tfs_fit_model_checkpoint.pkl`` from ``tfs-fit-model`` and +``tfs_posterior.h5`` from ``tfs-sample-posterior``. + +Inputs and output columns +------------------------- + +The model-based commands (``tfs-extract-params``, ``tfs-predict-theta``, +``tfs-predict-growth`` and ``tfs-predict-epistasis``) take two positional +arguments: the config YAML used for the fit and a parameter file. The +parameter file is normally the ``.h5`` written by ``tfs-sample-posterior``. + +A MAP checkpoint (``.pkl``) is also accepted. It holds a single point, so the +output has one ``q0.5`` column and no uncertainty. The predict commands +convert the checkpoint to a one-draw posterior on the fly and leave it behind +as ``{out_prefix}_map_posterior.h5``. SVI and NUTS checkpoints are refused; +run ``tfs-sample-posterior`` on them first. For a MAP fit, run +``tfs-sample-posterior`` too if you want intervals: it builds the Laplace +approximation and writes an ``.h5`` like any other. + +Every posterior summary is written as bare ``q`` columns with no +feature-name prefix. A posterior file gives 17 levels: + +.. code-block:: text + + q0.001 q0.005 q0.01 q0.025 q0.05 q0.1 q0.159 q0.25 q0.5 + q0.75 q0.841 q0.9 q0.95 q0.975 q0.99 q0.995 q0.999 + +``q0.5`` is the posterior median and the recommended point estimate. +``q0.025`` and ``q0.975`` bound the 95% credible interval. ``q0.159`` and +``q0.841`` bound the central 68% interval, so ``(q0.841 - q0.159) / 2`` is a +robust one-sigma width. The table-based tools below (``tfs-extract-epistasis``, +``tfs-cat-response``, ``tfs-compare-runs``) use ``q0.5`` and that half-width +as their default value and uncertainty columns, so the outputs of one command +feed the next without renaming. + +Parameter tables +---------------- + +``tfs-extract-params`` writes one CSV per parameter group. + +.. code-block:: bash + + tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 + +The files are named ``{out_prefix}_{parameter}.csv`` with the default prefix +``tfs_params``. Which files appear depends on the configured components. A +``hill_geno`` fit gives ``tfs_params_log_hill_K.csv``, +``tfs_params_hill_n.csv``, ``tfs_params_theta_low.csv`` and +``tfs_params_theta_high.csv``, keyed by ``genotype`` and ``titrant_name``. +``hill_mut`` adds per-mutation effects such as +``tfs_params_d_log_hill_K.csv`` (keyed by ``mutation``) and, with pairs, +epistasis terms such as ``tfs_params_epi_log_hill_K.csv`` (keyed by +``pair``). ``hill_relative`` writes ``X_low`` and ``X_high`` in place of the +theta baselines. Other components add their own files, for example +``tfs_params_dk_geno.csv``, ``tfs_params_growth_k.csv`` and +``tfs_params_growth_m.csv`` (keyed by ``condition_rep``, plus +``replicate`` when replicates have their own growth parameters), +``tfs_params_lam.csv`` for the ``mixture`` transformation, +``tfs_params_k_ref.csv`` when base-growth data were supplied, and, with +tube offsets, ``tfs_params_sample_offset_offset.csv`` (``level``) or +``tfs_params_sample_offset_delta_k.csv`` (``normal``), one row per tube +keyed by its design (``replicate``, ``library``, ``condition_pre``, +``condition_sel``, ``titrant_name``, ``titrant_conc``, ``t_pre``, +``t_sel``), with the offsets' SD in ``tfs_params_sample_offset_sigma.csv``. +A held SD (``sigma_fixed``) is a deterministic site, so a checkpoint has no +SD file. + +The thermodynamic PddG and PnnC models also write one table per structure +of the per-mutation ΔΔG, ``tfs_params_ddG_{structure}.csv`` (for example +``tfs_params_ddG_H.csv``). The models fit the constants, and the ΔΔG is +computed from them: exactly in the models with an unfolded state, and as +the most probable per-structure values given the prior in those without +one, where the shift shared by all structures comes from the structural +prior. They also write a table of its structural prior mean, +``tfs_params_ddG_prior_mean_{structure}.csv`` (the ΔΔG CSV values for PddG, +the network's prediction for PnnC). Both are keyed by ``mutation`` and named +by the model's structures, whatever order the input file lists them in. The +ΔΔG and the prior mean are deterministic sites, so they need a posterior +file (a checkpoint has neither). The models +also write ``tfs_params_ddG_sigma.csv``, one row per ``structure``: the SD +σ_s of the ΔΔG about the structural prior mean, which the fit estimates per +structure. + +``log_hill_K`` is the natural log of the Hill constant in the units of +``titrant_conc``. It is not a base-10 log. + +Given a ``.pkl`` MAP checkpoint, ``tfs-extract-params`` writes a single +``q0.5`` column. Deterministic sites such as ``ln_cfu0`` are absent from a +checkpoint, so their files are skipped with a warning. Run +``tfs-sample-posterior`` first to get them. + +Predicted occupancy +------------------- + +``tfs-predict-theta`` predicts operator occupancy as a function of titrant +concentration. + +.. code-block:: bash + + tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + +The output is ``tfs_pred_theta.csv``, one row per ``genotype``, +``titrant_name`` and ``titrant_conc``, with the ``q`` columns and an +``in_training_data`` column. ``in_training_data`` is 1 when that genotype, +titrant name and concentration triple was in the training data and 0 +otherwise. + +By default the command predicts every training genotype at every training +titrant point. Three plain-text files add to that grid, one value per line +with ``#`` comments allowed. ``--genotypes_file`` adds genotypes, written as +slash-separated mutations such as ``M42I/K84L`` or as ``wt``. +``--titrant_names_file`` and ``--titrant_concs_file`` add titrant points and +must be given together. They pair line by line, and a single name is +broadcast across every concentration. The added genotypes and points are +unioned with the training set. Pass ``--only_files`` to predict only at the +file inputs. + +A genotype that was not in the training data needs a theta component that can +predict it. ``hill_mut`` assembles the genotype from its per-mutation effects. +``hill_geno`` and ``hill_relative`` have no per-mutation structure and give +every unseen genotype the population average for its titrant. +``--genotype_batch_size`` (default 2000) caps the memory used for unseen +genotypes. ``--num_samples N`` adds ``sample_0`` to ``sample_{N-1}`` columns +holding joint posterior draws. + +A ``hill_relative`` fit predicts the wt-relative growth variable X rather +than an occupancy. Its output carries an extra ``theta_scale`` column with the +value ``X`` so that nothing downstream reads it as theta. X is 1 for wt at the +low gauge concentration and 0 for wt at the high one. See :doc:`model` for the +gauge. + +Predicted growth +---------------- + +``tfs-predict-growth`` predicts ln(CFU) from the fitted model. + +.. code-block:: bash + + tfs-predict-growth tfs_configure_config.yaml tfs_posterior.h5 + +The output is ``tfs_pred_growth.csv``, one row per genotype, replicate, +condition, titrant and time point, with the ``q`` columns, the observed +``ln_cfu`` and ``ln_cfu_std`` where an observation exists, and +``in_training_data``. + +``--genotypes_file`` and ``--titrant_concs_file`` extend the prediction grid +as in ``tfs-predict-theta``, and ``--only_files`` restricts it to the file +inputs. ``--titrant_names_file`` behaves differently here. It is a +restrict-only filter applied after prediction: it narrows which titrant names +appear in the output and never adds points. + +Growth prediction over a full library is the most memory-hungry step in this +page. ``--genotype_batch_size`` sets how many genotypes go through the model +at once. Left unset, it is estimated from the available device memory. Each +batch costs one JAX recompilation. ``--num_marginal_samples`` limits how many +posterior draws are run through the model when computing quantiles; by +default all are used. ``--num_samples N`` adds ``sample_*`` columns as in +``tfs-predict-theta``. + +For a quick check of the fit without a full sweep, ``--subset_genotypes`` +predicts a single memory-sized block. The block always holds the binding +genotypes, the spiked genotypes and any genotypes from ``--genotypes_file``, +and the rest is a random draw from the other genotypes. ``--seed`` makes the +draw reproducible. + +.. code-block:: bash + + tfs-predict-growth tfs_configure_config.yaml tfs_posterior.h5 \ + --subset_genotypes --seed 1 --out_prefix tfs_pred_growth_subset + +Epistasis +--------- + +Two commands compute second-order epistasis on mutant cycles. Each cycle is a +double mutant, its two single-mutant parents and wt. They differ in how they +treat the posterior. + +``tfs-predict-epistasis`` works from the joint posterior. It draws theta for +all four corners of each cycle from the same posterior sample, computes +epistasis within that draw and then reports quantiles across draws. The +interval therefore carries the posterior covariance between the corners. + +.. code-block:: bash + + tfs-predict-epistasis tfs_configure_config.yaml tfs_posterior.h5 + +The output is ``tfs_pred_epistasis.csv`` with ``genotype`` (the double +mutant), ``titrant_name``, ``titrant_conc``, the ``q`` columns and a +trailing ``in_regime`` column. The default ``--scale logit`` takes additive +epistasis of logit(theta). ``add`` takes ``(Y11 - Y10) - (Y01 - Y00)`` and +``mult`` takes ``(Y11 / Y10) / (Y01 / Y00)``. ``--scale_constant`` multiplies +the transformed value before the difference. Since logit(theta) equals +-dG/RT, passing ``--scale_constant -0.6159`` reports logit epistasis as an +interaction free energy in kcal/mol at 310.15 K. Only genotypes seen in +training are supported, and an unseen genotype in ``--genotypes_file`` is an +error. + +``in_regime`` is 1 only when the theta posterior of all four corners lies +inside the resolvable band ``[regime_eps, 1 - regime_eps]``. The posterior is +judged by its central ``--regime_ci`` interval, 95% by default, and +``--regime_eps`` defaults to 0.01. Near 0 or 1, logit(theta) saturates and +growth constrains it weakly, so epistasis there leans on the theta model's +functional form and on cross-genotype covariance. Treat rows with +``in_regime`` 0 as model-conditional, even when their interval is narrow. The +flag checks posterior mass only. It does not test whether the growth signal +exceeds the growth noise. With a MAP checkpoint there is one draw, so the +check reduces to the point value. + +A ``hill_relative`` fit predicts X rather than theta. Pass ``--scale add`` +for it. The output then has no ``in_regime`` column. + +``tfs-extract-epistasis`` works from any long-form table with one row per +genotype per condition. It computes epistasis from each corner's marginal +estimate and propagates the errors as if the corners were independent. Use +it for tables that do not come from the model, or for a quick look at a +``tfs-predict-theta`` table. + +.. code-block:: bash + + tfs-extract-epistasis tfs_pred_theta.csv \ + --group_by titrant_name titrant_conc --scale logit + +``--y_obs`` defaults to ``q0.5`` when that column exists, and ``--y_std`` to +``(q0.841 - q0.159) / 2``. ``--group_by`` names the columns that define a +condition. Epistasis is computed within each condition, so for a predict-theta +table pass both ``titrant_name`` and ``titrant_conc``. The default scale here +is ``add``, unlike ``tfs-predict-epistasis``. ``logit`` requires values in +(0, 1) and clamps them to ``[logit_eps, 1 - logit_eps]``. ``--scale_constant`` +works as above. On a ``hill_relative`` table only ``add`` is allowed. The +output is ``tfs_epistasis.csv`` with one row per double mutant and +condition, holding ``ep_obs`` and, when an uncertainty column is available, +``ep_std``. ``--keep_extra`` keeps every input column. + +Prefer ``tfs-predict-epistasis`` for model output. Corners of a cycle share +the fitted shared parameters, so their errors are correlated, and the +marginal calculation can be off in either direction. + +Response shapes +--------------- + +``tfs-cat-response`` fits a family of empirical curve models to each group's +``y_obs`` versus ``x_obs`` curve. It answers two separate questions: what +shape the curve has, and whether the curve can be told apart from zero. + +.. code-block:: bash + + tfs-cat-response tfs_pred_theta.csv titrant_conc --group_by titrant_name + +The positional arguments are the data file and ``x_obs``, the name of the +independent-variable column. ``--y_obs`` is a flag. It defaults to ``q0.5`` +when that column exists. ``--y_std`` defaults to ``(q0.841 - q0.159) / 2`` +when both quantiles exist; otherwise the fit is unweighted. Groups are the +``genotype`` column plus any ``--group_by`` columns. The same command works on +epistasis tables, for example ``tfs-cat-response tfs_pred_epistasis.csv +titrant_conc --group_by titrant_name``. The per-group fits run in parallel; +``--num_workers`` defaults to -1, which uses all but one CPU. + +**Model x-scale.** The models fall into two camps, and they must be handed +raw concentration in both cases. The Hill family (``repressor``, +``inducer``, ``hill_repressor``, ``hill_inducer``) and the ``biphasic_*`` +models are written in raw concentration and take the log internally, so they +are already sigmoids or peaks in log concentration. ``bell_peak_log``, +``bell_dip_log`` and ``linear_log`` are shapes in log10 concentration and +also apply the transform themselves. Concentrations of 0 are placed two +decades below the lowest nonzero concentration before the log. The plain +``bell_peak``, ``bell_dip`` and ``linear`` models are shapes in raw x. +``flat`` does not depend on x. Never log-transform the x column yourself. + +**Shape.** ``--select_by`` picks the model. The default, ``shape``, is a +liberal classifier meant for exploration. It first decides flat versus curvy +from the residual autocorrelation of the flat fit: the curve is curvy when the +autocorrelation p-value is below ``--curvy_cutoff`` (default 0.1). It then +names the curvy shape by best weighted R², preferring the simpler model when +two are close. In this mode the default model set is ``flat``, ``inducer``, +``repressor``, ``bell_peak_log``, ``bell_dip_log``, ``biphasic_peak`` and +``biphasic_dip``. ``curvy_cutoff`` is the knob to sweep; run a few values and +inspect the curves. ``aicc`` picks the lowest small-sample AIC on weighted +residuals. It is conservative and often calls a well-fit curve flat at small +n. ``adequacy`` keeps the AICc pick unless a runs test finds clustered +residuals, then moves to the best adequate model that is no simpler. It never +demotes a curve to a simpler model. Outside ``shape`` mode the default model +set is ``flat``, ``linear_log``, ``repressor``, ``inducer``, +``bell_peak_log`` and ``bell_dip_log``. ``--models`` reaches any model in the +library. + +The selected model is reported as ``best_model``, with ``shape`` giving its +qualitative form: ``flat``, ``linear``, ``step``, ``peak``, ``dip`` or +``biphasic``. ``aicc_best_model`` records the AICc pick for comparison. +``shape_status`` reports the runs test on the selected model. + +**Magnitude.** Whether a curve is distinguishable from zero is judged on the +observed data, not on the fitted curve. The test statistic is +``sum((y_obs / y_std)**2)`` against a chi-squared distribution with n degrees +of freedom. Its p-value, ``nonzero_p``, is corrected across curves by +Benjamini-Hochberg to give ``nonzero_q``. ``fittable`` is True when +``nonzero_q`` is below ``--alpha`` (default 0.05). ``all_equiv_zero`` is True +when every point's interval lies inside the region of practical equivalence +``[-rope_cutoff, rope_cutoff]``. By default ``rope_cutoff`` is +``--rope_multiplier`` (default 2) times the median observed ``y_std``. That +value scales with the noise, so whole intervals rarely fit inside it. Pass an +explicit ``--rope_cutoff`` with biological meaning to make the call useful. +The two flags together give three outcomes: ``fittable`` True is a real +response; ``fittable`` False with ``all_equiv_zero`` True is confidently flat +at zero; ``fittable`` False with ``all_equiv_zero`` False cannot be told. + +Shape and magnitude are independent. The intended read is to filter on +``fittable`` and then look at ``shape``. + +The outputs, with the default prefix ``tfs_cat_response``: + +``tfs_cat_response.csv`` + One row per group: ``best_model``, ``aicc_best_model``, ``shape``, + ``shape_status``, per-model fit statistics and parameters, and the + magnitude rollups (``nonzero_p``, ``nonzero_q``, ``n_nonzero``, + ``all_equiv_zero``, ``fittable``). ``omnibus_p`` and ``omnibus_q`` are a + model-based test reported for reference only. They gate nothing. + +``tfs_cat_response_assessment.csv`` + One row per group and x for the best model: ``model``, ``fittable``, + ``x``, the observed ``y_obs`` and ``y_std``, the model's ``y_model`` and + its propagated error ``y_model_std``, then ``z`` (``y_obs / y_std``) and + the per-point ``sig_nonzero``. + +``tfs_cat_response_predictions.csv`` + The best model's curve for each group. ``--write_all_predictions`` writes + every model's curve. + +``tfs_cat_response_{model}.csv`` + One file per fitted model with its parameter table and per-group fit + statistics. + +Comparing runs +-------------- + +``tfs-compare-runs`` measures how much N independent estimates of the same +quantity disagree, and whether each run's reported uncertainty explains the +disagreement. The runs might differ by seed or by which data were held out. +It accepts any long-form table with a point estimate: predict-theta, -growth +and -epistasis output, and ``tfs-extract-params`` files alike. + +.. code-block:: bash + + tfs-compare-runs seed1/tfs_pred_theta.csv seed2/tfs_pred_theta.csv \ + seed3/tfs_pred_theta.csv + +Two or more arguments are read as CSV paths. A single argument is read as a +manifest file listing one CSV path per line, unless it ends in ``.csv``. +``--reference full/tfs_pred_theta.csv`` switches to reference mode, where each +run is scored by its deviation from that run, as for k-fold dropouts against a +full-data fit. Without it, runs are compared to their cross-run mean. + +Three sets of columns control the comparison, and all three are printed and +recorded in the metadata. The **match key** is what makes a row the same row +across runs. By default it is every column shared by all runs except the value +columns (``q``, ``y_obs``, ``y_std``) and bookkeeping columns such as +``in_training_data`` and ``in_regime``. ``--match_by`` overrides it. The key +must be unique within each run. **index_by** is the entity being scored. It is +``genotype`` if present, else ``parameter``, else the only match-key column. +A growth-parameter file keyed by both ``replicate`` and ``condition_rep`` +has no single entity column, so it needs ``--index_by condition_rep``. ``--group_by`` breaks each entity out +further. The report key is ``index_by`` plus ``group_by``, with one output row +per value. The match-key columns outside the report key are pooled over. + +``--group_by`` is a statistical zoom. At the finest grouping only N - 1 +degrees of freedom remain per row. Check ``n_rows`` and ``n_eff`` for the +pooling depth. + +.. code-block:: bash + + tfs-compare-runs seed1/tfs_params_growth_k.csv seed2/tfs_params_growth_k.csv \ + --index_by condition_rep + +The command applies no thresholds and assigns no grades. Every quantity is a +number, and any cutline belongs in the downstream analysis that uses it, such +as ``df["overdispersion"] > 2``. The main output, ``tfs_compare_runs.csv``, +has one row per report key with these columns: ``n_runs``, ``n_present``, +``n_rows``, ``n_eff``, ``mode``, ``spread_estimator``, ``rms_sd``, +``max_sd``, ``mean_value``, ``dynamic_range``, ``mean_reported_sigma``, +``chi2``, ``dof``, ``overdispersion``, ``overdispersion_p`` and +``overdispersion_q``. + +``rms_sd`` and ``max_sd`` measure run-to-run spread in the estimate's own +units, so they cannot be compared across parameters. ``overdispersion`` is +chi-squared over its degrees of freedom. It is unit-free and asks whether the +spread exceeds what each run's sigma predicts; ``overdispersion_q`` is its +Benjamini-Hochberg q-value. The default sigma is ``(q0.841 - q0.159) / 2``. +That stays a good scale for spread but biases the overdispersion test for +skewed posteriors, such as ``theta_low`` or ``theta_high`` near 0 or 1 or +``hill_n`` near its bound. ``dynamic_range`` is the range of the target over +the pooled axes. It is NaN when nothing is pooled. + +``tfs_compare_runs_aggregate.csv`` mixes the per-run posteriors as an +equal-weight mixture, row by row on the match key. Its quantiles fold in both +each run's width and the spread between runs, and they do not shrink with N. +It needs at least two shared ``q`` columns and never includes a +reference run. It is the slow step on a large library, and ``--no_aggregate`` +skips it. ``tfs_compare_runs_metadata.json`` records every resolved setting. + +When the tables have a ``genotype`` column, rows come back in canonical +genotype order: wt, then singles, then doubles, by site. Otherwise the main +table is sorted by ``rms_sd``. diff --git a/docs/source/fitting.rst b/docs/source/fitting.rst new file mode 100644 index 00000000..84b3dfa8 --- /dev/null +++ b/docs/source/fitting.rst @@ -0,0 +1,593 @@ +================= +Fitting the model +================= + +A fit takes a processed growth table, binding data or both, and ends with +posterior draws of every model parameter. It runs in up to four commands: + +1. ``tfs-configure-model`` checks the data and writes the model + configuration. +2. ``tfs-prefit-calibration`` pins each condition's growth baseline and slope. + It runs only for models with binding data. +3. ``tfs-fit-model`` finds the MAP point or fits a variational posterior. +4. ``tfs-sample-posterior`` turns the fit into posterior draws. + +The model itself is described in :doc:`model` and its inputs in +:doc:`model-inputs`. :doc:`cli` lists every flag. What to do with the +posterior file is in :doc:`downstream`. + +Recommended routes +================== + +**For a screen, use the two-stage fit,** ``tfs-fit-two-stage``, described +in :doc:`screen-analysis`. It chains the commands on this page in the +order and with the settings that gave calibrated intervals on simulations +of the 2026 screen. The single fits below are its building blocks, and the +place to look for what each setting does. + +Joint growth and binding model +------------------------------ + +*Experimental.* Binding and growth have not been reconciled on real data: +no single protein concentration or monotone map carries the binding +curves onto growth for every genotype, so binding cannot yet set the +absolute scale of occupancy. Use this model for simulations and for +method development, not for a screen's reported curves. + +With binding data for some genotypes, *θ* is an absolute occupancy. The +configuration defaults are the current recommendations: the count likelihood, +one level offset per tube and one starting abundance per replicate, +pre-growth condition and genotype. + +.. code-block:: bash + + tfs-configure-model --growth_df growth.csv --binding_df binding.csv \ + --library_config library_config.yaml + tfs-prefit-calibration tfs_configure_config.yaml --seed 1 + tfs-fit-model tfs_configure_config.yaml --seed 1 --analysis_method map + tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --skip_growth_observations + +Growth-only relative-X model +---------------------------- + +This is the model the two-stage fit uses. Fitted once on a full library, +as below, its MAP makes the growth slope too steep and compresses *X* +about 1.45-fold; the two-stage fit avoids that by taking the slope from +wt, the spikes and the singles. + +Without binding data, growth fixes occupancy only up to an affine map. The +``hill_relative`` occupancy component fits a wt-relative variable *X* +instead (see :doc:`model`). The pre-fit refuses a growth-only model, so the +per-condition growth priors come from ``--growth_priors`` or +``--growth_priors_wt_rates`` at configuration, and the population SDs that +ran away on real data are held with ``--set_priors``. + +On a full library the route is a MAP fit followed by the arrowhead Laplace. +A MAP with level tube offsets runs in stages by default (see `Staged MAP`_ +below), which keeps the offsets from settling in a mode where they carry the +population's growth. + +.. code-block:: bash + + tfs-configure-model --growth_df growth.csv \ + --library_config library_config.yaml \ + --theta_model hill_relative \ + --growth_shares_replicates \ + --growth_priors growth_priors.csv \ + --set_priors theta_log_hill_n_hyper_scale_fixed=0.5 sigma_fixed=0.17 + tfs-fit-model tfs_configure_config.yaml --seed 1 --analysis_method map + tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --laplace arrowhead --skip_growth_observations + +On a small library the low-rank guide also +recovers *X*: + +.. code-block:: bash + + tfs-fit-model tfs_configure_config.yaml --seed 1 --analysis_method svi \ + --guide_type auto_low_rank_multivariate_normal --guide_rank 20 + +Configure +========= + +``tfs-configure-model`` reads the data, checks it against the library, +resolves the model components and writes four files: + +``{out_prefix}_config.yaml`` + The configuration every later step reads: data paths, the component + choices and settings, the names of the priors and guesses files, the + library provenance and a ``provenance:`` block recording the tfscreen + version and command line. +``{out_prefix}_priors.csv`` + Every prior, one row per value. +``{out_prefix}_guesses.csv`` + Starting values for the fit. +``{out_prefix}_library.csv`` + The per-genotype library composition, for growth models. + +The default prefix is ``tfs_configure``. Like every command that takes +``--out_prefix``, it also writes ``{out_prefix}_provenance.json``. A census of parameters and +observations is printed and written to ``{out_prefix}_model_stats.csv`` and +``{out_prefix}_model_stats.json`` unless ``--skip_model_stats`` is given. Do +not edit these files by hand. The flags below set everything that needs +setting. + +Data +---- + +``--growth_df`` and ``--binding_df`` are both flags, and at least one is +required. With both, the joint model is configured. With only +``--binding_df``, a binding-only model infers *θ* from the binding +measurements alone. With only ``--growth_df``, a growth-only model infers +*θ*, or *X* for ``hill_relative``, from growth alone. ``--presplit_df`` and +``--base_growth_df`` add optional observations. :doc:`model-inputs` +describes each table. + +``--library_config`` is required whenever ``--growth_df`` is given. It is the +library YAML handed to ``tfs-process-fastq``. ``tfs-configure-model`` fails +if any genotype in the data is missing from the library, which catches a +residue-numbering or wt-sequence mismatch at once. It also fails if a spiked +genotype named in the library has no growth data, since that usually means +the counts were called against a different library file. +``--allow_missing_spikes`` turns that error into a report, for an experiment +in which a spike truly dropped out. + +Components +---------- + +Each ``--_model`` flag picks a component. :doc:`model` describes every +option. The defaults are ``linear`` condition growth, ``instant`` +transition, ``hierarchical`` starting abundance, ``hierarchical_geno`` +pleiotropy, ``fixed`` activity, ``hill_geno`` occupancy, ``single`` +transformation, ``level`` tube offsets and the ``counts`` likelihood. +``--batch_size`` sets the mini-batch of genotypes used by the optimizer, +default 1024. + +Priors +------ + +``--set_priors name=value ...`` sets any scalar prior in the priors file. A +name is a full row name, such as ``growth.sample_offset.sigma_fixed``, or any +unique dotted suffix of one, such as ``sigma_fixed``. An unknown or +ambiguous name is an error. The common uses hold a population SD that would +otherwise run away: + +.. code-block:: bash + + --set_priors sigma_fixed=0.17 theta_log_hill_n_hyper_scale_fixed=0.5 + +``--growth_priors table.csv`` sets per-condition priors for ``linear`` +condition growth. It has a ``condition_rep`` column, plus ``replicate`` when +replicates do not share conditions, and any of ``k_loc``, ``k_scale``, +``m_loc`` and ``m_scale``. Conditions and values it leaves out keep their +defaults. + +.. code-block:: text + + condition_rep,k_loc,k_scale,m_loc,m_scale + kanR+kan,0.015,0.01,0.0,0.01 + pheS+4CP,0.015,0.01,0.0,0.01 + +``--growth_priors_wt_rates rates.csv`` derives the priors from wt +monoculture growth rates, for ``hill_relative`` only. Its columns are +``condition_sel``, ``titrant_conc``, ``rate_mean``, ``rate_sd`` and +``num_replicates``. For each listed condition, *k* is wt's rate at the high +gauge concentration and *m* is its rate at the low one minus *k*. Each SD is +the standard error of the mean, floored at ``--growth_priors_sd_floor``, +default 0.002 per minute. List only the conditions where the monoculture +stands for the library's wt. These rows override ``--growth_priors`` for +the conditions they list. + +Pre-fit calibration +=================== + +``tfs-prefit-calibration`` pins the per-condition growth parameters of a +model with binding data. Condition baselines *k* and genotype effects +*dk_g* trade off against each other, and without a pin the whole system can +slide by a constant (see :doc:`model`). The pre-fit breaks the tie. + +It restricts the growth and binding data to the genotype and concentration +cells present in both and builds a reduced model. Occupancy comes straight +from the binding data, *dk_g* is 0, or pinned from ``--base_growth_df`` when +given, and the other hierarchies are held at their prior locations. Only +the ``condition_growth`` and ``growth_transition`` components are fit. After +a MAP fit it computes Hessian SDs at the MAP point and rewrites the +production priors and guesses files in place, after copying each to +``.bak``. Each condition's *k* and *m* location becomes an indexed row in the +priors file, and the scales are set from the Hessian between floors and +ceilings: + +.. list-table:: + :header-rows: 1 + :widths: 30 15 55 + + * - Flag + - Default + - Meaning + * - ``--k_scale_floor`` + - 0.002 + - Smallest prior SD on *k*, the day-to-day variation the Hessian + cannot see. + * - ``--m_scale_floor`` + - 0.001 + - Smallest prior SD on *m*. + * - ``--k_scale_ceiling`` + - 0.1 + - Largest prior SD on *k*, so a flat Hessian direction never loosens + the prior. + * - ``--m_scale_ceiling`` + - 0.01 + - Largest prior SD on *m*. + * - ``--pin_m`` + - off + - Clamp *m* to the calibrated value instead of a soft prior. + * - ``--hessian_chunk_size`` + - 64 + - Hessian rows per device batch. Lower it if the device runs out of + memory. + +``--pin_m`` exists because the growth likelihood can pull *m* far from even +a very tight soft prior. *k* always keeps a soft prior, since it carries real +tube-to-tube variation. + +``--seed`` is required unless resuming with ``--checkpoint_file``. The +optimizer and convergence flags are those of ``tfs-fit-model``. Its own +outputs, with the default prefix ``tfs_prefit``, are diagnostics: +``tfs_prefit_params.npz``, ``tfs_prefit_checkpoint.pkl``, +``tfs_prefit_losses.txt`` and ``tfs_prefit_convergence.csv``. + +The pre-fit refuses a growth-only configuration, because it has no binding +data to calibrate against. Set those priors at configuration instead. + +Fit +=== + +``tfs-fit-model config_file`` fits the model. ``--seed`` is required for a +new fit. ``--analysis_method`` picks the method: + +``map`` + Maximum a posteriori optimization. The result is a point. + ``tfs-sample-posterior`` builds a Laplace approximation around it. +``svi`` (default) + Stochastic variational inference. A MAP warm-up runs first and the + variational guide starts at its point. ``tfs-sample-posterior`` samples + the guide. +``nuts`` + The No-U-Turn sampler, started at the MAP warm-up's point. It is exact + but only practical for small models. ``--nuts_num_warmup``, + ``--nuts_num_samples``, ``--nuts_num_chains``, + ``--nuts_target_accept_prob`` and ``--nuts_dense_mass`` control it. + +Choosing a method +----------------- + +``svi`` with the component guide is the default, but the evidence so far +favors the other options. On simulations of the growth-only relative model +(``planning/studies/relative-fit/``, 30 runs), the component guide gave +biased *X*, with 95% coverage of 0.06 under Poisson count noise and 0.38 +under realistic noise. The low-rank guide at rank 20 recovered *X*, with +coverage of 0.81 and 0.76. MAP with the Laplace covered *X* and *k* and *m* +best, with the widest intervals and the noisiest point. The arrowhead +Laplace covered *X* at 0.97 and 0.94. On the full real library the low-rank +guide slid along the ridge between *m* and *X* and ended far worse than the +MAP, so MAP and the arrowhead Laplace is the full-library route. In every +study, every variational guide collapsed the uncertainty in *k* and *m*: +expect *X* intervals from a guide to undercover somewhat and *k* and *m* +intervals to be too narrow (``planning/studies/svi-overconfidence/``). These +are results on particular simulated designs and one real library, not +general guarantees. + +Guides +------ + +For ``svi``, ``--guide_type`` picks the variational family: + +``component`` (default) + The mean-field guide assembled from the model components. +``auto_normal``, ``auto_diagonal_normal`` + NumPyro's mean-field normal autoguides. +``auto_multivariate_normal`` + A dense multivariate normal. Its memory grows with the square of the + number of parameters, and a warning is printed above 4 GB. +``auto_low_rank_multivariate_normal`` + A normal with a low-rank covariance. ``--guide_rank`` sets the rank, + numpyro's default when omitted. Rank 20 worked in the relative-fit + study. +``delta`` + A point mass, which is MAP through the SVI path. + +NumPyro class names such as ``AutoNormal`` also work. The guide flags are +refused with ``map`` and ``nuts``. A resumed fit must use the guide its +checkpoint was written with. + +``--guide_init_scale``, default 1e-4, sets the guide's starting width in +each parameter's unconstrained units. For ``component`` it caps every guide +scale at the start, and for an autoguide it is numpyro's ``init_scale``. It +must be small. The same number applies to every parameter, and at 0.1 a +growth rate per minute starts with an SD of many ln units over a selection. +SVI then started about 1000 times above the warm-up's loss and descended +into other optima. SVI widens the scales itself. ``--init_param_jitter``, +default 0, multiplies the component guide's starting values by random noise. +Leave it off when starting from a MAP point, because 0.1 moves a starting +abundance near 15 by about 1.5 ln units. + +Starting points +--------------- + +A fresh MAP starts at the configured guesses. ``svi`` and ``nuts`` first run +a MAP warm-up for at most ``--pre_map_num_epoch`` epochs, default 10000, which +stops early when it converges. Its files carry the ``{out_prefix}_premap`` +prefix. ``--pre_map_num_epoch 0`` skips it. + +``--init_from params.npz`` starts the fit at the point saved by an earlier +MAP fit, its ``{out_prefix}_params.npz``. Every site that file names starts +there, and every other site starts at its guess. The earlier fit may be of a +different model, such as a fit without tube offsets. + +Staged MAP +^^^^^^^^^^ + +The starting point matters because Adam moves every parameter by about one +step size per step. On real data a cold MAP with level tube offsets carried +the offsets and *k* and *m* by whole units in its first window, the +step-size cuts then froze them there, and the offsets settled near ±2.8. +Starting from a MAP without offsets plus each tube's best offset was 1.2e5 +nats better. ``tfs-fit-model`` builds that start itself. A fresh MAP of a +model with ``sample_offset: level`` runs three MAPs: + +1. The tube offsets held at 0 (and a learned offset SD held at its prior + scale, since with every offset at 0 its MAP is 0). Files + ``{out_prefix}_stage1_*``. +2. Every other site held at stage 1's MAP, so the offsets are the only + latents. Given the rest the tubes do not couple, so this is each tube's + best offset. Files ``{out_prefix}_stage2_*``. +3. The joint MAP, started at stage 1's point plus stage 2's offsets, at + ``--staged_step_size`` (default 1e-4). This writes the run's own + ``{out_prefix}_*`` files. + +Each stage writes its own checkpoint and convergence record. Rerunning the +same command reuses a stage whose ``_params.npz`` exists and resumes one +that was interrupted, so to redo a stage delete its files. To resume the +joint stage, pass its checkpoint with ``--checkpoint_file``. +``--stage_offsets`` is ``auto`` by default: it stages a fresh level-offset +MAP and nothing else (not SVI, not a resumed fit, not ``--init_from``). +``off`` fits in one go; ``on`` insists on staging. + +On the dev data the offset mode is not only a trap for the optimizer. Run to +convergence from the hand-built start, a fit moved into it and scored 7.9e4 +nats better than the staged MAP, with offsets that follow IPTG in the +selection conditions and unphysical *k* and *m*. The staged MAP keeps the +fit in the physical basin, but a longer or better fit can leave it. +``tfs-summarize-fit`` checks every level-offset fit for this: it reports the +tube offsets in prior SDs and flags a fit whose offsets trend with titrant or +time within a condition (see :doc:`summarize-fit`, "Tube offsets"). The cause +is filed for study (``planning/offset-mode-growth-transition.md``). + +``--checkpoint_file`` resumes a fit from its ``{out_prefix}_checkpoint.pkl`` +at the checkpoint's step size and convergence state. It cannot be combined +with ``--init_from``. A new fit refuses to overwrite an existing checkpoint, +so resume it, delete it or change ``--out_prefix``. + +Outputs +------- + +With the default prefix ``tfs_fit_model``: + +``tfs_fit_model_checkpoint.pkl`` + The optimizer state, written every ``--checkpoint_interval`` epochs, + default 10. It records the guide, step size, convergence state and + provenance. ``tfs-sample-posterior`` reads it. +``tfs_fit_model_params.npz`` + The fitted parameters. For a MAP fit these are the ``{site}_auto_loc`` + arrays that ``--init_from`` reads. +``tfs_fit_model_convergence.csv``, ``tfs_fit_model_losses.txt`` + The convergence record, described below. +``checkpoints/`` + Numbered checkpoints every ``--epoch_checkpoint_interval`` epochs, default + 1000. 0 disables them. + +A NUTS fit also writes its posterior file. + +Convergence +=========== + +``tfs-fit-model`` for MAP and SVI, the MAP warm-up and +``tfs-prefit-calibration`` all stop the same way. Optimization runs in +windows of ``--convergence_window_steps`` optimizer steps, default 2000 and +never fewer than 10 epochs, so that with mini-batches every genotype is seen +several times per window. At the end of each window the monitor asks three +questions. + +**Is the loss still improving?** A line through the medians of 20 blocks of +the window's losses gives the drop per window and its standard error. The +loss is improving if the drop exceeds ``--convergence_z`` standard errors, +default 3, and ``--loss_rtol`` of the loss, default 1e-6. The loss is judged +against its own noise, so a noisy SVI loss neither stops early nor runs +forever. A rising or oscillating loss is not improving. Before acting on +``--patience`` stalled windows, default 3, the monitor fits one line through +all of them. A slow descent hidden by noise in each window shows up there, +and a significant drop resets the stall count. + +**Is any parameter still moving?** Each parameter's trend over the window is +measured in units of its posterior SD: the paired guide scale for SVI, the +prior SD for MAP, and log or logit units for positive or bounded +parameters. The posterior SD is floored at 1% of the prior SD, because a +mean-field guide can shrink a hierarchical scale's SD far below any honest +width, and a negligible creep in those units would never settle. Movement +explained by noise, or smaller than the optimizer can resolve at the current +step size, does not count. A parameter moving more than +``--param_tolerance``, default 0.05, is moving. + +**Does the median represent the mean?** The loss being minimized is a mean, +but the test uses medians. If rare draws carry a huge penalty, the medians +ignore them. The window's skew, how many robust SDs the mean sits above the +median, exposes this. Benign noise stays below about 1, and the monitor +treats a skew above 3 as a hidden penalty. + +The decisions follow from those answers. Above the final step size, after +``--patience`` windows in a row without loss improvement, the step size is +multiplied by ``--adam_step_size_cut``, default 0.1, down to +``--adam_final_step_size``, default 1e-6, starting from ``--adam_step_size``, +default 1e-3. Only the loss decides a cut. Some directions never settle, and +holding the step size high for them wastes time, while a smaller step does +not remove rare penalties. At the final step size the run converges after +``--patience`` windows in which the loss has not improved, no parameter has +moved and the skew is at most 3. A still-moving parameter or a skewed loss +keeps the run going and is named in the log. + +**A MAP is judged by its exact loss.** A MAP's objective is a fixed +function of the current point, so the only noise in its window losses comes +from the mini-batches. On a full library that noise is large: at batch size +4096 of 218,000 genotypes the trend's SE was about 5e4 nats per window, so a +descent of 1e5 nats per window read as a stall and the step size was cut to +its floor while the fit was still descending. A MAP fit (``tfs-fit-model +--analysis_method map``, every stage of the staged MAP, the SVI pre-MAP and +``tfs-prefit-calibration``) therefore computes the exact full-batch loss, +``-log p``, at the end of each window: one forward pass over the library, +in chunks of the batch size, a few percent of the run's time. A line +through the last ``--patience`` + 1 exact losses gives the drop per window +and its SE, which now measures only the optimizer's jitter about its path. +The window is improving when the drop exceeds both ``--convergence_z`` +times the SE and ``--loss_rtol`` times the loss. A steady descent passes +however slow it is; jitter about a level does not. The skew check is +skipped, since the exact loss is the objective itself. SVI keeps the +mini-batch test: its ELBO is noisy by nature. + +A window whose loss falls below −1000 times the magnitude of the run's first +block ends the run as ``diverged``. That means the objective is unbounded, +from a density singularity or from float32 error in a dense guide's own +density. It is not convergence. + +``--max_num_epochs``, default 100000, is a cap only. A hierarchical MAP +often has no finite optimum, because a hyperparameter scale can shrink +forever, so MAP runs often end at the cap. A run that reaches it says so and +names the largest remaining movement. + +The record +---------- + +``{out_prefix}_convergence.csv`` has one row per window: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Column + - Meaning + * - ``step``, ``epoch``, ``step_size`` + - Where the window ended and the step size it ran at. + * - ``loss``, ``loss_mean`` + - Median and mean mini-batch loss over the window. + * - ``loss_exact`` + - The exact full-batch loss at the end of the window (MAP only). The + loss test used it, and two MAP runs on the same model can be + compared by it. + * - ``loss_drop``, ``loss_drop_se``, ``loss_t`` + - Drop per window, its standard error and their ratio. + * - ``loss_improving`` + - Whether the loss test passed. + * - ``loss_skew``, ``loss_skewed`` + - The skew and whether it exceeds 3. + * - ``worst_param``, ``worst_param_excess``, ``worst_param_drift`` + - The parameter moving most, its movement beyond noise and its raw + drift. + * - ``params_moving`` + - Whether any parameter exceeds ``--param_tolerance``. + * - ``plateau``, ``plateau_count`` + - Whether this window stalled, and the run of stalled windows. + * - ``pooled_loss_t`` + - The pooled trend's t statistic, when one was computed. + * - ``decision`` + - ``continue``, ``cut``, ``converged`` or ``diverged``. + +``{out_prefix}_losses.txt`` has columns ``epoch``, ``loss``, ``step`` and +``step_size``, one row per block of steps, where ``loss`` is the block's +median. + +The optimizer +------------- + +The optimizer is plain Adam. ``--adam_clip_norm`` opts into numpyro's +``ClippedAdam``, which clips each gradient element to that value. Leave it +off. These losses have gradients of 1e3 to 1e6 per element, so a clip +applies to every element on every step, and Adam then follows the sign of +each draw's gradient. Its fixed point is where those signs balance, not +where the expected gradient is zero. With a rare large penalty, such as a +steep, precisely measured binding curve that an occasional draw crosses, +that point sat in violation for 5 to 90% of draws on every +congression-calibration run. Without clipping the same fits converged. + +``--elbo_num_particles``, default 2, sets the draws per SVI step. + +Sample the posterior +==================== + +``tfs-sample-posterior config_file checkpoint_file`` writes +``{out_prefix}.h5``, default ``tfs_posterior.h5``, which the prediction and +extraction commands read. The checkpoint type decides what happens. An SVI +checkpoint draws ``--num_posterior_samples`` samples from the guide, default +10000. A NUTS checkpoint uses its saved samples. A MAP checkpoint, including +SVI with the ``delta`` guide, becomes a Laplace approximation chosen by +``--laplace``: + +``auto`` (default) + ``full`` up to ``--laplace_max_params`` MAP parameters, default 20000, and + ``arrowhead`` above. +``full`` + The Laplace from the dense Hessian. Its memory grows with the square of the + parameter count, which puts a full library out of reach. +``arrowhead`` + The same Gaussian computed one genotype at a time. Given the shared + parameters the genotypes do not couple, so the Hessian is per-genotype + blocks plus their coupling to the shared parameters: *k*, *m*, + hyperparameters and tube offsets. The shared parameters are drawn from + their marginal and each genotype from its conditional. It runs on a full + library. +``blocks`` + Per-genotype blocks with the shared parameters held at the MAP. It leaves + out the *k* and *m* uncertainty. +``point`` + The MAP point itself, one sample and no Hessian. + +Each Hessian eigenvalue is floored at the prior's curvature along its +direction, so no direction of the Laplace is wider than the prior. A MAP that +stopped at its epoch cap is not exactly at an optimum, and the arrowhead +Laplace may find directions of the shared block with negative curvature. +Those directions are held at the MAP and written to +``{out_prefix}_held_directions.csv``, since they have no interval. Its +columns are ``direction``, ``eigenvalue``, ``parameter`` and ``loading``, +with the five largest loadings of each direction. On the relative-fit +simulations the arrowhead Laplace covered *k* and *m* at 0.92 and 1.0 under +Poisson noise and at 0.50 and 0.42 under realistic noise. + +``--skip_growth_observations`` leaves the per-observation growth sites out +of the file. They are most of its size, about 100 GB per site at 500 draws on +a 200,000-genotype library, and ``tfs-predict-growth`` recomputes growth from +the parameters. ``--hessian_chunk_size`` and ``--genotype_chunk_size`` trade +speed for memory if the device runs out. ``--sampling_batch_size`` and +``--forward_batch_size`` do the same for sampling. + +Per-genotype fits +================= + +``tfs-fit-genotypes`` is a non-Bayesian alternative. It fits the growth model +to each genotype independently by maximum likelihood, holding each +condition's *k* and *m* fixed at a calibration, and writes per-genotype Hill +parameters and *θ* predictions. It applies no congression correction and no +pooling across genotypes. + +.. code-block:: bash + + tfs-fit-genotypes growth.csv tfs_configure_priors.csv --num_workers -1 + +The calibration is a priors file written by ``tfs-prefit-calibration`` or a +wide table with columns ``condition_rep``, ``growth_k`` and ``growth_m``. The +default output prefix is ``tfs_fit_genotypes``. + +Running on a cluster +==================== + +``examples/tfmodel/run.srun`` is a Jinja template that ``tfs-setup-grid`` +renders into each run directory of a grid, not a standalone script. It shows +the joint route as a Slurm job. See :doc:`grid`. diff --git a/docs/source/grid.rst b/docs/source/grid.rst index 1c7e8f23..2f30da9b 100644 --- a/docs/source/grid.rst +++ b/docs/source/grid.rst @@ -1,96 +1,101 @@ -==================================== -Grid Setup and Summarisation -==================================== - -``tfscreen`` provides two scripts for setting up parameter sweeps over model -configurations or simulation settings, and one script for summarising the -results once runs are complete. - -Overview --------- - -Both grid-setup scripts share the same concept: - -1. You write a **grid YAML** file describing a base configuration and a set of - parameter axes, each with a list of variants. -2. The setup script takes the **Cartesian product** of all variant lists and - creates one subdirectory per combination. -3. Inside each subdirectory the script writes a per-run configuration file and - optionally renders a **Jinja2 template** (e.g. a Slurm submission script). -4. After all runs complete, ``tfs-summarize-grid`` collects results into a - summary CSV. +============= +Grids of runs +============= + +A grid is a sweep: the same pipeline run over every combination of a few +settings. ``tfs-setup-grid`` sets up a grid of model fits and +``tfs-setup-sim-grid`` a grid of simulations. ``tfs-summarize-grid`` and +``tfs-summarize-calibration`` collect the results once the runs finish. + +How a grid works +---------------- + +A **grid YAML** lists blocks of variants. The setup command takes the +Cartesian product of every block and makes one run directory per +combination. In each directory it writes the run's configuration, renders a +Jinja2 template (a shell or Slurm script) with that run's template +variables, and records the combination in ``combo.json``. It also writes +``grid_summary.json`` at the top, listing every run. + +Run directories are named ``run_{index:04d}_``, where +```` is the rendered ``run_name`` template with unsafe characters +replaced by underscores. Without a ``run_name`` the suffix is built from the +variable values. .. code-block:: text my_grid/ - ├── grid_summary.json ← written by tfs-setup-grid - ├── linear__instant__seed0/ - │ ├── combo.json ← variable assignments for this run + ├── grid_summary.json + ├── inputs/ copies of every input file + │ ├── binding.csv + │ ├── growth.csv + │ └── library_config.yaml + ├── run_0001_linear__instant__hill_geno__seed0/ + │ ├── combo.json │ ├── tfs_configure_config.yaml │ ├── tfs_configure_priors.csv │ ├── tfs_configure_guesses.csv - │ └── run.srun ← rendered Jinja2 template - ├── linear__instant__seed1/ - │ └── ... + │ ├── tfs_configure_library.csv + │ └── run.srun + ├── run_0002_linear__instant__hill_geno__seed1/ └── ... -tfs-setup-grid --------------- - -Sets up a grid of model-fitting runs. For each combination the script calls -``tfs-configure-model`` in the corresponding subdirectory, so all three -configuration files (``_config.yaml``, ``_priors.csv``, ``_guesses.csv``) are -already present when the grid is created. +A grid directory is self-contained, so it can be moved as a unit on and off +a cluster. Every input file a run needs is copied once into +``/inputs/``, and each run's config and script refer to it as +``../inputs/``. Two different files with the same name are kept apart +by a numeric suffix. Relative paths in the grid YAML resolve against the +grid YAML's own directory. Setup checks every combination before writing +anything and fails on a missing input file, an input that is a directory, +a config value naming a file the grid does not know to copy, or a template +that does not render. Launch each run from inside its own directory. + +Two kinds of variables go into the product. Variables in the +``configure_model`` or ``simulate`` blocks go into the run's configuration. +Variables in the ``template`` blocks go only into the rendered script. To +use a value in both places, list it in both. ``run_name`` can use either +kind. A variant with several keys keeps them together, so use one when +settings only make sense in combination. The ``basename`` filter strips the +directory from a file-valued variable: +``run_name: "{{ growth_df | basename }}__{{ condition_growth }}"``. + +Model grids +----------- + +``tfs-setup-grid`` calls ``tfs-configure-model`` in each run directory, so +the configuration files exist as soon as the grid does: .. code-block:: bash - tfs-setup-grid grid.yaml --out_prefix my_grid + tfs-setup-grid grid.yaml --out_dir my_grid -An :download:`annotated example <../../examples/tfmodel/grid.yaml>` is provided -in ``examples/tfmodel/``. Once the grid is set up, submit all jobs with: +The :download:`annotated example <../../examples/tfmodel/grid.yaml>` and +its Slurm template :download:`run.srun <../../examples/tfmodel/run.srun>` +are in ``examples/tfmodel/``. The template runs the pre-fit, a MAP fit, a +Laplace posterior, parameter extraction, growth and *θ* prediction, and +``tfs-cat-response``. Submit every run with: .. code-block:: bash - for d in my_grid/*/; do - cd "$d" && sbatch run.srun && cd - - done + for d in my_grid/run_*/; do (cd "$d" && sbatch run.srun); done -Grid YAML format -^^^^^^^^^^^^^^^^ +A shortened grid YAML: .. code-block:: yaml - # Directory name template (Jinja2); variables from both sections available. - run_name: "{{ condition_growth }}__{{ growth_transition }}__seed{{ seed }}" - - # Jinja2 template rendered into each subdirectory (relative to this YAML). + run_name: "{{ condition_growth }}__{{ theta }}__seed{{ seed }}" output_file: run.srun - # --- configure_model blocks ----------------------------------------------- - # Variables here are forwarded to tfs-configure-model. - # They are NOT injected into the Jinja2 template. - configure_model: - - # Fixed arguments — single variant = always selected. - name: data variants: - - binding_df: ../data/binding.csv - growth_df: ../data/growth.csv - library_config: ../data/run_config.yaml + - binding_df: data/binding.csv + growth_df: data/growth.csv + library_config: data/library_config.yaml - # 'auto' enumerates every registered component for an axis. - # Incompatible combinations are skipped automatically. - name: condition_growth auto: condition_growth - # Manual enumeration — list only the components you want. - - name: growth_transition - variants: - - growth_transition: instant - - growth_transition: baranyi - - # Joint (co-varying) block — keys in the same dict always move together. - name: theta_and_epistasis variants: - theta: hill_geno @@ -98,67 +103,56 @@ Grid YAML format - theta: hill_mut epistasis: true - # --- template blocks ------------------------------------------------------ - # Variables here are injected into the Jinja2 template only. - # They are NOT forwarded to tfs-configure-model. - template: - name: seed variants: - seed: 0 - seed: 1 - - name: predict_genotypes - variants: - - predict_genotypes_file: /path/to/predict_genotypes.txt - -Key rules: - -* The Cartesian product is taken across **all** blocks (``configure_model`` + - ``template``). -* ``configure_model`` variables accept any flag accepted by - ``tfs-configure-model`` (without the ``--`` prefix and ``_model`` suffix for - component axes; e.g. ``condition_growth`` rather than - ``--condition_growth_model``). -* Relative paths in ``configure_model`` blocks (``binding_df``, ``growth_df``, - ``library_config``, ``thermo_data``) are resolved relative to the grid YAML - and re-expressed relative to each subdirectory in the written config. -* The ``auto`` form enumerates every registered component for the given axis. - Incompatible combinations (e.g. ``power`` growth + ``logit`` theta_rescale) - are caught by ``tfs-configure-model``, skipped, and logged in - ``grid_summary.json``. -* Use the ``basename`` Jinja2 filter to strip directory paths from - file-valued variables in ``run_name``: - ``"{{ binding_df | basename }}__{{ condition_growth }}"``. - -tfs-setup-sim-grid ------------------- - -Sets up a grid of simulation runs. Each subdirectory receives a -``tfs_sim_config.yaml`` derived from a base config with per-run overrides -applied. +A ``configure_model`` variable is any ``tfs-configure-model`` argument +without the leading ``--``. Component choices drop the ``_model`` suffix: +``condition_growth`` rather than ``condition_growth_model``, and likewise +``growth_transition``, ``ln_cfu0``, ``dk_geno``, ``activity``, ``theta``, +``transformation``, ``theta_rescale``, ``theta_growth_noise``, +``theta_binding_noise``, ``growth_noise`` and ``sample_offset``. The input +files (``binding_df``, ``growth_df``, ``presplit_df``, ``base_growth_df``, +``library_config``, ``thermo_data``) are copied into ``inputs/``. + +``auto: `` in place of ``variants`` enumerates every registered +component on that axis. It is safe for small axes such as +``condition_growth`` or ``growth_transition``. For ``theta`` it also yields +``_simple``, a private component used only by the pre-fit, and every +thermodynamic model, so list theta variants by hand. A combination that +``tfs-configure-model`` refuses is skipped, and the reason is logged in +``grid_summary.json``. For example, the ``mixture`` transformation with its +default ``homodimer`` rule refuses any activity component but ``fixed``. + +Each run's configuration files are always ``tfs_configure_config.yaml``, +``tfs_configure_priors.csv``, ``tfs_configure_guesses.csv`` and, for a +growth model, ``tfs_configure_library.csv``. The template refers to them by +those names. + +Simulation grids +---------------- + +``tfs-setup-sim-grid`` writes a ``tfs_sim_config.yaml`` in each run +directory: a base simulate config with the run's overrides applied. .. code-block:: bash - tfs-setup-sim-grid simulate_grid.yaml --out_prefix my_sim_grid - -An :download:`annotated example <../../examples/simulate/simulate_grid.yaml>` -is provided in ``examples/simulate/``. + tfs-setup-sim-grid simulate_grid.yaml --out_dir my_sim_grid + for d in my_sim_grid/run_*/; do (cd "$d" && bash run.sh); done -Grid YAML format -^^^^^^^^^^^^^^^^ - -The simulate grid YAML follows the same structure as the model grid, with two -differences: - -1. A ``base_config`` key is required, pointing to the base ``simulate_config.yaml``. -2. The blocks are named ``simulate`` (not ``configure_model``). +The :download:`example grid <../../examples/simulate/simulate_grid.yaml>` +and its template :download:`run.sh <../../examples/simulate/run.sh>` are in +``examples/simulate/``. The grid YAML has the same form as a model grid, +with two differences. A ``base_config`` key names the base simulate config, +and the config blocks are called ``simulate``: .. code-block:: yaml - base_config: ../simulate_config.yaml - - run_name: "{{ theta_component }}__noise{{ tube_noise_sigma }}__seed{{ random_seed }}" + base_config: simulate_config.yaml + run_name: "{{ theta_component }}__noise{{ tube_noise_sigma }}__seed{{ seed }}" output_file: run.sh simulate: @@ -174,52 +168,89 @@ differences: - name: seed variants: - - random_seed: 0 - - random_seed: 42 + - seed: 0 + - seed: 42 template: - name: num_replicates variants: - num_replicates: 3 -Key rules: - -* ``simulate`` variables override top-level keys in the base config. Nested - keys are not supported — override the entire top-level key if needed. -* The ``auto`` form is not supported for simulate grids. - -Jinja2 template variables -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The rendered template receives all variables from the ``template`` blocks. -``simulate`` / ``configure_model`` variables are **not** available in the -template — add them to ``template`` as well if they are needed in both places. - -The ``run.srun`` and ``run.sh`` files in ``examples/tfmodel/`` and -``examples/simulate/`` show realistic Slurm and shell templates. Inside a -template, each run's configuration file is always named: - -* ``tfs_configure_config.yaml`` (model grid) -* ``tfs_sim_config.yaml`` (simulate grid) - -tfs-summarize-grid ------------------- - -Scans a grid directory for completed runs and writes a flat summary CSV. +A ``simulate`` variable replaces a top-level key of the base config. Nested +keys cannot be overridden one at a time, so to change one key inside a +block such as ``binding_data``, give the whole block as the variant. The +keys must be valid simulate-config keys (see :doc:`simulation`); setup does +not check them, but ``tfs-simulate`` refuses an unknown key when the run +starts. The random seed is ``seed``. ``auto`` is not available in simulation +grids. A library subsample (see :doc:`simulation`) is set the same way, +e.g. a block whose variants are ``library_subsample: {double-1-2: 500}`` and +``library_subsample: {double-1-2: 2000}``; keep ``library_subsample_seed`` +fixed so the runs compare nested libraries. + +File-valued simulate keys are copied into ``inputs/`` like model-grid +inputs: ``thermo_data``, ``design``, ``library_genotypes``, +``empirical.phenotype_model``, ``od600.calibration`` and any +``binding_data`` ``choose_by`` that names a file. A relative path in the base config resolves against the base config's +directory; one in a ``simulate`` override resolves against the grid YAML. + +A simulation grid becomes a calibration study when its template also +configures and fits each simulated data set and runs ``tfs-summarize-fit``. +Fit settings then go in the ``template`` blocks, since only the script +sees them. + +Summarizing a model grid +------------------------ + +``tfs-summarize-grid`` collects a model grid into one table: .. code-block:: bash tfs-summarize-grid my_grid -Output (default: ``my_grid/grid_summary.csv``): +It writes ``my_grid/grid_summary.csv`` (``--out_prefix`` changes the path) +with one row per run directory that has a ``combo.json``. The columns are +the run name, the ``configure_model`` and ``template`` variables, +``configure_complete`` (whether ``tfs_configure_config.yaml`` exists), and, +when the run has a ``*_fit_summary.json`` from ``tfs-summarize-fit``, its +statistics flattened into columns such as ``theta_training_rmse``, +``theta_test_rmse``, ``growth_training_rmse`` and ``final_loss``. +``tfs-summarize-fit`` writes that file to ``summary/`` by default, and +``tfs-summarize-grid`` looks only in the run directory itself, so run it +with ``--out_prefix tfs_summarize`` inside the run directory if the grid +summary should pick it up. + +Summarizing calibration across a simulation grid +------------------------------------------------ + +``tfs-summarize-calibration`` asks whether the posterior intervals of a +simulate-and-fit grid are honest. Each run directory needs its +``combo.json`` and, once finished, the ``tfs-summarize-fit`` outputs in +``summary/``, where the posterior quantiles are joined to the simulated +truth for *θ* and for every fitted parameter with ground truth. -* One row per subdirectory that contains a ``combo.json``. -* Columns from the ``configure_model`` / ``template`` variable assignments. -* ``configure_complete`` — whether ``tfs-configure-model`` finished (i.e. - ``tfs_configure_config.yaml`` is present). -* Flattened fit-summary statistics from ``*_fit_summary.json`` if present - (e.g. ``theta_training_rmse``, ``growth_training_rmse``, ``final_loss``). -* Calibration statistics from ``*_calib_stats.json`` if present (prefixed - with ``calib_``). +.. code-block:: bash -Use ``--out_prefix`` to write the CSV to a different location. + tfs-summarize-calibration my_sim_grid --baseline guide_type=component --facet_by guide_type + +For every run, quantity and stratum it computes the coverage of central +intervals from 50% to 99%, the mean calibration error and bias (negative +means intervals too narrow), PIT uniformity, mean interval widths, RMSE and +Pearson r. Widths are there so a method cannot look calibrated just by being +vague. Genotype-level quantities are also split by whether the genotype has +binding data and by its purity (spike, bulk or mixed), and *θ* by whether +the true value is resolvable (inside ``[regime_eps, 1 - regime_eps]``) or +saturated. Runs that share every grid variable except the replicate keys +(``--replicate_keys``, default ``seed`` and ``fit_seed``) form an arm and +are averaged. With ``--baseline key=value ...`` every other run is paired +with the baseline run fit to the same simulated data, and the paired +differences are reported. As with ``tfs-compare-runs``, every value is a +raw number with no thresholds or grades. + +Outputs, under ``--out_prefix`` (default ``tfs_calibration``): +``_runs.csv`` (per run, quantity and stratum), ``_run_status.csv`` (which +runs are finished and what is missing), ``_arms.csv`` (mean and SD over +replicates), ``_paired.csv`` and ``_paired_summary.csv`` (with +``--baseline``), a calibration-curve PDF for ``--plot_quantity`` (default +``theta_test``), and ``_metadata.json`` with the resolved settings. + +See :doc:`cli` for every flag of these four commands. diff --git a/docs/source/index.rst b/docs/source/index.rst index 18efbd57..32d351ac 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,13 +9,18 @@ tfscreen :caption: Contents: quickstart - simulation + screen-analysis + pipeline process-raw model-inputs - analysis + fitting + model + downstream summarize-fit + simulation grid ligandmpnn-features + cli Goal ---- @@ -33,32 +38,27 @@ The thermodynamic observable is operator occupancy (*θ*), inferred from *E. col The model in ``tfscreen.tfmodel`` is designed to jointly capture high-throughput growth data and low-throughput binding data, learning the quantitative relationship between operator occupancy and growth rate in the process. -Quick Start ------------ -New to ``tfscreen``? The :doc:`quickstart` page walks through a complete -simulate-and-analyze run using the bundled example in -``examples/simulate-and-analyze/``. - -Simulation ----------- -`tfscreen` allows you to simulate high-throughput screens starting from thermodynamic models of transcription factor binding and activity. See the :doc:`simulation` page for more details. - -Raw Data Processing -------------------- -`tfscreen` includes utilities to convert raw sequencing data into quantitative inputs for analysis. See the :doc:`process-raw` page for more details. - -Model Input Data ------------------ -The hierarchical model accepts binding, growth, base-growth, pre-split, and congression-calibration data. See the :doc:`model-inputs` page for what each contributes, how much is useful, and how to pass it in. - -Analysis --------- -`tfscreen` provides robust statistical tools, including Bayesian hierarchical models, to extract biochemical parameters from screen data. See the :doc:`analysis` page for more details. - -Grid Setup ----------- -Both the simulation and model-fitting workflows support parameter sweeps via a grid mechanism. See the :doc:`grid` page for the grid YAML format and the ``tfs-setup-grid``, ``tfs-setup-sim-grid``, and ``tfs-summarize-grid`` commands. - +Using tfscreen +-------------- + +:doc:`quickstart` runs the bundled simulate-and-fit example. +:doc:`screen-analysis` is the route for analyzing a real screen, from read +counts to per-genotype curves with intervals; start there with lab data. +:doc:`pipeline` walks through every step from sequencing reads to a +posterior, with the command for each. The pages after it cover each part +in detail: + +- :doc:`process-raw`: reads to counts, OD600 to tube totals, counts to the + growth table. +- :doc:`model-inputs`: what each data file and prior input gives the model. +- :doc:`fitting`: configuring, fitting, convergence and posteriors. +- :doc:`model`: the generative model and its components. +- :doc:`downstream`: parameters, predictions, epistasis, curve classes and + cross-run comparison. +- :doc:`summarize-fit`: diagnostics for one run. +- :doc:`simulation` and :doc:`grid`: simulated experiments and parameter + sweeps. +- :doc:`cli`: every command and flag, generated from the code. Indices and tables ================== diff --git a/docs/source/ligandmpnn-features.rst b/docs/source/ligandmpnn-features.rst index 1469cf82..20d35fcc 100644 --- a/docs/source/ligandmpnn-features.rst +++ b/docs/source/ligandmpnn-features.rst @@ -60,9 +60,9 @@ Create a YAML file with a top-level ``structures:`` mapping from short structure names to PDB file paths, plus ``n_chains_bearing_mut`` (the number of chains that carry each mutation: ``1`` for a monomer, ``2`` for a homodimer). The structure names must match what the downstream theta -component expects. For the four-state (``O2_C4_*``) topologies the required -keys are ``H``, ``HD``, ``L``, and ``LE2``, representing the four -thermodynamic states: +component expects. For the ``O2_C4_*`` models the required keys are ``H``, +``HD``, ``L``, and ``LE2``, the four structures of their ΔΔG basis (HD is H +bound to operator DNA, LE2 is L with both effector sites filled): .. code-block:: yaml @@ -162,6 +162,13 @@ For each structure name ``{name}`` the file contains a group with: ``{name}/n_chains_bearing_mut`` ``int32`` scalar: the ``n_chains_bearing_mut`` value from the input YAML. +``{name}/wt_aa`` + ``str`` (L,): the one-letter wild-type residue at each residue number, + read from the PDB (``X`` when unknown). Optional; files written before + it existed still load. When present, the loader checks it against the + library's wild-type residues and stops on a mismatch, which usually + means the PDB and the library number residues differently. + Using the HDF5 File in a Run Config ------------------------------------ @@ -170,14 +177,19 @@ selecting a ``PnnC`` theta component: .. code-block:: bash - tfs-configure-model binding.csv --growth_df growth.csv \ + tfs-configure-model --binding_df binding.csv --growth_df growth.csv \ --theta_model thermo.O2_C4_K3_U0_a.PnnC \ - --thermo_data ensemble.h5 + --thermo_data ensemble.h5 \ + --library_config library_config.yaml -The path is saved into ``tfs_configure_config.yaml`` as the top-level -``thermo_data`` key and read by ``tfscreen.tfmodel.generative.components.theta.thermo.io.load_struct_ensemble`` +The path is saved into ``tfs_configure_config.yaml`` under +``components.thermo_data`` and read by ``tfscreen.tfmodel.generative.components.theta.thermo.io.load_struct_ensemble`` at model-build time; you do not need to edit the config by hand. See ``configure_model_cli.py``'s ``thermo_data`` docstring for the full set of registry keys and their structure-name requirements (``PddG`` models instead take a hand-supplied ΔΔG CSV via the same flag — see ``load_ddG_prior_csv``). + +Every library residue must be in every structure. If the PDB numbering is +offset from the library's, ``--thermo_residue_offset N`` adds N to the file's +residue numbers. diff --git a/docs/source/model-inputs.rst b/docs/source/model-inputs.rst index f278ff6d..0828ada3 100644 --- a/docs/source/model-inputs.rst +++ b/docs/source/model-inputs.rst @@ -1,237 +1,467 @@ -================= -Model Input Data -================= +============ +Model inputs +============ -``tfs-configure-model`` (see :doc:`analysis`, Step 1) accepts up to five -pieces of experimental input. Only ``binding_df`` is strictly required; -everything else is optional and adds additional constraints to the -model. This page describes what each input contributes to the fit, the -rough amount of data that is useful in practice, and how it is passed in. +``tfs-configure-model`` builds a model from data files and a few +experimental numbers, and writes the configuration ``tfs-fit-model`` +reads. This page describes what each input contributes to the fit and how +to pass it. The commands that produce the data files are in +:doc:`process-raw`, every flag is in :doc:`cli`, and the model components +are in :doc:`model`. + +At least one of ``--growth_df`` and ``--binding_df`` is required. Both are +flags. With both, the joint model is configured; with growth only, a +growth-only model; with binding only, a binding-only model. .. list-table:: :header-rows: 1 - :widths: 20 15 65 + :widths: 26 24 50 - * - Input - - Required? + * - Flag + - When - What it anchors - * - ``binding_df`` - - Yes - - Absolute scale/shape of *θ*, independent of growth - * - ``growth_df`` - - No (binding-only model if omitted) - - ln_cfu\ :sub:`0`, dk_geno, activity, and *θ* jointly, via growth rate - * - ``base_growth_df`` - - No - - The condition-growth baseline / dk_geno identifiability confound - * - ``presplit_df`` - - No - - ln_cfu\ :sub:`0` directly, from pre-split sequencing counts - * - ``transformation_lambda`` - - No (required by some ``transformation_model`` choices) - - The congression-correction prior - -Binding Data (``binding_df``) -============================== - -**Role** - -Direct, low-throughput measurements of operator occupancy (*θ*) as a -function of titrant concentration, independent of any growth-rate -observation. Because it measures *θ* directly rather than through the -growth likelihood, binding data anchors the absolute scale and shape of -the occupancy curve — resolving degeneracies (e.g. between *θ* and -activity *A*) that growth data alone cannot separate. It is the only -required input to ``tfs-configure-model``: with ``growth_df`` omitted, a -binding-only model is configured that infers *θ* from ``binding_df`` -alone. - -**Scale** - -Binding curves are comparatively expensive to collect, so useful fits -typically rely on **dozens** of measured genotype/titrant curves (e.g. a -handful of calibration genotypes each measured across ~5-10 titrant -concentrations) rather than a full library. Because growth data typically -outnumber binding rows by several orders of magnitude, the binding -log-likelihood is upweighted by ``binding_weight`` (auto-computed as -``N_growth_rows / N_binding_rows`` unless overridden) so each binding -observation carries a comparable gradient contribution to the average -growth observation. - -**Format** - -A CSV (or ``pd.DataFrame``) with one row per (genotype, titrant_name, -titrant_conc) measurement: + * - ``--growth_df`` + - The screen itself + - ``ln_cfu0``, ``dk_geno``, activity and *θ* jointly, through growth + * - ``--library_config`` + - Required with growth data + - Spiked genotypes, purity and pool shares for the congression model + * - ``--binding_df`` + - Optional + - The absolute scale and shape of *θ* + * - ``--presplit_df`` + - Optional + - ``ln_cfu0`` directly, from the presplit tube + * - ``--base_growth_df`` + - Optional + - The growth baseline against ``dk_geno`` + * - ``--transformation_lambda`` + - Required by ``mixture`` + - The congression rate + * - ``--theta_gauge_conc`` + - ``hill_relative`` only + - Where wt's relative X is 1 and 0 + * - ``--thermo_data`` + - Thermodynamic *θ* models + - Structural features of each mutation + * - ``--thermo_ddG_units``, ``--thermo_temperature_K``, + ``--thermo_column_map``, ``--thermo_residue_offset``, + ``--thermo_max_missing_fraction`` + - Optional + - How ``--thermo_data`` is read: ΔΔG units, column names, residue + numbering, coverage warning + * - ``--set_priors``, ``--growth_priors``, ``--growth_priors_wt_rates`` + - Optional + - Prior values, without editing the priors CSV + +Growth data +----------- + +``--growth_df`` is the table ``tfs-process-counts`` writes (or +``tfs-simulate`` for a simulated experiment). It is the high-throughput +part of the experiment: one sequencing run reports every genotype in a +tube, so a screen has hundreds of thousands of rows. Growth identifies each +genotype's starting abundance, its pleiotropic growth effect ``dk_geno``, its +activity and its occupancy *θ* through + +``ln_cfu = ln_cfu0 + (k_pre + dk_geno + m_pre·A·θ)·t_pre + (k_sel + dk_geno + m_sel·A·θ)·t_sel`` + +The model reads these columns: .. list-table:: :header-rows: 1 - :widths: 25 75 + :widths: 26 74 * - Column - Meaning * - ``genotype`` - - Genotype string (``wt``, ``M42I``, ``M42I/K84L``, ...) + - Genotype name. + * - ``library`` + - The transformed library. Two libraries may share condition names. + * - ``replicate`` + - Biological replicate; 1 when absent. + * - ``condition_pre``, ``condition_sel`` + - Pre-selection and selection conditions. + * - ``t_pre``, ``t_sel`` + - Minutes of pre-selection and selection growth. + * - ``titrant_name``, ``titrant_conc`` + - Titrant and its concentration. + * - ``ln_cfu``, ``ln_cfu_var`` + - The genotype's cells in the tube, in log form, and its variance. + ``ln_cfu_std`` may be given instead of the variance, or ``cfu`` with + ``cfu_std`` or ``cfu_var``. + * - ``counts`` + - Reads of the genotype in the tube. Count likelihood only. + * - ``sample_reads`` + - The tube's total reads, ``__unknown__`` included. Count likelihood + only. Older files without it can supply ``adjusted_counts`` and + ``frequency``, from which it is derived about 1% high, by the same + factor for every genotype in a tube. + * - ``sample_ln_cfu`` + - The tube's total cells, in log form. Count likelihood only. + ``sample_cfu`` with ``sample_cfu_std`` or ``sample_cfu_var`` works + too. + +**Likelihood.** The default, ``--growth_likelihood counts``, observes the +reads directly. Each genotype's count in a tube is negative binomial with +mean equal to the tube's reads times the genotype's predicted frequency, +and a learned dispersion whose variance has a part proportional to the mean +and a quadratic part. There is no pseudocount, so a genotype with zero +reads is an observation rather than a floor. Real counts vary several times +more than Poisson, and the dispersion describes that. The count likelihood +needs ``--growth_noise_model zero``, the default. It pairs with the default +``--sample_offset_model level``: one ``ln_cfu`` offset per tube, shared by +every genotype in it, with a learned SD. A tube's total enters the count +likelihood as given, so the level offset is what absorbs an error in it. +The SD columns of the totals are not used. ``--set_priors sigma_fixed=0.17`` +holds that SD instead of learning it. On simulations the count likelihood +cut the RMSE of *θ* by 35 to 60% against the alternative. + +The alternative, ``--growth_likelihood lncfu``, observes each row's +``ln_cfu`` with a Student-t likelihood whose scale is the row's +``ln_cfu`` SD. It needs none of the count columns. + +Binding data +------------ + +``--binding_df`` holds direct, low-throughput measurements of *θ* against +titrant concentration. Because they measure *θ* without going through +growth, they fix its absolute scale and shape and separate effects growth +alone cannot, such as *θ* against activity *A*. Useful fits rely on dozens +of curves, a handful of genotypes each measured at 5 to 10 concentrations. +Growth rows outnumber binding rows by orders of magnitude. The binding +log-likelihood is weighted by ``--binding_weight``, 1 by default, so each +binding observation counts once. ``tfs-process-binding`` makes this table +from a lab table of single measurements. + +.. list-table:: + :header-rows: 1 + :widths: 26 74 + + * - Column + - Meaning + * - ``genotype`` + - Genotype name. * - ``titrant_name`` - - Name of the titrant (matches ``growth_df`` naming when both are used) + - Titrant; it must match the growth table's naming. * - ``titrant_conc`` - - Titrant concentration (float) + - Titrant concentration. * - ``theta_obs`` - - Measured fractional occupancy, in [0, 1] + - Measured fractional occupancy. * - ``theta_std`` - - Standard deviation / uncertainty of ``theta_obs`` + - SD of ``theta_obs``. -Passed as the required positional argument: +In a joint model, binding rows whose genotype and titrant do not appear in +the growth table are dropped with a message. .. code-block:: bash - tfs-configure-model binding.csv --growth_df growth.csv - -Growth Data (``growth_df``) -============================== - -**Role** - -High-throughput growth-rate observations (``ln_cfu`` over time, per -genotype/replicate/condition), produced by ``tfs-process-counts`` or -``tfs-simulate``. This is the primary data source: it is what makes the -approach *high-throughput*, and it jointly identifies ln_cfu\ :sub:`0`, -``dk_geno`` (pleiotropic growth effect), activity, and *θ* through the -growth likelihood -``ln_cfu = ln_cfu0 + (k_pre + dk_geno + m_pre·A·θ)·t_pre + (k_sel + dk_geno + m_sel·A·θ)·t_sel``. -Omitting it configures a binding-only model (see above). - -**Scale** - -Growth data is cheap per-genotype relative to binding data, since a -single sequencing run reports abundances for the whole library -simultaneously. Useful fits typically involve **hundreds of thousands** -of rows — the product of library size (thousands of genotypes), -replicates, conditions, titrant concentrations, and timepoints. - -**Format** - -A CSV with exactly the schema produced by ``tfs-process-counts`` / -``tfs-simulate``; see :doc:`process-raw` for the full column -specification (``genotype``, ``library``, ``replicate``, -``condition_pre``, ``condition_sel``, ``titrant_name``, ``titrant_conc``, -``t_pre``, ``t_sel``, ``ln_cfu``, ``ln_cfu_std``). - -Passed via the optional flag: + tfs-configure-model --binding_df binding.csv --growth_df growth.csv \ + --library_config library_config.yaml + +Without binding data, growth fixes *θ* only up to an affine map: the growth +slope ``m`` and baseline ``k`` absorb any rescaling of *θ*, so its absolute +scale rests on the priors. A growth-only model has no binding likelihood, +so ``--theta_binding_noise_model`` stays ``zero`` and ``--binding_weight`` +stays unset. ``tfs-prefit-calibration`` calibrates the growth link on the +binding genotypes, so it refuses a growth-only model. The growth-only route +is ``--theta_model hill_relative`` with its growth priors set from wt +monoculture rates; see `Prior inputs`_ and :doc:`fitting`. + +Library description +------------------- + +``--library_config`` is the library YAML, the same file given to +``tfs-process-fastq``; its keys are described in :doc:`process-raw`. It is +required with growth data and refused without it. + +``tfs-configure-model`` resolves it into ``{out_prefix}_library.csv``, one row +per genotype with ``is_wt``, ``in_spiked_origin`` (encoded by a spiked +sequence), ``pool_fraction`` (its expected share of the pool) and +``bulk_fraction`` (the share of its cells that come from the bulk, +congression-prone sub-libraries rather than from a monoclonal spike). +``tfs-fit-model`` reads this snapshot, never the YAML, so the CSV is the +place to override the design's numbers. ``in_spiked_origin`` sets which +genotypes get the spiked ``ln_cfu0`` prior. ``bulk_fraction`` and +``pool_fraction`` feed the ``mixture`` congression model. + +Two checks run against the data. Every genotype in the growth, presplit and +base-growth tables must be in the library; a mismatch usually means the +counts were called with a different config, and a residue-numbering +difference mismatches nearly every mutant. Every spiked genotype must have +growth data; a missing spike also usually means a different config. +``--allow_missing_spikes`` turns the second check into a warning, for a +spike that really dropped out. + +``library_mixture`` cannot be checked: no data file ever saw it. It is +recorded as given in the configuration's ``library`` block, and that record +is its only defense. Give the best estimate of what went into the pool. + +A simulation's library YAML can restrict the library to a subsample +(``library_genotypes``, ``library_subsample``; see :doc:`simulation`). Given +that same YAML, ``tfs-configure-model`` builds the snapshot from the same +subset, with the composition the simulation used, and records the +subsample keys under ``library.subsample``. + +Presplit data +------------- + +``--presplit_df`` holds the abundances measured in the presplit tube, before +the culture was divided into tubes (t = -t_pre). It constrains each covered +genotype's starting abundance ``ln_cfu0`` directly instead of leaving it to +the extrapolation of the growth fit. It is cheap, one tube per library, +replicate and pre-selection condition, so it can cover the whole library. + +``tfs-process-counts --presplit`` writes it, with columns ``library``, +``replicate``, ``condition_pre``, ``genotype``, ``ln_cfu`` and ``ln_cfu_std``. +Rows for genotypes not in the growth table are dropped. Genotypes in the +growth table but not in the presplit table are kept and simply have no +presplit constraint. .. code-block:: bash - tfs-configure-model binding.csv --growth_df growth.csv - -base_growth Data (``base_growth_df``) -======================================== - -**Role** - -Direct, reference-condition growth-rate measurements for a small subset -of genotypes (``wt`` at minimum). The ``condition_growth`` components' -per-condition baseline (*k*) and the per-genotype pleiotropic effect -(``dk_geno``) are only jointly identified up to an additive constant — -``k += C, dk_geno -= C`` leaves the growth likelihood unchanged. Because -``dk_geno`` is fixed to 0 for ``wt``, a direct measurement of wt's growth -rate anchors the new ``k_ref`` scalar via -``rate_obs ~ Normal(k_ref + dk_geno, rate_std)``, resolving this -identifiability confound. See the **Per-condition growth priors** section -of ``CLAUDE.md`` for the full mechanism (this is the complementary, -insufficient-alone anchor; per-condition priors pinned during -``tfs-prefit-calibration`` are the primary fix). - -**Scale** + tfs-configure-model --growth_df growth.csv --library_config library_config.yaml \ + --presplit_df presplit.csv -Small. ``wt`` is required after filtering against ``growth_df``; beyond -that, ``base_growth_df`` may cover anywhere from just ``wt`` up to a -handful of additional genotypes. Multiple rows for the same genotype are -combined via inverse-variance weighting. +Base growth data +---------------- -**Format** +``--base_growth_df`` holds direct growth-rate measurements in a reference +condition for a few genotypes, wt at minimum. The condition baselines ``k`` +and the genotype effects ``dk_geno`` are identified only up to a shared +constant: ``k + C`` with ``dk_geno - C`` leaves the growth likelihood +unchanged. wt's ``dk_geno`` is fixed at 0, so a measurement of wt's rate +pins a reference rate ``k_ref`` through +``rate ~ Normal(k_ref + dk_geno, rate_std)``. This anchor is complementary; +the per-condition priors set by ``tfs-prefit-calibration``, or by +``--growth_priors``, are the main fix. .. list-table:: :header-rows: 1 - :widths: 25 75 + :widths: 26 74 * - Column - Meaning * - ``genotype`` - - Genotype string; rows not present in ``growth_df`` are dropped + - Genotype name. Rows not in the growth table are dropped, and wt must + remain. * - ``rate`` - - Measured reference-condition growth rate + - Measured growth rate in the reference condition, per minute. * - ``rate_std`` - - Standard deviation of ``rate``. **Must be strictly positive** — a - value of 0 (e.g. from an unset ``noise`` in simulated data) causes - a division-by-zero that surfaces as a NaN-prior crash the first - time the model is traced (at ``tfs-prefit-calibration`` or - ``tfs-fit-model``, not at ``tfs-configure-model`` time). + - SD of ``rate``. It must be greater than 0. -Passed via the optional flag: +Several rows for one genotype are combined by inverse-variance weighting. .. code-block:: bash - tfs-configure-model binding.csv --growth_df growth.csv \ + tfs-configure-model --growth_df growth.csv --library_config library_config.yaml \ --base_growth_df base_growth.csv -Pre-split Data (``presplit_df``) -=================================== - -**Role** - -Sequencing-derived abundance measurements taken *before* the library is -divided into separate selection conditions (t = -t_pre). Genotypes it -covers get a direct constraint on their initial population -(ln_cfu\ :sub:`0`) instead of relying solely on the extrapolation implicit -in the growth-rate fit. +Congression rate +---------------- -**Scale** +``--transformation_lambda MEAN STD`` is not a data file. It is a measured +congression rate, the tendency of a transformed cell to take up more than +one plasmid, as a mean and SD in linear space. It sets a LogNormal prior on +the ``mixture`` transformation's lambda. It is required when +``--transformation_model`` is ``mixture`` and refused when it is ``single``, +the default. -Ideally the whole library, since it is collected from a single pooled -sample at one timepoint (cheap relative to a full growth time-course). -Genotypes present in ``growth_df`` but absent from ``presplit_df`` are -not an error — they are kept and their ln_cfu\ :sub:`0` is masked out of -this constraint. - -**Format** - -Produced by ``tfs-process-presplit``; see :doc:`process-raw` for the full -column specification (``library``, ``replicate``, ``condition_pre``, -``genotype``, ``ln_cfu``, ``ln_cfu_std``). - -Passed via the optional flag: +Lambda is the rate of a zero-truncated Poisson. A transformant, a cell that +took up at least one plasmid and survived selection, carries ``M`` plasmids +with ``P(M = m) = Poisson(m; lambda) / (1 - exp(-lambda))``. A measured mean +number of distinct plasmids per transformant, +``E[M] = lambda / (1 - exp(-lambda))``, has to be converted to lambda first. +This is the same lambda as the simulator's +``transformation_poisson_lambda``. .. code-block:: bash - tfs-configure-model binding.csv --growth_df growth.csv \ - --presplit_df presplit.csv + tfs-configure-model --binding_df binding.csv --growth_df growth.csv \ + --library_config library_config.yaml \ + --transformation_model mixture --transformation_lambda 0.36 0.05 + +Relative-X gauge +---------------- + +``--theta_gauge_conc C_LO C_HI`` applies only to ``--theta_model +hill_relative``, which fits growth alone on a wt-relative scale X: wt's X is +1 at ``C_LO`` and 0 at ``C_HI``. The default is the lowest and highest +titrant concentration in the growth table, and the resolved values are +written to the configuration. Other theta models refuse the flag. + +Structural data +--------------- + +``--thermo_data`` is required by the ``PddG`` and ``PnnC`` thermodynamic +theta models and ignored by the others. For ``PnnC`` models it is the HDF5 +file written by ``scripts/generate_struct_ensemble.py``; see +:doc:`ligandmpnn-features`. For ``PddG`` models it is a CSV with a mutation +column and one column of prior mean ΔΔG per structure: ``H``, ``HD``, ``L`` +and ``LE2`` for the ``O2_C4`` models, and ``H``, ``HO``, ``L``, ``LO``, +``HE2`` and ``LE2`` for the ``O2_C12`` models. + +The mutation column may be called ``mut``, ``Mutation`` or ``mutation``, with +labels like ``H29A``. An unnamed pandas index column is dropped. Columns the +model does not use are ignored, and the loader prints their names. A missing +structure column is an error that lists the columns found. +``--thermo_column_map HD=HO`` reads a model structure from a column with +another name. Rows that are not single mutations are skipped. + +**Units and sign.** ΔΔG\ :sub:`X` is the change a mutation makes to the free +energy of state X, so a positive value destabilizes X. For an equilibrium +constant K\ :sub:`A→B` = [B]/[A] the models use +Δln K\ :sub:`A→B` = ΔΔG\ :sub:`A` − ΔΔG\ :sub:`B`, which needs ΔΔG in units of +kT. By default the CSV values are used as they are, in kT. +``--thermo_ddG_units kcal/mol`` (or ``kJ/mol``) divides them by RT once at +load, at ``--thermo_temperature_K`` (default 298.15 K). At 298 K one kcal/mol +is 1.69 kT, so kcal/mol values read as kT understate every effect by that +factor. A predictor that reports folding ΔG, where negative means more +stable, must be sign-flipped first. + +**Projection.** The effector constants are per site, and a doubly liganded +structure XE2 has weight K\ :sub:`x_e`\ :sup:`2`\ ·e\ :sup:`2` relative to X, +so ΔΔG\ :sub:`X` − ΔΔG\ :sub:`XE2` is twice Δln K\ :sub:`x_e`. The models +map ΔΔG to constant shifts as follows: -transformation_lambda -======================== - -**Role** - -Not a data file — a single experimentally measured ``(mean, std)`` pair -describing plasmid congression (multiple plasmids entering one cell -during transformation), in linear space (e.g. ``(0.36, 0.05)``). It is -used to moment-match a LogNormal prior for the transformation model's -lambda parameter, replacing the manual step of hand-editing the priors -and guesses CSVs with rescaled log-space values. See -:ref:`model-components`'s Transformation Correction section for what the -``empirical``/``logit_norm``/``single`` models each do with it. - -**Scale** - -A single measurement (one mean, one uncertainty) from an independent -congression-rate experiment — not a per-genotype or per-row dataset. - -**Format** - -Passed directly as two floats via ``--transformation_lambda``, required -when ``--transformation_model`` is ``empirical`` or ``logit_norm``, and -forbidden (must be omitted) when it is ``single``: +.. list-table:: + :header-rows: 1 + :widths: 25 35 40 + + * - Constant + - ``O2_C12`` structures + - ``O2_C4`` structures + * - K\ :sub:`h_l` (C4: K\ :sub:`HL`) + - ΔΔG\ :sub:`H` − ΔΔG\ :sub:`L` + - ΔΔG\ :sub:`H` − ΔΔG\ :sub:`L` + * - K\ :sub:`h_o` (C4: K\ :sub:`op`) + - ΔΔG\ :sub:`H` − ΔΔG\ :sub:`HO` + - ΔΔG\ :sub:`H` − ΔΔG\ :sub:`HD` + * - K\ :sub:`h_e` + - (ΔΔG\ :sub:`H` − ΔΔG\ :sub:`HE2`)/2 + - fixed at 0 + * - K\ :sub:`l_o` + - ΔΔG\ :sub:`L` − ΔΔG\ :sub:`LO` + - fixed at 0 + * - K\ :sub:`l_e` (C4: K\ :sub:`E`) + - (ΔΔG\ :sub:`L` − ΔΔG\ :sub:`LE2`)/2 + - (ΔΔG\ :sub:`L` − ΔΔG\ :sub:`LE2`)/2 + * - K\ :sub:`u` (``U1`` only) + - ΔΔG\ :sub:`H` + - ΔΔG\ :sub:`H` + +Before 2026-10-11 the ``O2_C4`` K\ :sub:`E` row was ΔΔG\ :sub:`L` − +ΔΔG\ :sub:`LE2`, for a two-effector M\ :sup:`-2` constant. + +**Numbering and coverage.** The loader matches CSV labels to the library's +mutations and prints how many it covers. It stops with an error when the +CSV covers none of them, or when the CSV's wild-type residue at a library +position differs from the library's. Both usually mean the two number +residues differently. The error suggests the offset that lines them up, and +``--thermo_residue_offset N`` adds N to every CSV residue number, for +example 1 for a 0-based file. A library mutation absent from the CSV gets a +prior mean of 0, no structural prediction. When more than +``--thermo_max_missing_fraction`` of them (default 0.1) are absent, a +warning lists them. + +The ``PnnC`` HDF5 path applies the same residue offset. Every library +residue must be present in every structure, since there is no neutral +feature value for a missing one. When the file stores ``wt_aa``, the wt +residues are checked against the library too. + +The five ``--thermo_*`` flags are stored in the configuration as +``components.thermo_options``. A configuration without it reads the file as +before: kT, file numbering, model column names. + +A fit uses the CSV values as the prior means of each mutation's ΔΔG and +estimates one SD σ_s per structure for how far the true ΔΔG stray from +them, ΔΔG ~ Normal(CSV value, σ_s). The data see only the constants, so the +fit puts this prior on the constant shifts it implies: each mutation's +Δln K is multivariate normal around the projected CSV values, with the +covariance that per-structure scatter produces. σ_s has a LogNormal prior +whose median is 1 kT and whose 95% range is about 0.1 to 10 kT +(``theta_ddG_sigma_loc`` = 0 and ``theta_ddG_sigma_scale`` = 1.2 in the +priors file, in log kT). Every σ_s is identified by how the constants vary +together, in every model. The fit starts at the projected CSV values and +σ_s = 1 kT. Under MAP, σ_s settles near the scatter of the fitted constants +about the CSV. A σ_s that runs toward 0 means the data cannot resolve the +constants more finely than the CSV gives them, not that the CSV is exact. +This can happen when the data are weak. + +To hold σ_s at a known value instead of estimating it, set +``theta_ddG_sigma_fixed`` to that value in kT, for example +``--set_priors theta_ddG_sigma_fixed=1.69``. The value is in kT whatever +``--thermo_ddG_units`` says. Every structure gets the same value; there is +no per-structure hold. The fit then has no σ_s parameters, the LogNormal +prior is unused, and the extracted ``ddG_sigma`` table reports the held +value. The default 0 estimates σ_s. A held σ_s suits a simulation study +that compares a fit told the true noise with one that estimates it. + +The fit reports a per-structure ΔΔG for each mutation. In the models with +an unfolded state (``U1``) it follows from the constants exactly. In the +models without one (``U0``) every constant is a difference of structures, so +a shift shared by all structures changes nothing the data see. There the +reported ΔΔG is the most probable per-structure ΔΔG given the constants and +the prior, and its shared shift comes from the CSV, not from the data. +Compare U0 fits on their constants. To test whether the structural prior +helps, simulate with the simulate-config key ``ddG_truth_sd`` (see +:doc:`simulation`), which sets how far the simulated truth sits from the +same CSV, and fit with the unchanged CSV. + +Prior inputs +------------ + +``tfs-configure-model`` writes every prior at its component default into +``{out_prefix}_priors.csv``. Three flags change it at configure time, so the +file is never edited by hand. Growth priors are applied first, then +``--set_priors``. + +**--set_priors name=value ...** sets scalar priors. A name is a full row +name of the priors CSV, such as +``growth.sample_offset.sigma_fixed``, or a dotted suffix that matches +exactly one row, such as ``sigma_fixed``. An unknown or ambiguous name is an +error, and per-condition priors are set with ``--growth_priors`` instead. +Common uses: ``sigma_fixed=0.17`` holds the SD of the level tube offsets; +``theta_log_hill_n_hyper_scale_fixed=0.5`` holds the population SD of +``log(hill_n)``; for ``hill_relative``, ``theta_X_low_hyper_scale_fixed``, +``theta_X_delta_hyper_scale_fixed`` and +``theta_log_hill_K_hyper_scale_fixed`` hold the population SDs of its other +parameters; for the ``PddG`` and ``PnnC`` models, +``theta_ddG_sigma_fixed`` holds the per-structure ΔΔG scale σ_s (full name +``theta.theta_ddG_sigma_fixed``). A value of 0 leaves the SD learned. + +**--growth_priors table.csv** sets per-condition priors for ``linear`` +condition growth. The table has a ``condition_rep`` column, the condition +name as it appears in ``condition_pre`` or ``condition_sel``, and any of +``k_loc``, ``k_scale``, ``m_loc`` and ``m_scale``, in rates per minute. With +a ``replicate`` column each row applies to one replicate; without it, to +every replicate. Conditions and values the table leaves out keep the +defaults, and a condition the model does not have is an error. +``m_scale`` sets the scale for selective and non-selective conditions +alike. Use it where ``tfs-prefit-calibration`` does not run, such as a +growth-only model; the pre-fit overwrites ``k_loc`` and ``m_loc``. + +.. code-block:: text + + condition_rep,k_loc,k_scale,m_loc,m_scale + kanR+kan,0.015,0.002,-0.010,0.003 + kanR-kan,0.020,0.002,, + +**--growth_priors_wt_rates rates.csv** derives those priors for +``hill_relative`` from wt monoculture growth rates. The columns are +``condition_sel``, ``titrant_conc``, ``rate_mean``, ``rate_sd`` and +``num_replicates``, with one row at each gauge concentration for every +listed condition. In the X gauge wt grows at ``k`` at ``C_HI`` and at +``k + m`` at ``C_LO``, so ``k`` is the rate at ``C_HI`` and ``m`` is the rate at +``C_LO`` minus ``k``. Each SD is the standard error of the replicate mean, +floored at ``--growth_priors_sd_floor`` (default 0.002 per minute). List only +the conditions where the monoculture stands for the library's wt. These +values override a ``--growth_priors`` table for the conditions they list, +and they cannot be combined with a per-replicate table. .. code-block:: bash - tfs-configure-model binding.csv --growth_df growth.csv \ - --transformation_model empirical \ - --transformation_lambda 0.36 0.05 + tfs-configure-model --growth_df growth.csv --library_config library_config.yaml \ + --theta_model hill_relative \ + --growth_priors_wt_rates wt_rates.csv \ + --set_priors sigma_fixed=0.17 theta_log_hill_n_hyper_scale_fixed=0.5 diff --git a/docs/source/model.rst b/docs/source/model.rst new file mode 100644 index 00000000..f80dfa57 --- /dev/null +++ b/docs/source/model.rst @@ -0,0 +1,609 @@ +========= +The model +========= + +``tfscreen.tfmodel`` is a hierarchical Bayesian model written in JAX and +NumPyro. It infers each genotype's operator occupancy *θ* as a function of +titrant concentration from growth data, binding data or both. This page +describes the generative model and every component that can be swapped into +it. :doc:`fitting` describes how to fit it, and :doc:`model-inputs` describes +the data it reads. + +The growth equation +=================== + +Each genotype *g* grows in a tube at a rate set by its occupancy. With the +default components, linear condition growth and an instant transition between +the pre-growth and selection phases, the predicted abundance is + +.. math:: + + \ln \mathrm{cfu} = \ln \mathrm{cfu}_0 + + (k_\mathrm{pre} + dk_g + A_g\, m_\mathrm{pre}\, \theta_g)\, t_\mathrm{pre} + + (k_\mathrm{sel} + dk_g + A_g\, m_\mathrm{sel}\, \theta_g)\, t_\mathrm{sel} + + \delta_\mathrm{tube} + +*k* and *m* are the baseline growth rate and the occupancy slope of each +condition, shared by every genotype. *dk_g* is the genotype's pleiotropic +growth effect, independent of the transcription factor. *A_g* is its TF +activity. *θ_g* is its occupancy at the tube's titrant concentration. +*δ_tube* is an offset shared by every genotype in one tube. Growth rates are +per minute and times are in minutes. + +Each term comes from a component chosen at configuration time. The +``condition_growth`` component sets how *k*, *m* and *θ* combine, the +``growth_transition`` component sets how the two phases join, and so on down +the list below. With the ``mixture`` transformation each genotype's cells are +split into a clean class and several congressed classes, each grown through +the same equation, and the classes are mixed as + +.. math:: + + \ln \mathrm{cfu} = \ln \mathrm{cfu}_0 + + \log \sum_c w_c \exp(G_c) + \delta_\mathrm{tube} + +where *G_c* is the growth term of class *c* and *w_c* its weight. + +The predicted ``ln_cfu`` is observed through the growth likelihood. Binding +data, when supplied, observe *θ* directly. Pre-split and base-growth data, +when supplied, add their own observations of the starting abundance and of +reference growth rates. + +.. _model-components: + +Components +========== + +``tfs-configure-model`` selects one component per axis with a +``--_model`` flag and writes the choice into the ``components:`` block +of the configuration YAML. Every option, with its default, is in +:doc:`cli`. The defaults are the current recommendations for a joint +growth and binding model. + +.. list-table:: + :header-rows: 1 + :widths: 30 25 45 + + * - Flag + - Default + - Options + * - ``--condition_growth_model`` + - ``linear`` + - ``linear``, ``power``, ``saturation`` + * - ``--growth_transition_model`` + - ``instant`` + - ``instant``, ``memory``, ``baranyi``, ``baranyi_k``, ``baranyi_tau``, + ``two_pop`` + * - ``--ln_cfu0_model`` + - ``hierarchical`` + - ``hierarchical``, ``hierarchical_factored`` + * - ``--dk_geno_model`` + - ``hierarchical_geno`` + - ``hierarchical_geno``, ``fixed``, ``pinned`` + * - ``--activity_model`` + - ``fixed`` + - ``fixed``, ``hierarchical_geno``, ``horseshoe_geno``, + ``hierarchical_mut``, ``horseshoe_mut`` + * - ``--theta_model`` + - ``hill_geno`` + - ``hill_geno``, ``hill_relative``, ``hill_mut``, ``categorical_geno``, + ``thermo.*`` + * - ``--theta_rescale_model`` + - ``passthrough`` + - ``passthrough``, ``logit`` + * - ``--transformation_model`` + - ``single`` + - ``single``, ``mixture`` + * - ``--theta_growth_noise_model`` + - ``zero`` + - ``zero``, ``beta``, ``logit_normal`` + * - ``--theta_binding_noise_model`` + - ``zero`` + - ``zero``, ``beta``, ``genotype_offset`` + * - ``--growth_noise_model`` + - ``zero`` + - ``zero``, ``normal_kt`` + * - ``--sample_offset_model`` + - ``level`` + - ``level``, ``zero``, ``normal`` + * - ``--growth_likelihood`` + - ``counts`` + - ``counts``, ``lncfu`` + +Each component's priors are written to ``{out_prefix}_priors.csv`` with rows +named ``{group}.{component}.{field}``, such as +``growth.sample_offset.sigma_fixed``. Change a scalar prior with +``tfs-configure-model --set_priors name=value`` rather than by editing the +file. A name can be the full row name or any unique dotted suffix of it. + +Condition growth +---------------- + +The ``condition_growth`` component maps occupancy to a growth rate in each +condition. A condition is one marker and selection combination, such as +``kanR+kan`` or ``pheS+4CP``. + +``linear`` (default) + *g = k + dk_g + A·m·θ*. One *k* and one *m* per condition. Control + conditions, those without selection, get a tight prior on *m* set by + ``m_scale_minus``, default 0.001. Selection conditions get a looser one + set by ``m_scale_plus``, default 0.01. +``power`` + *g = k + dk_g + A·m·θ*\ :sup:`n`, with a per-condition exponent *n*. +``saturation`` + *g = min + dk_g + A·(max − min)·θ/(1 + θ)*, with per-condition minimum and + maximum rates. + +``power`` and ``saturation`` assume *θ* lies in [0, 1], so pair them with the +``passthrough`` rescale. ``--growth_shares_replicates`` gives every replicate +the same condition parameters. Without it each replicate has its own. + +The baseline *k* is only identified jointly with *dk_g*: adding a constant to +every *k* and subtracting it from every *dk_g* leaves the likelihood +unchanged. Wt's *dk_g* is fixed at 0, but that single anchor is diluted by the +size of the library, and the whole system can slide. The fix is a +per-condition prior on the baselines. Every prior location and scale of these +components accepts a scalar, applied to every condition, or one value per +condition. ``tfs-prefit-calibration`` writes per-condition *k* and *m* +locations into the priors file, as indexed rows labeled by ``condition_rep``, +with a tight floored scale. Its ``--pin_m`` option clamps *m* to the +calibrated value outright, because a soft prior on *m* can be overridden by the +growth likelihood. *k* keeps a soft prior, since it carries real +tube-to-tube variation. For a growth-only model, where the pre-fit does not +run, set the per-condition priors at configuration with ``--growth_priors`` +or ``--growth_priors_wt_rates``. See :doc:`fitting`. + +Growth transition +----------------- + +The ``growth_transition`` component joins the pre-growth and selection +phases. + +``instant`` (default) + Each genotype switches to its selection rate at the start of selection. +``memory`` + Cells keep the pre-growth rate for a lag *τ = τ0 + k1/(θ + k2)*, then grow + at the selection rate. The lag depends on occupancy. +``baranyi`` + The rate moves from the pre-growth to the selection rate along a sigmoid in + time, with a per-condition midpoint *τ* and steepness. +``baranyi_k`` + As ``baranyi``, with the steepness reduced as the rate change grows. +``baranyi_tau`` + As ``baranyi``, with the midpoint delayed as the rate change grows. +``two_pop`` + Cells leave the pre-growth state at a constant per-cell rate, so the + culture is a mix of the two growth modes during the transition. + +Starting abundance +------------------ + +The ``ln_cfu0`` component sets each genotype's starting abundance. + +``hierarchical`` (default, recommended) + One starting abundance per replicate, pre-growth condition and genotype. + Each library class has its own pooled prior, and spiked genotypes and wt + have their own priors. +``hierarchical_factored`` + One genotype baseline per replicate, shared by every pre-growth + condition, plus one offset per tube. This is valid only when the + pre-growth conditions are split from one culture. In the screen, the kanR + and pheS libraries are transformed and grown separately, so each + genotype's starting abundance differs between them. On simulations of that + design the factored model pushed the difference into a confident + per-genotype error in *θ*, with 95% coverage of 0.09 above 1000 reads + against 0.80 for ``hierarchical``. ``tfs-configure-model`` refuses + ``hierarchical_factored`` when the growth table's ``library`` column shows + a replicate's pre-growth conditions come from different libraries. + +Pleiotropic growth effect +------------------------- + +The ``dk_geno`` component sets *dk_g*, each genotype's effect on growth that +does not go through the transcription factor. + +``hierarchical_geno`` (default) + A pooled, left-skewed prior: a shift minus a log-normal draw. Most + mutations are near neutral, a few help, and a long tail is deleterious. + Wt is fixed at 0. +``fixed`` + *dk_g* = 0 for every genotype. +``pinned`` + *dk_g* is fixed to supplied values for a subset of genotypes and 0 for the + rest. The values come from a CSV given as the orchestrator's + ``dk_geno_pins_file`` setting. ``tfs-configure-model`` has no flag for that + file, so this component is only reachable by building a + ``ModelOrchestrator`` in Python. + +Activity +-------- + +The ``activity`` component sets *A_g*, a factor on each genotype's occupancy +term. + +``fixed`` (default) + *A* = 1 for every genotype. +``hierarchical_geno`` + One activity per genotype from a pooled log-normal prior. +``horseshoe_geno`` + One activity per genotype under a sparse horseshoe prior. +``hierarchical_mut`` + Log activity is a sum of per-mutation effects, plus pairwise epistasis with + ``--epistasis``, under hierarchical normal priors. +``horseshoe_mut`` + As ``hierarchical_mut``, under regularized horseshoe priors. + +A learned activity is refused with ``hill_relative`` and with the +``mixture`` transformation's ``homodimer`` and ``heterodimer`` rules. + +Occupancy +--------- + +The ``theta`` component sets *θ* as a function of titrant concentration *c*. + +``hill_geno`` (default) + A Hill curve per genotype, + *θ = θ_low + (θ_high − θ_low)·sigmoid(n·(ln c − ln K))*. The parameters + are ``theta_low``, ``theta_high``, ``log_hill_K`` and ``hill_n``. + ``log_hill_K`` is the natural log of the midpoint concentration. Each + parameter is drawn from a population distribution whose location and + scale are learned. +``hill_relative`` + The Hill curve for a growth-only fit on a wt-relative scale *X* in place + of *θ*. See below. +``hill_mut`` + A Hill curve whose transformed parameters are wt's values plus a sum of + per-mutation effects, plus pairwise epistasis under a regularized + horseshoe prior when ``--epistasis`` is set. It can predict genotypes that + were never measured. +``categorical_geno`` + An independent *θ* at every titrant concentration, with no curve. +``thermo.*`` + Thermodynamic partition-function models. See + :ref:`thermo-naming` below. The ``PddG`` and ``PnnC`` variants need + ``--thermo_data``. + +The population SD of each Hill parameter is learned by default. On real +data a learned SD can run away and stop shrinking poorly measured genotypes: +a MAP on the development screen drove the SD of log(hill_n) to about 19, and +*n* near 0.01 hid each genotype's response below the lowest nonzero +concentration. ``theta_log_hill_n_hyper_scale_fixed`` holds that SD at a +given value instead. A value of 0, the default, keeps it learned. + +.. code-block:: bash + + tfs-configure-model ... --set_priors theta_log_hill_n_hyper_scale_fixed=0.5 + +The relative-X fit +~~~~~~~~~~~~~~~~~~ + +Growth alone fixes an occupancy-like variable only up to an affine map, +because *k* and *m* absorb any shift and scale. ``hill_relative`` therefore +fits a wt-relative variable *X* with the Hill curve's form and real-valued +baselines, ``X_low`` and ``X_delta``. Wt is gauged to *X* = 1 at a low +concentration and *X* = 0 at a high one. ``--theta_gauge_conc c_lo c_hi`` +sets those two concentrations, by default the lowest and highest in the +growth table. The resolved values are written to the configuration. + +``hill_relative`` is refused with binding data, with an activity other than +``fixed``, a rescale other than ``passthrough``, condition growth other than +``linear``, theta growth noise other than ``zero`` and, under the ``mixture`` +transformation, any congression *θ* rule but ``max``. Downstream tools know +the output is *X*: ``tfs-predict-theta`` adds a ``theta_scale`` column set +to ``X``, and epistasis on *X* is defined only on the additive scale. + +Besides ``theta_log_hill_n_hyper_scale_fixed``, ``hill_relative`` takes +``theta_X_low_hyper_scale_fixed``, ``theta_X_delta_hyper_scale_fixed`` and +``theta_log_hill_K_hyper_scale_fixed``. Learned, these ran to 3.6, 4.3 and 11 +on the development screen, against about 0.5, 0.5 and 0.7 among +well-measured genotypes, so the fit shrank nothing. + +Occupancy rescale +----------------- + +The ``theta_rescale`` component transforms *θ* before it enters condition +growth. + +``passthrough`` (default) + *θ* is used as is. +``logit`` + *θ* is replaced by log(*θ*/(1 − *θ*)), which expands the range near 0 + and 1. + +Congression +----------- + +The ``transformation`` component models congression, more than one plasmid +entering a cell during transformation. + +``single`` (default) + Every cell carries one plasmid. +``mixture`` + Each genotype's cells are a mixture of clean cells and congressed cells + that also carry plasmids drawn from the bulk library. Transformants carry + a zero-truncated Poisson(λ) number of plasmids. A genotype's congressed + fraction is its ``bulk_fraction`` from the library table times + 1 − P(M = 1). Each class has its own cell-level *θ*, activity and + *dk_g*, grows through the full equation, and the classes are mixed at the + level of ``exp(ln_cfu)``. + +``mixture`` needs ``--transformation_lambda MEAN SD``, the measured +congression rate in linear space, which becomes a log-normal prior on λ. It +is refused with ``single``. Three settings fix how a congressed cell combines +its plasmids: + +``--congression_theta_rule`` + ``homodimer`` (default) combines the plasmids' occupancies through the + partition function of a homodimer with equal shares, ``heterodimer`` + through that of a heterodimer, and ``max`` takes the highest *θ*. The + ``homodimer`` and ``heterodimer`` rules require ``activity`` ``fixed``. +``--congression_dk_rule`` + ``dilution`` (default) takes the share-weighted mean of *dk_g*, ``min`` + the worst variant's, and ``softmin`` interpolates between them with + ``--congression_dk_alpha``. + +The congressed term is averaged over fixed co-resident sets drawn once per +genotype from the bulk library. ``congression_sets`` sets how many sets of +each co-resident count, default ``[12, 3, 1]`` for one, two and three +co-residents, and ``congression_seed`` (default 0) makes the draw +reproducible. ``tfs-configure-model`` writes both into the configuration's +``components:`` block and has no flag for them. + +The transformations ``empirical`` and ``logit_norm`` were removed. A +configuration that names them is refused with a message. + +Noise +----- + +``--theta_growth_noise_model`` adds noise to the *θ* that enters growth, and +``--theta_binding_noise_model`` to the *θ* that binding observes. + +``zero`` (default) + No noise. +``beta`` + *θ* is drawn from a beta distribution around the predicted value. +``logit_normal`` + Normal noise with a learned SD is added to logit(*θ*). Growth only. +``genotype_offset`` + Binding only. Each binding genotype's binding *θ* is its curve plus one + offset of its own, the same at every titrant concentration: + *δ_g = τ z_g*, *z_g ~ Normal(0, 1)*, *τ ~ HalfNormal(0.2)* (learned, or + held with ``offset_scale_fixed``). It absorbs an in vivo/in vitro + difference or a curve's day-normalization error for one genotype + without moving the growth side; *τ* says how far binding and growth + disagree across genotypes. + +``--growth_noise_model`` adds noise to growth itself. + +``zero`` (default) + No extra noise. +``normal_kt`` + A learned global SD on accumulated growth, added in quadrature to + ``ln_cfu_std``. It works only with the ``lncfu`` likelihood. + +Tube offsets +------------ + +The ``sample_offset`` component adds *δ_tube*, a shift shared by every +genotype in one tube. A tube is one sequenced sample: one replicate, time, +condition and titrant concentration. Anything that moves every genotype in a +tube together lands here, such as PCR and genotype-calling efficiency or an +error in the tube's total cell count. + +``level`` (default) + One ``ln_cfu`` offset per tube with a learned SD. ``sigma_fixed`` > 0 + holds the SD at that value. Learned, the SD grew from its 0.2 prior scale + to 0.53 on the first real-data fit, and the offsets took over the + population's growth from *k* and *m*. Holding it near the per-tube OD600 + scatter, about 0.17, keeps the tube totals binding: + ``--set_priors sigma_fixed=0.17``. +``zero`` + No offset. +``normal`` + One growth-rate offset per tube, multiplied by the tube's elapsed time. + +Observations +============ + +Growth likelihood +----------------- + +``--growth_likelihood`` sets how growth data are observed. + +``counts`` (default) + Reads are observed directly as negative binomial, + *reads ~ NegBin(μ, var = μ(1 + φ) + μ²/r)*, where + *log μ = ln(sample reads) + ln_cfu_pred − ln(sample cfu)*. That is the + predicted frequency times the tube's total reads. φ and 1/r are learned + with log-normal priors. No pseudocount is used. It needs a ``counts`` + column, each tube's total reads and each tube's total cells, which + ``tfs-process-counts`` writes. It refuses a growth noise other than + ``zero``. Pair it with ``level`` tube offsets, since an error in a tube's + total otherwise moves every genotype in it. On simulations it cut *θ* + RMSE by 35-60% against ``lncfu`` at similar coverage. +``lncfu`` + Student-t on ``ln_cfu`` with the table's ``ln_cfu_std``. + +``ModelOrchestrator`` itself still defaults to ``lncfu`` and no tube +offset, so that configurations written before the count likelihood existed +read back unchanged. ``tfs-configure-model`` writes the new defaults +explicitly. + +Other observations +------------------ + +Binding data, from ``--binding_df``, observe *θ* with a normal likelihood. +Each binding observation counts once: ``--binding_weight`` scales the +binding likelihood and defaults to 1. Until 2026-10-16 the default was the +ratio of growth rows to binding rows, which made the binding data weigh as +much as the whole growth dataset. A model without binding data has no +binding sites at all. + +Pre-split data, from ``--presplit_df``, observe ``ln_cfu`` at the start of +pre-growth and constrain the starting abundance. Base-growth data, from +``--base_growth_df``, observe reference-condition growth rates as +*rate ~ Normal(k_ref + dk_g, rate_std)* and anchor *dk_g*. See +:doc:`model-inputs` for both. + +.. _model-naming: + +Naming conventions +================== + +Components that infer one value per genotype carry a ``_geno`` suffix, such +as ``hill_geno``. Components that build each genotype from per-mutation +effects carry ``_mut``, such as ``hill_mut``. Components with no natural +per-mutation form, such as ``fixed``, have no suffix. + +.. _thermo-naming: + +Thermodynamic occupancy models +------------------------------ + +Occupancy models built from an explicit partition function are named +``thermo.{MODEL}.{PRIOR}``. *MODEL* has four fields and a letter: + +.. list-table:: + :header-rows: 1 + :widths: 10 60 + + * - Field + - Meaning + * - ``O`` + - Oligomeric state; ``O2`` is a homodimer + * - ``C`` + - Number of conformational states + * - ``K`` + - Number of independent equilibrium constants + * - ``U`` + - ``U0`` has folded states only; ``U1`` adds a folding equilibrium + +The trailing letter tells apart models with the same counts and implies no +order. The implemented models are: + +* ``O2_C12_K5_U0_a`` (M3): the standard MWC lac repressor homodimer with + five constants. +* ``O2_C12_K5_U1_a``: the same with a folding equilibrium. +* ``O2_C4_K3_U0_a`` (M2): M3 restricted to three constants. +* ``O2_C4_K3_U1_a``: the same with a folding equilibrium. + +The C4/K3 labels are historical. They named an earlier partition function +for M2, which this one replaced on 2026-10-11, and no longer count M2's +species. The registry keys keep them. + +**The MWC dimer.** The dimer is in a high operator affinity conformation H or +a low affinity conformation L. K\ :sub:`h_l` = [L]/[H]. Conformation X binds +operator with K\ :sub:`x_o` (M\ :sup:`-1`) and has two identical, +independent effector sites, each with the microscopic (per-site) constant +K\ :sub:`x_e` (M\ :sup:`-1`). The effector binding polynomial of +conformation X is therefore (1 + K\ :sub:`x_e`\ ·e)\ :sup:`2` = +1 + 2K\ :sub:`x_e`\ ·e + K\ :sub:`x_e`\ :sup:`2`\ ·e\ :sup:`2`. The 2 +counts the two ways to fill one site. Within a conformation the two sites do +not interact, so the second macroscopic constant is one quarter of the first. +Cooperativity comes only from the conformational equilibrium, when +K\ :sub:`l_e` > K\ :sub:`h_e` and K\ :sub:`h_l` > 1. The ``U1`` models add an +unfolded state U, K\ :sub:`u` = [U]/[H], that binds nothing. + +Until 2026-10-11 M3 used the polynomial of Sochor (2014), +1 + K·e + K\ :sup:`2`\ ·e\ :sup:`2`, which lacks the factor of 2. That sets +the ratio of the two macroscopic constants to 1 instead of 1/4, a fixed +fourfold cooperativity inside each conformation that no constant can remove. +M2 used a third form, one-step binding of two effectors to L with an +M\ :sup:`-2` constant. + +M3 fits all five constants: K\ :sub:`h_l`, K\ :sub:`h_o`, +K\ :sub:`h_e`, K\ :sub:`l_o` and K\ :sub:`l_e`. M2 is M3 with +K\ :sub:`h_e` = 0 (H does not bind effector) and K\ :sub:`l_o` = 0 (L does +not bind operator). Its three constants keep their older names: +``ln_K_op`` is ln K\ :sub:`h_o`, ``ln_K_HL`` is ln K\ :sub:`h_l` and +``ln_K_E`` is ln K\ :sub:`l_e`, a per-site constant in M\ :sup:`-1`. Both +families compute θ with the same function, so M2 is nested in M3 exactly. + +TF is in large excess over operator: ``theta_tf_total_M`` defaults to +1e-5 M of monomer, about 400 times a typical operator concentration. The +models therefore leave operator-bound TF out of the TF mass balance, and the +operator concentration does not enter them. With r = tf_total/2 the dimer +concentration and F(e) the polynomial over all states not bound to operator, +the free apo H dimer is h = r/F(e), the operator sees +W = h·(K\ :sub:`h_o`\ ·P\ :sub:`H` + K\ :sub:`h_l`\ ·K\ :sub:`l_o`\ ·P\ :sub:`L`), +and θ = W/(1 + W). Free effector e comes from the effector mass balance, a +cubic in e, so effector bound to TF is subtracted from the total. At 1e-5 M +TF this matters below about 10 µM effector. The equations are in +``theta/thermo/mwc.py``. + +**Default wild type.** ``theta_tf_total_M`` is 1e-5 M monomer in both +families, the PI's estimate of the in vivo TF level. The M3 defaults aim +for strong repression and near-complete induction at that level. M3 takes +K\ :sub:`h_l` = 6.3, K\ :sub:`h_o` = 4.2×10\ :sup:`8` M\ :sup:`-1` and +K\ :sub:`l_o` = 0.1 M\ :sup:`-1` from Sochor (2014). At saturating IPTG only +the doubly liganded HE2 and LE2 remain, so with +ρ = K\ :sub:`l_e`/K\ :sub:`h_e` the operator sees + + W\ :sub:`sat` = r·(K\ :sub:`h_o` + K\ :sub:`h_l`\ ·K\ :sub:`l_o`\ ·ρ\ :sup:`2`) / (1 + K\ :sub:`h_l`\ ·ρ\ :sup:`2`) + +and θ at saturation depends on ρ alone. We set θ(saturating) = 0.05, +W\ :sub:`sat` = 1/19, and solved for ρ: + + ρ\ :sup:`2` = (r·K\ :sub:`h_o` − W\ :sub:`sat`) / (K\ :sub:`h_l`\ ·(W\ :sub:`sat` − r·K\ :sub:`l_o`)) + +which gives ρ ≈ 79.6. This departs from Sochor's fitted ρ ≈ 13.6. With +Sochor's ρ the H conformation that remains with two effectors bound still +occupies the operator at 1e-5 M TF, and θ falls only to 0.64. We then +scaled K\ :sub:`h_e` and K\ :sub:`l_e` by one common factor, keeping ρ, +so that the midpoint of the default wild-type θ(IPTG) curve is 80 µM IPTG. +The midpoint is where θ is halfway between θ(0) and θ at saturating IPTG, +with effector depletion at 1e-5 M TF. That gives K\ :sub:`h_e` ≈ +3.7×10\ :sup:`3` and K\ :sub:`l_e` ≈ 3.0×10\ :sup:`5` M\ :sup:`-1`, 0.39 times +Sochor's K\ :sub:`l_e`. M2 takes the same K\ :sub:`h_l` and K\ :sub:`h_o` +with K\ :sub:`l_e` = 2.5×10\ :sup:`5` M\ :sup:`-1`, also calibrated to an +80 µM midpoint. Both start at θ(0) = 0.997. M3 falls to 0.05 at saturating +IPTG and M2 to 0. The M3 curve has a Hill-like n of 1.7 (EC10 27 µM, EC90 +340 µM) and the M2 curve 2.1 (EC10 29 µM, EC90 230 µM). + +*PRIOR* sets how the equilibrium constants are parameterized: + +.. list-table:: + :header-rows: 1 + :widths: 10 60 + + * - Name + - Description + * - ``PK`` + - An independent normal prior on each log *K*, built from per-mutation + effects. + * - ``PddG`` + - Priors centered on supplied ΔΔG estimates, a CSV given with + ``--thermo_data`` with one column per structure. Values are in kT + unless ``--thermo_ddG_units`` says otherwise; positive ΔΔG + destabilizes that structure. See :doc:`model-inputs`. + * - ``PnnC`` + - A neural network that predicts per-conformation ΔΔG from structural + features, an HDF5 file given with ``--thermo_data``. See + :doc:`ligandmpnn-features`. + +In the ``U1`` models, ``PddG`` and ``PnnC`` take each mutation's effect on +unfolding from the same ΔΔG values. A mutation shifts the free energy of each +folded structure and leaves the unfolded state U unchanged. Because +K\ :sub:`U` = [U]/[H], a mutation changes ln K\ :sub:`U` by its ΔΔG for H. A +mutation that raises every folded structure by the same ΔΔG leaves the +folded-state constants alone and raises K\ :sub:`U` by ΔΔG, a pure stability +effect. The fit reports this shift per mutation as ``d_ln_K_U`` and per pair +as ``epi_ln_K_U``, or ``d_ln_K_u`` and ``epi_ln_K_u`` for +``O2_C12_K5_U1_a``. ``PK`` uses the same names for its free per-mutation +unfolding shift. Before 2026-10-10 these models held K\ :sub:`U` at its wt +value for every genotype, which moved U together with H and left every +mutation's stability unchanged. + +Full names look like ``thermo.O2_C4_K3_U0_a.PK`` or +``thermo.O2_C12_K5_U1_a.PnnC``. + +All the thermodynamic models read ``titrant_conc`` in mM, the units of the +growth and binding tables, and multiply it by the prior +``theta_conc_unit_scale`` (default 1e-3) so that the equations run in M. +Every operator and effector constant is in M\ :sup:`-1`, and +``theta_tf_total_M`` is in M of monomer. Priors files and checkpoints for +these models written before 2026-10-11 hold constants with the older +meanings. A priors file still reads, its ``theta_op_total_M`` row is +ignored, and its old values would be taken with the new meanings, so +regenerate it with ``tfs-configure-model``. Old checkpoints are not +supported. diff --git a/docs/source/pipeline.rst b/docs/source/pipeline.rst new file mode 100644 index 00000000..acfaf9ff --- /dev/null +++ b/docs/source/pipeline.rst @@ -0,0 +1,201 @@ +======== +Pipeline +======== + +This page lists every step from sequencing reads to a posterior, the command +for each and the file each one hands to the next. The pages linked from each +step explain the choices. :doc:`cli` lists every flag. + +To analyze a screen, follow :doc:`screen-analysis`: it runs steps 3-6 as one +command, ``tfs-fit-two-stage``, with the settings that gave calibrated +intervals. This page describes the commands that one chains. + +Overview +-------- + +.. code-block:: text + + reads (one fastq pair per tube) + │ tfs-process-fastq library_config.yaml + ▼ + counts/counts_.csv + │ tfs-process-counts tube table, OD600 table, OD600 calibration + ▼ (tfs-calibrate-od600 makes the calibration) + growth.csv + │ tfs-configure-model library_config.yaml, optional binding.csv, + ▼ growth priors, held prior values + tfs_configure_config.yaml + _priors.csv + _guesses.csv + _library.csv + │ tfs-prefit-calibration joint models only: pins growth k and m + │ tfs-fit-model MAP or SVI + ▼ + tfs_fit_model_checkpoint.pkl + _params.npz + │ tfs-sample-posterior Laplace (MAP) or guide draws (SVI) + ▼ + tfs_posterior.h5 + │ tfs-extract-params, tfs-predict-theta, tfs-predict-growth, + │ tfs-predict-epistasis + ▼ + per-parameter and per-prediction tables with q columns + +Every command prints its provenance when it starts: the tfscreen version, the +git commit with a flag for uncommitted changes, and the command line. A +command that takes ``--out_prefix`` also writes it to +``{out_prefix}_provenance.json``. The model configuration and every fit +checkpoint carry the same record. + +1. Count reads +-------------- + +Run ``tfs-process-fastq`` once per tube. It calls each read pair against the +library described by the library YAML and writes ``counts_.csv`` +with one row per library genotype, plus an ``__unknown__`` row for reads that +matched none. + +.. code-block:: bash + + tfs-process-fastq library_config.yaml tube01_R1.fastq.gz tube01_R2.fastq.gz --out_dir counts + +The same library YAML goes to ``tfs-configure-model`` later. Keep one per +experiment. See :doc:`process-raw`. + +2. Tube totals and the growth table +----------------------------------- + +The model needs the number of cells in each tube. The lab measures OD600, so +the totals come from an OD600 calibration made once per instrument and strain +with ``tfs-calibrate-od600``. ``tfs-process-counts`` reads the tube table, the +OD600 readings, the calibration and the culture volume, computes each tube's +total and its uncertainty, and turns the counts into the growth table. All +libraries go through in one call; the tube table's ``library`` column keeps +them apart. + +.. code-block:: bash + + tfs-calibrate-od600 replicates.csv plate_counts.csv --out_prefix od600_calibration + tfs-process-counts tube_table.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix growth + +Check the tube labels with ``tfs-check-tubes``, which flags tubes whose +genotype composition is out of order with their IPTG label, and record +swaps, relabels and exclusions in a corrections file that +``tfs-correct-tubes`` applies to the tube table before +``tfs-process-counts`` (see :doc:`screen-analysis`). Pooling resequenced +files of one tube belongs in the lab's own script. ``examples/process_raw/`` has a synthetic tube table, OD600 +table and count files to run these commands on. See :doc:`process-raw`. + +3. Configure the model +---------------------- + +``tfs-configure-model`` reads the data, checks it against the library and +writes the model configuration. The defaults are the current recommendations: +the count likelihood, one level offset per tube and a separate starting +abundance per library. Priors that need values other than the defaults are +set here, never by editing the priors file. + +A joint model, with binding data: + +.. code-block:: bash + + tfs-configure-model --growth_df growth.csv --binding_df binding.csv \ + --library_config library_config.yaml + +A growth-only model on the wt-relative X scale, with per-condition growth +priors and held population SDs: + +.. code-block:: bash + + tfs-configure-model --growth_df growth.csv \ + --library_config library_config.yaml \ + --theta_model hill_relative \ + --growth_shares_replicates \ + --growth_priors growth_priors.csv \ + --set_priors sigma_fixed=0.17 theta_log_hill_n_hyper_scale_fixed=0.5 + +See :doc:`model-inputs` for the inputs and :doc:`model` for the components. + +4. Fit +------ + +For a joint model, ``tfs-prefit-calibration`` first fits a small model to the +binding genotypes and pins each condition's growth baseline and slope in the +priors file. It refuses growth-only models; their growth priors come from step +3. Then ``tfs-fit-model`` runs MAP or SVI until the convergence monitor stops +it. + +.. code-block:: bash + + tfs-prefit-calibration tfs_configure_config.yaml --seed 1 + tfs-fit-model tfs_configure_config.yaml --seed 1 --analysis_method map + +On a full library, MAP is the route that has worked; the SVI guides slid +along the growth slope and X ridge of the growth-only model. A MAP with level +tube offsets runs in three stages by default, so the offsets cannot settle in +a mode where they carry the population's growth. See :doc:`fitting`. + +5. Posterior +------------ + +``tfs-sample-posterior`` turns a checkpoint into posterior draws. For a MAP +checkpoint it builds a Laplace approximation. ``--laplace auto`` (the default) +uses the full Laplace up to 20,000 parameters and the arrowhead Laplace above, +which gives the same Gaussian computed one genotype at a time. Directions it +had to hold at the MAP are written to ``{out_prefix}_held_directions.csv``, +because they have no interval. + +.. code-block:: bash + + tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --skip_growth_observations + +6. Results +---------- + +The posterior file feeds the extraction and prediction commands. Each writes +bare quantile columns (``q0.025``, ``q0.5``, ``q0.975`` and so on). + +.. code-block:: bash + + tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 + tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + tfs-predict-epistasis tfs_configure_config.yaml tfs_posterior.h5 + +A growth-only relative fit predicts X rather than an occupancy, so its +epistasis is only defined on the additive scale: pass +``tfs-predict-epistasis --scale add``. ``tfs-cat-response`` classifies the +response curves, ``tfs-extract-epistasis`` +computes epistasis from any long-form table, and ``tfs-compare-runs`` +measures agreement between runs. See :doc:`downstream`. + +7. Check the fit +---------------- + +``tfs-summarize-fit`` gathers a run's diagnostics from its directory: the +loss history, observed against predicted growth, and, for a fit with tube +offsets, the offsets in prior SDs with a flag for offsets that trend with +titrant or time. A flagged fit has offsets carrying growth the model leaves +out, so read its curves with care. On a simulation it also compares every +parameter and prediction with the truth. See :doc:`summarize-fit`. + +.. code-block:: bash + + tfs-summarize-fit . + +Simulations +----------- + +``tfs-simulate`` writes a simulated experiment and its ground truth from one +YAML file. It writes the experiment in the raw formats a lab's data come in, +one counts file per tube, the tube table and, with an ``od600`` block, the +OD600 table and calibration, so a simulation enters the pipeline at step 2 +through the same ``tfs-process-counts`` command as real data; the command is +printed. (Step 1 is replaced by the simulator's own sequencing model.) Its +growth table, written directly, is the same table and can enter at step 3. +A simulation can also follow a real experiment's tube table (the ``design`` +key), so the same chain runs on a simulated copy of the experiment with only +the data paths changed. A full-size simulation of a real screen is large, +tens of GB of memory and over an hour for one seed. ``tfs-setup-sim-grid`` +and ``tfs-setup-grid`` lay out grids of simulations and fits. See +:doc:`simulation` and :doc:`grid`. diff --git a/docs/source/process-raw.rst b/docs/source/process-raw.rst index 768dfb19..e9225e61 100644 --- a/docs/source/process-raw.rst +++ b/docs/source/process-raw.rst @@ -1,205 +1,476 @@ =================== -Processing Raw Data +Processing raw data =================== -A typical TF screen involves growing bacteria transformed with a plasmid-encoded -library of TF variants under one or more selection conditions (e.g. antibiotic resistance -driven by a TF-regulated promoter). At each time-point an aliquot is -plated to measure total colony-forming units (CFU) and deep-sequenced so -that the absolute abundance of every genotype can be determined. The raw -inputs to the pipeline are paired-end FASTQ files (one pair per sample) and -a sample metadata table (``sample_df``) that links each sequenced tube to its -biological context and its measured CFU count. The pipeline converts read -counts into per-genotype log-CFU estimates (``ln_cfu``) that feed the -hierarchical Bayesian growth model. - -There are three primary scripts for processing raw data: - -1. ``tfs-process-fastq``: Analyses paired-end FASTQ files to count the - occurrence of each genotype. -2. ``tfs-process-counts``: Aggregates counts across multiple samples and - computes adjusted log-counts (``ln_cfu``) for downstream modelling. -3. ``tfs-process-presplit``: Like ``tfs-process-counts``, but for the - pre-split time-point (before the library is divided into separate - selection conditions). The output anchors the initial genotype - abundances used by the growth model. - -Configuration File (run_config.yaml) -------------------------------------- - -``tfs-process-fastq`` requires a ``run_config.yaml`` file describing the -library of expected sequences. You can view or download an -:download:`example run_config.yaml <../../examples/process_raw/library_config.yaml>` file. Expected fields: - -* ``reading_frame``: Amino acid reading frame offset (0, 1, or 2). -* ``first_amplicon_residue``: Amino acid residue number for the first in-frame - residue. -* ``wt_seq``: The wildtype nucleic acid sequence. -* ``degen_sites``: Degenerate codon pattern the same length as ``wt_seq`` - (e.g. ``NNT``, ``NNK``, or ``.`` for wildtype). -* ``tiles``: Contiguous blocks of library components cloned together. - ``.`` indicates wildtype; each unique character besides ``.`` defines a - tile (blocks must be contiguous). -* ``expected_5p`` / ``expected_3p``: Flanking sequences immediately upstream - and downstream of the amplicon. -* ``tile_combos``: List of strings such as ``single-x`` or ``double-x-y``, - where ``x`` and ``y`` match characters in ``tiles``. ``single-x`` - specifies all single-mutation variants in tile ``x``; ``double-x-y`` - specifies all pairwise combinations between tiles ``x`` and ``y``. -* ``spiked_seqs``: Specific nucleic acid sequences (not part of the combinatorial - library) that should be identified as controls. - -tfs-process-fastq ------------------ - -Reads paired-end FASTQ files and counts the protein genotype observed in each -read pair. Each read is matched against the predefined library after quality -filtering and flanking-sequence detection. +A screen grows bacteria carrying a plasmid library of transcription factor +variants under one or more selection conditions. Each time-point is its own +tube. Every tube is deep-sequenced, and its total cell count is estimated, +usually from OD600. Together these give the absolute abundance of every +genotype in every tube. + +The raw-data half of the pipeline has four steps: + +1. ``tfs-process-fastq`` calls the genotype of every read pair in one + tube's FASTQ files and writes a count file for that tube. +2. ``tfs-calibrate-od600`` fits the lab's OD600-to-CFU calibration. This is + done once per reader, plate type, volume and strain, not once per + experiment. +3. You write a tube table that names every sequenced tube and gives its + design: library, replicate, conditions, titrant and times. When the + data show that a tube's label is wrong, ``tfs-correct-tubes`` applies a + documented correction and writes a corrected copy. +4. ``tfs-process-counts`` combines the count files, the tube table and each + tube's OD600 into the growth table that ``tfs-configure-model`` reads. + With ``--presplit`` it writes the presplit table instead. + +Every flag and default is listed in :doc:`cli`. The model's side of these +tables is described in :doc:`model-inputs`. + +To analyze a screen, :doc:`screen-analysis` runs steps 3 and 4 in order, +with ``tfs-check-tubes`` to find mislabeled tubes, and then the fit. + +One tube per time-point +----------------------- + +The protocol: + +1. Transform the library. +2. Grow the transformed culture to a set OD. +3. Take the ``presplit`` sample and read its OD600. +4. Dilute the culture by the same factor into every tube, one per + condition, titrant concentration and planned time-point. Every tube of a + replicate therefore starts from the same known total: the presplit + OD600 estimate divided by the dilution factor. For a titration this is a + grid. Rows are time-points and columns are IPTG concentrations. +5. At each time-point, pull every tube in that row. Read each pulled tube's + OD600, then spin the tubes down and freeze them at once. The spread in + stop time within a row is seconds, which is negligible on the time scale + of growth. +6. After the experiment, process all samples together. Thaw and extract DNA + in batches of about 20, so three batches for 60 samples. Then run one + PCR block for every sample, then sequence. Record each sample's + extraction batch. For the current dataset it was not recorded; batches + generally, but not reliably, followed processing order. + +Pre-growth and selection both happen in the tubes, after the split. OD600 +is read on three bioreplicates of the experiment and two are sequenced. + +Each tube's total CFU comes from its OD600 through an OD-to-CFU +calibration. The calibration depends on the plate reader, plate, volume and +strain, so each lab makes its own with ``tfs-calibrate-od600``. A +calibration gives CFU per mL. The pipeline works in cells in the whole +tube, so ``tfs-process-counts`` multiplies by the tube's culture volume: +``ln_cfu`` is ``ln(cfu/mL x tube volume)``. Under the strongest selection a +tube may never reach the reader's detection threshold. Its OD600 then bounds +its total from above rather than measuring it. + +Why tubes and not aliquots: pulling aliquots from a single tube over time +meant stopping the shaker, pipetting and returning the tube across many +IPTG conditions. That disturbed the growth trajectories more than the +tube-to-tube differences of this design do. + +Consequences for the model: + +- A sequenced sample *is* a tube, and the time-points of one condition come + from different tubes. Anything that affects a whole tube is independent + from one time-point to the next, not carried along a trajectory. That + includes its growth environment over ``t_pre + t_sel``, its OD600 reading + and its PCR. The model's per-tube offset (``sample_offset``) is indexed + the same way, one offset per replicate x time x condition x titrant. The + default, ``level``, is a constant shift of the tube's ``ln_cfu`` shared by + every genotype in it, with a learned SD. It absorbs the tube's composition + offset and any error in its supplied total. ``normal`` is the older + alternative, a growth-rate offset scaled by ``t_pre + t_sel``, and + ``zero`` adds no offset. +- A growth difference shared by every cell in a tube changes every + genotype's abundance by the same factor, so it does not change the + genotype *frequencies* the reads measure. It enters ``ln_cfu`` only + through the tube's total CFU. This is why a total smoothed across + bioreplicates gives much steadier ``ln_cfu`` trajectories than each tube's + own OD600 estimate. Earlier analyses used the pooled mean per condition + and time-point. +- Everything before the split, such as transformation and outgrowth, is + shared by all tubes of a replicate. It sets each genotype's starting + abundance, ``ln_cfu0``, which the ``presplit`` sample measures directly. +- No step is shared by the tubes of one row and not by the others. Rows + are stopped within seconds and processed with everything else. The one + batched step is DNA extraction. Extraction yield does not change a + sample's genotype frequencies, so it should not shift ``ln_cfu``. If it + matters at all, it is through the amount of template going into PCR, + which changes how noisy the counts are. + +The library config +------------------ -**Outputs** (written to ``out_dir``): +One YAML file describes the screened library. Pass the same file to +``tfs-process-fastq`` and to ``tfs-configure-model --library_config``. +Nothing checks that the two saw the same file, and a genetics mismatch +shows up only later, when ``tfs-configure-model`` finds genotypes the +library does not contain. An example is +:download:`examples/process_raw/library_config.yaml +<../../examples/process_raw/library_config.yaml>`. + +``reading_frame`` + Reading frame of the amplicon: 0, 1 or 2. +``first_amplicon_residue`` + Residue number of the first in-frame codon. It sets the numbering of + every genotype name, so a wrong value mismatches every mutant. +``wt_seq`` + The wildtype DNA sequence of the amplicon. +``degen_sites`` + Degenerate codon pattern, the same length as ``wt_seq``: ``.`` for a + wildtype base, or a code such as ``nnt`` or ``nnk`` at library codons. +``tiles`` + Contiguous blocks of library sites cloned together. ``.`` is wildtype + and each other character names a tile. Blocks must be contiguous. +``expected_5p``, ``expected_3p`` + Flanking sequences immediately upstream and downstream of the amplicon. + Read only by ``tfs-process-fastq``. +``tile_combos`` + Which variants the library holds: ``single-x`` is every single mutant in + tile ``x`` and ``double-x-y`` every pairwise combination between tiles + ``x`` and ``y``. +``spiked_seqs`` + DNA sequences of monoclonal controls spiked into the pool. Their + genotypes may also occur in the bulk sub-libraries. +``library_mixture`` + Relative amount of each sub-library in the pool, keyed by the + ``tile_combos`` entries plus ``spiked``. Only ratios matter. Read only by + ``tfs-configure-model``, which uses it to compute each genotype's share + of the pool and how much of it comes from the bulk libraries. Nothing in + the data can check these numbers, so give the best estimate of what + went into the pool, not just the design. + +Any other key is ignored, so a ``tfs-simulate`` config can serve as the +library config of a simulated experiment. + +Counting reads: tfs-process-fastq +--------------------------------- + +``tfs-process-fastq`` reads one tube's paired FASTQ files, trims each read +to the amplicon between its flanks, masks low-quality bases and calls the +protein genotype against every sequence the library config allows. The +library config comes first: -* ``stats_{filename}.csv`` — overall read success/failure statistics. -* ``counts_{filename}.csv`` — raw counts for each expected genotype. +.. code-block:: bash -**Usage:** + tfs-process-fastq library_config.yaml \ + kanR_kanRpkan_0mM_t0_R1.fastq.gz kanR_kanRpkan_0mM_t0_R2.fastq.gz \ + --out_dir counts + +It writes two files into ``--out_dir`` (default ``counts``), named after the +read-1 file: + +- ``counts_.csv``, with columns ``genotype`` and ``counts``. It has + one row for every genotype in the library, zeros included, plus one + ``__unknown__`` row. ``__unknown__`` counts read pairs that oriented and + trimmed cleanly but matched no library genotype, or matched more than + one. It is not a genotype. ``tfs-process-counts`` adds it to the tube's + read depth, the denominator of every frequency, and then drops it. +- ``stats_.csv``, with columns ``success``, ``result``, ``counts`` + and ``fraction``: how many read pairs ended in each outcome, such as a + missing flank, a read too short after quality trimming, or a disagreement + between the forward and reverse reads. Check it when a tube has few + reads or a large ``__unknown__`` share. + +``tfs-process-counts`` finds a tube's count file by globbing +``counts**.csv``, so name the FASTQ files after the tubes. Each +tube's name must match exactly one file. A tube named ``t1`` next to one +named ``t10`` matches two files and is an error. + +The main tuning flags are ``--phred_cutoff`` (default 10), +``--min_read_length`` (default 50), ``--allowed_num_flank_diffs`` (default 1) +and ``--allowed_diff_from_expected`` (default 2). ``--num_workers`` defaults to +-1, which uses every CPU but one, and ``--chunk_size`` (default 10000) is the +number of reads sent to a worker at a time. + +The tube table +-------------- + +The tube table has one row per sequenced tube. It is a CSV, TSV or Excel +file. + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Column + - Meaning + * - ``sample`` + - Unique tube name, matched to its count file. + * - ``library`` + - The transformed library the tube came from. Optional; ``default`` + when absent. kanR and pheS libraries are transformed and grown up + separately, so they are separate libraries. + * - ``replicate`` + - Biological replicate. + * - ``condition_pre`` + - Growth condition before selection, such as ``kanR-kan``. + * - ``t_pre`` + - Minutes of pre-selection growth. + * - ``condition_sel`` + - Selection condition, such as ``kanR+kan``. + * - ``t_sel`` + - Minutes of selection growth. + * - ``titrant_name`` + - The titrant, such as ``iptg``. + * - ``titrant_conc`` + - Titrant concentration; 0 for none. + * - ``od600`` + - The tube's OD600 reading. Optional; it can come from a separate + ``--od600_file`` instead. + +``tfs-process-counts`` itself needs only ``sample``, the tube totals and, +with ``--presplit``, ``replicate`` and ``condition_pre``. Every other column +is carried through to the output, and ``tfs-configure-model`` needs the +design columns above. A selection condition is one that never appears as a +``condition_pre`` within its library; the control arm repeats the +pre-selection condition as its ``condition_sel``. + +Lab-specific cleanup belongs in the lab's own script, before the tube table +is written: tubes resequenced and pooled, sample sheets merged. tfscreen +takes the tube table as the record of what each tube is. When the data +later show that a record is wrong, correct it with ``tfs-correct-tubes`` +(below) rather than editing the table by hand. + +Correcting the tube table: tfs-correct-tubes +-------------------------------------------- + +Sometimes a tube's contents disagree with its label. In one screen the +genotype composition of the tube labeled 0.1 uM IPTG looked like a 1 mM +tube, and the tube labeled 1 mM looked like a low-IPTG tube. A correction +like that is data: it should say which tubes, what changes, why, who +decided and when, and it should be applied by a tool so the original +table stays as it was. ``tfs-correct-tubes`` does that: .. code-block:: bash - tfs-process-fastq [options] + tfs-correct-tubes tube_table.csv corrections.yaml \ + --out_prefix tube_table_corrected + +The corrections file is YAML and lives with the experiment's data, under +version control, next to the tube table it corrects: + +.. code-block:: yaml + + description: Corrections to the screen's tube table. + corrections: + - id: pheS-rep2-t82-swap + action: swap + tubes: [tube_a, tube_b] + reason: > + Composition QC: the tube labeled 0.1 uM looks like a 1 mM tube + and the one labeled 1 mM like a low-IPTG tube. + by: A. Researcher + date: 2026-10-15 + evidence: qc/apparent_iptg.csv + +``id``, ``action``, ``tubes`` (the tube table's ``sample`` names), +``reason``, ``by`` and ``date`` are required. ``evidence`` is optional, a +string or a list; one that names a file, relative to the corrections file, +is hashed into the provenance. The table ``tfs-check-tubes`` writes, which +flags tubes whose composition disagrees with their titrant label, is the +usual evidence for a swap. There are three actions: + +- ``swap``: two tubes of one library exchange their design columns. +- ``relabel``: one tube takes the design values given in ``set``, for + example ``set: {t_sel: 125}``. Three relabels express a rotation of three + tubes. +- ``exclude``: one or more tubes are left out. + +**What a correction moves.** Only the design columns: ``replicate``, +``condition_pre``, ``t_pre``, ``condition_sel``, ``t_sel``, +``titrant_name`` and ``titrant_conc``. Every other column stays with the +physical sample: its ``sample`` name and so its count file, its OD600 and +tube totals, its barcodes and sequencing run. That is right however the +swap happened. If the tubes were swapped at the bench, the reads and the +OD600 were measured on the culture now known to have the other design. If +the sample sheet mapped files to the wrong tubes, the count file came from +the culture whose design is the other label, and that culture's OD600 was +read when it was pulled. Either way a sample's measurements stay together +and only its label changes. ``library`` never changes: libraries are +transformed and grown separately, so a swap between them is refused. + +No tube may appear in two corrections, so every correction applies to the +original table and their order does not matter. The command also refuses +an unknown tube, a swap that changes nothing, a relabel of a column that +is not a design column, a result in which two tubes share a library and +design, an empty ``reason``, ``by`` or ``date``, a table that was already +corrected, and an output that would overwrite an input. + +**Output.** ``{out_prefix}.csv`` is the corrected tube table, in the +original row order without excluded tubes, plus a ``correction_id`` column +(empty for tubes no correction touched). Pass it to +``tfs-process-counts`` in place of the original; ``correction_id`` is +carried into the growth table. ``{out_prefix}_changes.csv`` has one row per +changed value or excluded tube, with its original and corrected value. +``{out_prefix}_provenance.json`` records the run, the SHA-256 of the tube +table, the corrections file, every evidence file and the corrected table, +and the corrections themselves. + +OD600 calibration +----------------- -**Positional arguments:** +``tfs-calibrate-od600`` fits the calibration from two small experiments, both +done with the same handling as production samples: same reader, plate type, +volume and strain. + +**Repeated readings of a dilution series.** Read each dilution several +times, repeating the whole step each time: swirl the culture, pipette into +the plate, read. The file has one row per reading, with columns +``dilution`` (relative to the undiluted culture) and ``od600``. The largest +relative SD across dilutions is the reading noise. The detection threshold +is midway between the mean readings of the two most dilute samples, where +the series has flattened onto the reader's floor. ``--detection_threshold`` +overrides it. + +**Plate counts of cultures whose OD600 was read.** The file has one row per +plate, with columns ``od600``, ``colonies``, ``dilution`` (the total dilution +factor before plating), ``plated_volume_mL``, ``num_dilutions`` and +``plating_steps``. CFU/mL is ``colonies * dilution / plated_volume_mL``. Its +relative variance is ``1 / colonies`` for counting plus +``pipette_rel_error^2`` for each dilution and plating step +(``--pipette_rel_error``, default 0.02). + +A polynomial of CFU/mL in OD600 is fit by weighted least squares +(``--degree``, default 2): -* ``f1_fastq``: Path to the read-1 FASTQ file (gzip-compressed accepted). -* ``f2_fastq``: Path to the read-2 FASTQ file. -* ``out_dir``: Directory to write output CSV files (created if absent). -* ``run_config``: Path to the library configuration YAML file. +.. code-block:: bash -**Optional arguments:** + cd examples/od600 + python make_example_data.py + tfs-calibrate-od600 replicates.csv plate_counts.csv --out_prefix od600_calibration -* ``--phred_cutoff``: Minimum Phred quality score; bases below this threshold - are replaced with ``N`` (default: 10). -* ``--min_read_length``: Discard reads shorter than this length (default: 50). -* ``--allowed_num_flank_diffs``: Allowed mismatches when locating 5′ and 3′ - flanks (default: 1). -* ``--allowed_diff_from_expected``: Allowed mismatches from library genotypes - (default: 2). -* ``--print_raw_seq``: If set, prints sequence matches to stdout for debugging. -* ``--max_num_reads``: Stop after this many reads. -* ``--chunk_size``: Block size for multiprocessing batches. -* ``--num_workers``: Number of parallel workers (default: available CPUs − 1). +``od600_calibration.yaml`` holds the coefficients and their full covariance, +the reading noise, the detection threshold and the calibrated OD600 range. +This is the file ``tfs-process-counts`` reads. ``od600_calibration.pdf`` +shows the dilution series, the fit and the calibrated CFU's relative error. +The two CSVs hold the per-dilution noise and the per-plate fit with its +standardized residuals. -tfs-process-counts ------------------- +The covariance matters. The curve's error is one error shared by every tube +calibrated with it. It does not average out across tubes and must not be +treated as independent per-tube noise, so the pipeline keeps it apart from +the reading noise: ``tfscreen.process_raw.od600.cfu_per_mL_error_components`` +returns the two parts separately, and ``tfs-process-counts`` writes them as +separate columns. -Takes the per-sample count CSVs produced by ``tfs-process-fastq``, aggregates -them according to a sample metadata file, and converts raw counts into -ln(CFU) values using per-sample CFU estimates. +The files in ``examples/od600/`` are synthetic and show the formats. They +are not any instrument's calibration; do not use them for real data. -**Output:** A single CSV file (``output_file``) containing ``ln_cfu`` per -genotype across all samples, ready for hierarchical modelling. +Counts to the growth table: tfs-process-counts +---------------------------------------------- -**Usage:** +``tfs-process-counts`` takes the tube table and the directory of count +files: .. code-block:: bash - tfs-process-counts [options] - -**Positional arguments:** - -* ``sample_df``: Path to a CSV file describing samples. Must contain a unique - ``sample`` column (used as the row index). Required columns: - - * ``sample`` *(index)* — unique identifier for each sequenced tube; used - to match this row to a counts CSV file. - * ``library`` — name of the physical library this sample belongs to. Genotypes - are filtered and frequency-normalised within each library. - * ``sample_ln_cfu`` — natural log of the total colony-forming units (CFU) - measured for this tube. - * ``sample_ln_cfu_std`` — standard deviation of ``sample_ln_cfu``. - - Instead of ``sample_ln_cfu``/``sample_ln_cfu_std``, you may supply - ``sample_ln_cfu_var``, or the linear-space ``sample_cfu`` with - ``sample_cfu_std`` (or ``sample_cfu_var``); the log-space columns are then - inferred. Log-space columns are used when both are present. - - The following columns are not used by ``tfs-process-counts`` itself but are - carried through into the ``ln_cfu`` output and are **required by the growth - model** (``tfs-fit-model``): - - * ``replicate`` — integer replicate index distinguishing biological - replicates that share the same condition. - * ``condition_pre`` — name of the pre-selection growth condition (e.g. - ``-kan``). Identifies the baseline growth arm. - * ``condition_sel`` — name of the selection condition (e.g. ``+kan``). - * ``titrant_name`` — name of the chemical titrant applied (e.g. - ``IPTG``). Use a constant placeholder (e.g. ``none``) when no titrant - is present. - * ``titrant_conc`` — concentration of the titrant (float; 0 for no titrant). - * ``t_pre`` — duration (minutes) of pre-selection growth. - * ``t_sel`` — duration (minutes) of selection growth. - -* ``counts_csv_path``: Directory containing the per-sample count CSV files. - Each file is found by globbing - ``{counts_glob_prefix}*{sample}*.csv`` within this directory. -* ``output_file``: Path for the output ln_cfu CSV. - -**Optional arguments:** - -* ``--counts_glob_prefix``: File prefix used when globbing for count files - (default: ``counts``). -* ``--min_genotype_obs``: Minimum total counts across all samples for a - genotype to be retained (default: 10). -* ``--pseudocount``: Pseudocount added to zero counts before log transformation - (default: 1). -* ``--verbose``: If set, prints a summary of matched samples and file paths. - -tfs-process-presplit --------------------- - -Processes count files from the pre-split time-point — the single pooled -sample collected **before** the library is divided into separate selection -arms. These abundances anchor the initial per-genotype CFU values (``ln_cfu0``) -in the growth model. The output CSV is passed to ``tfs-fit-model`` via the -``presplit_df`` argument. - -The interface is identical to ``tfs-process-counts``, but ``sample_df`` must -also include ``replicate`` and ``condition_pre`` columns so that the output -can be matched to the correct growth conditions. - -**Output:** A single CSV file (``output_file``) with columns ``replicate``, -``condition_pre``, ``genotype``, ``ln_cfu``, and ``ln_cfu_std``. - -**Usage:** + tfs-process-counts tube_table.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix growth + +It writes ``growth.csv`` (default ``tfs_growth.csv``) and prints which count +file it matched to each tube. ``--no_verbose`` turns that listing off. + +**Tube totals.** Every tube needs its total cells, given one of two ways. +The first is OD600. Give one reading per tube, either as ``--od600_file`` +(columns ``sample`` and ``od600``) or as an ``od600`` column in the tube +table, together with ``--od600_calibration_file`` and ``--tube_volume_mL``. +The calibration gives CFU per mL and the volume turns that into cells in the +tube. These columns are then added: + +- ``sample_cfu``: cells in the tube. +- ``sample_cfu_std``: its SD, with the curve and reading errors combined. +- ``sample_cfu_curve_std``: the calibration curve's part, shared by every + tube read through the same calibration. +- ``sample_cfu_reading_std``: the reading's own part, independent per tube. +- ``od600_in_calibrated_range``: False for a reading outside the OD600 range + the plate counts covered, where the curve is extrapolated. + +A reading below the calibration's detection threshold is an error, since +its total is only an upper bound. Drop those tubes from the tube table. + +The second way is to supply the totals yourself, as ``sample_cfu`` with +``sample_cfu_std`` (or ``sample_cfu_var``) in the tube table, or the log +forms ``sample_ln_cfu`` with ``sample_ln_cfu_std`` (or +``sample_ln_cfu_var``). Totals must be cells in the whole tube. Use one +route for the whole table. A tube table that supplies totals and OD600 +together is an error. + +**Several libraries.** All libraries go through one call. Genotypes are +filtered and filled in within each library: a genotype with fewer than +``--min_genotype_obs`` reads (default 10) summed over a library's tubes is +dropped from that library, and a genotype kept in a library gets a row with +zero counts in every tube of that library where it was not seen. + +**Output.** One row per genotype and tube. It carries the tube table's +columns and the tube totals, plus: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Column + - Meaning + * - ``genotype`` + - Genotype name: ``wt``, ``M42I``, ``M42I/K84L``. + * - ``counts`` + - Reads of the genotype in the tube, without pseudocount. + * - ``sample_reads`` + - The tube's total reads, ``__unknown__`` included, without + pseudocounts. The count likelihood uses it as the tube's depth. + * - ``adjusted_counts`` + - ``counts + --pseudocount`` (default 1). + * - ``frequency`` + - ``adjusted_counts`` over the tube's read depth plus one pseudocount + per genotype. + * - ``sample_ln_cfu``, ``sample_ln_cfu_std`` + - The tube's total in log form. + * - ``ln_cfu``, ``ln_cfu_var`` + - ``ln(frequency) + sample_ln_cfu``, the genotype's cells in the tube, + and its variance: binomial sampling of the frequency plus + ``sample_ln_cfu_std^2``. + * - ``cfu``, ``cfu_var`` + - The same in linear form. + +The model takes this file as it is (``tfs-configure-model --growth_df``). By +default it observes ``counts`` against ``sample_reads`` and ``sample_ln_cfu`` +rather than ``ln_cfu``; see :doc:`model-inputs`. + +**Presplit.** With ``--presplit``, the tube table lists the presplit tubes, +one per library, replicate and ``condition_pre``, and must have +``replicate`` and ``condition_pre`` columns. The output, ``tfs_presplit.csv`` +by default, has columns ``library``, ``replicate``, ``condition_pre``, +``genotype``, ``ln_cfu`` and ``ln_cfu_std``. Pass it to +``tfs-configure-model --presplit_df``. + +A worked example +---------------- + +``examples/process_raw/`` holds a small synthetic experiment: two libraries +(kanR and pheS), each with a selection and a control condition, two IPTG +concentrations and three time-points, for 24 tubes. ``make_example_data.py`` +writes ``tube_table.csv``, ``od600.csv`` and ``counts/``, the count files in +the format ``tfs-process-fastq`` writes. It uses the calibration in +``examples/od600/``: .. code-block:: bash - tfs-process-presplit [options] - -**Positional arguments:** - -* ``sample_df``: Path to a CSV file describing the pre-split samples. Must - contain a unique ``sample`` column plus: + cd examples/process_raw + python make_example_data.py + tfs-process-counts tube_table.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file ../od600/od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix growth - * ``library`` — physical library name (optional; defaults to ``default`` when - absent, but must be consistent with the growth ``sample_df``). - * ``replicate`` — replicate index; matched to the growth data. - * ``condition_pre`` — pre-selection condition name; matched to the growth - data. - * ``sample_ln_cfu`` — natural log of the total CFU for this tube. - * ``sample_ln_cfu_std`` — standard deviation of ``sample_ln_cfu``. +``growth.csv`` is a growth table ready for the next step: - As for ``tfs-process-counts``, these may instead be inferred from - ``sample_ln_cfu_var`` or from ``sample_cfu`` with ``sample_cfu_std`` (or - ``sample_cfu_var``). - -* ``counts_csv_path``: Directory containing the per-sample count CSV files. -* ``output_file``: Path for the output presplit CSV. +.. code-block:: bash -**Optional arguments:** + tfs-configure-model --growth_df growth.csv --library_config library_config.yaml -* ``--counts_glob_prefix``: File prefix for globbing count files (default: - ``counts``). -* ``--min_genotype_obs``: Minimum total counts for a genotype to be retained - (default: 10). -* ``--pseudocount``: Pseudocount added before log transformation (default: 1). -* ``--verbose``: If set, prints a summary of matched samples and file paths. +The numbers are made up. They show the formats, not a realistic screen. diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index d1fa59ae..e60d1ca8 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -1,33 +1,44 @@ =========== -Quick Start +Quick start =========== -This page walks through a complete simulate-and-analyze run using the -bundled example in ``examples/simulate-and-analyze/``. By the end you -will have simulated a synthetic high-throughput TF-library screen, fitted -the hierarchical Bayesian model, and produced a full set of diagnostic -plots and statistics. +This page runs the bundled example in ``examples/simulate-and-analyze/`` +from start to finish. It simulates a small TF-library screen, fits the +hierarchical Bayesian model to the simulated data, and compares the fit with +the known truth. -The example uses a small library (~600 genotypes across three tiles) -so the full pipeline finishes in roughly 20–60 minutes on a laptop CPU or a -few minutes on a GPU. +The example library has 483 genotypes: wild type, the single mutants at +three NNT sites (M42 in tile 1, H74 and K84 in tile 2), the double mutants +between the two tiles, and nine spiked control sequences, three of which +are not in the bulk library. +It is grown under kanR and pheS selection at eight IPTG concentrations in +two replicates. The fit is the slow step. Expect tens of minutes on a +laptop CPU and a few minutes on a GPU. + +To analyze a real screen, follow :doc:`screen-analysis` instead. Prerequisites ------------- -Install ``tfscreen`` and verify the entry points are on your PATH: +Install ``tfscreen`` and check that the commands are on your PATH: .. code-block:: bash git clone https://github.com/harmslab/tfscreen cd tfscreen pip install -e . - tfs-simulate --help # should print usage + tfs-simulate --help + +JAX and Numpyro are installed as dependencies. On Linux, pip's plain +``jax`` runs on the CPU only; for a GPU with CUDA 12, install the +``cuda12`` extra instead and check that JAX sees the GPU: + +.. code-block:: bash -The example requires JAX and Numpyro, which are installed as dependencies. -On Apple Silicon you may need to install the Metal-accelerated ``jax-metal`` -package separately; on a Linux cluster with a GPU, install the appropriate -``jaxlib`` wheel for your CUDA version. + pip install -e ".[cuda12]" + python -c "import jax; print(jax.devices())" + +tfscreen needs Python 3.12 or newer. Getting the example ------------------- @@ -39,118 +50,160 @@ Copy the example directory to a working location: cp -r examples/simulate-and-analyze/ ~/tfscreen-example cd ~/tfscreen-example -The directory contains: - -* ``simulate_config.yaml`` — simulation parameters (library genetics, - growth conditions, binding data). -* ``hill_params.csv`` — per-genotype Hill binding parameters that set the - ground-truth θ values used during simulation. -* ``run.sh`` — the pipeline script. +It holds three files. ``simulate_config.yaml`` sets up the simulation: +library genetics, conditions, growth parameters, binding and presplit data. +It also serves as the library description for ``tfs-configure-model``. +``hill_params.csv`` holds the Hill parameters of wild type and three spiked +single mutants, which become their true *θ* curves and their binding data. +``run.sh`` is the pipeline. Running the pipeline -------------------- -.. code-block:: bash - - bash run.sh simulate_config.yaml out/ 1 +Run it from the example directory, since the config names +``hill_params.csv`` by a relative path: -The three positional arguments are: - -1. The simulation config file. -2. The output directory (created automatically). -3. The random seed (``1`` reproduces the documented example outputs). - -On a local machine the script uses JAX's CPU parallelism -(``XLA_FLAGS="--xla_force_host_platform_device_count=8"``). On a -cluster, comment that line out and uncomment ``module load cuda/...`` -instead. - -The script runs nine steps in sequence, printing a ``>>>`` header before -each one: +.. code-block:: bash -.. list-table:: - :header-rows: 1 - :widths: 5 30 65 - - * - Step - - Command - - What it does - * - 1 - - ``tfs-simulate`` - - Builds the genotype library, draws per-genotype θ from the Hill - model, converts θ to growth rates, and writes simulated sequencing - data. - * - 2 - - ``tfs-configure-model`` - - Validates inputs, selects model components, writes - ``tfs_configure_config.yaml``. - * - 3 - - ``tfs-prefit-calibration`` - - Fast MAP fit to calibrate the growth-linking-function priors. - * - 4 - - ``tfs-fit-model`` - - Main SVI inference; writes ``tfs_fit_model_checkpoint.pkl``. - * - 5 - - ``tfs-sample-posterior`` - - Draws posterior samples; writes ``tfs_posterior.h5`` (~5 GB). - * - 6 - - ``tfs-extract-params`` - - Summarises the posterior into per-parameter CSV files. - * - 7 - - ``tfs-predict-theta`` - - Predicts θ at every (genotype, titrant concentration) in the - training data. - * - 8 - - ``tfs-predict-growth`` - - Predicts ln(CFU) with posterior quantiles for all training - observations. - * - 9 - - ``tfs-summarize-fit`` - - Computes fit statistics and writes diagnostic plots and CSVs to - ``out/summary/``. + bash run.sh simulate_config.yaml out 1 + +The arguments are the simulate config, the output directory (created if +needed) and the random seed (default 1). The seed goes to the simulation, +the pre-fit and the fit. + +On a laptop the script has JAX spread work over eight CPU devices +(``XLA_FLAGS="--xla_force_host_platform_device_count=8"``). On a cluster +with a GPU, comment that line out and uncomment ``module load cuda/...``. +The ``#SBATCH`` lines at the top let you submit the same script with +``sbatch``. + +The script runs nine steps and prints a ``>>>`` header before each one. + +1. ``tfs-simulate simulate_config.yaml --out_prefix out/tfs_sim --seed 1`` + builds the library, draws every genotype's *θ* curve from the + ``hill_mut`` model with sparse epistasis, grows two replicates and + sequences them. It writes the growth, binding and presplit tables plus + the ground truth, and the experiment in raw form: one counts file per + tube in ``tfs_sim_counts/`` and the tube table ``tfs_sim_tubes.csv``. The + script then moves into ``out/`` and runs ``tfs-process-counts`` on the + raw files, exactly as on a lab's data, to make + ``tfs_sim_processed_growth.csv``. + +2. ``tfs-configure-model`` picks the model. The script passes the binding, + processed growth and presplit tables and the simulate config as + ``--library_config``, and chooses components that match how the data + were simulated: ``linear`` condition growth, an ``instant`` growth + transition, ``hierarchical`` ln_cfu0, ``hierarchical_geno`` dk_geno, + ``fixed`` activity, ``hill_mut`` theta with ``--epistasis``, the + ``single`` transformation (the simulation puts one plasmid in each + cell), ``passthrough`` theta rescaling, ``logit_normal`` growth-side + theta noise and ``zero`` binding-side noise. Growth is observed through + the ``counts`` likelihood with a ``level`` offset per tube and ``zero`` + growth noise, the recommended default. ``--growth_shares_replicates`` + gives both replicates the same growth parameters. + +3. ``tfs-prefit-calibration`` runs a MAP fit on a simplified model of the + genotypes with binding data and writes each condition's growth baseline + *k* and slope *m* into the priors and guesses CSVs. + +4. ``tfs-fit-model --analysis_method svi`` runs a MAP warm-up, then fits the + variational guide. This is the long step. + +5. ``tfs-sample-posterior --skip_growth_observations`` draws 10,000 + posterior samples from the guide. The flag leaves the stored + per-observation growth sites out of the file. They would be several GB + here, and the later steps recompute growth from the parameter samples. + +6. ``tfs-extract-params`` writes posterior quantiles for each parameter + group. + +7. ``tfs-predict-theta`` predicts *θ* at every genotype and concentration in + the training data. + +8. ``tfs-predict-growth --num_marginal_samples 500`` predicts every + training ln_cfu from 500 posterior samples. + +9. ``tfs-summarize-fit .`` compares the predictions with the simulated + truth and writes plots and statistics to ``out/summary/``. + +:doc:`pipeline` explains each step and :doc:`cli` lists every flag. Expected outputs ---------------- -After the run completes, ``out/`` will contain: +Every command writes a ``*_provenance.json`` next to its outputs with the +tfscreen version, git commit and command line. After the run, ``out/`` +holds: .. list-table:: :header-rows: 1 - :widths: 38 62 + :widths: 40 60 - * - File / directory + * - File - Contents - * - ``tfs_sim_library.csv`` - - All genotypes in the simulated library. + * - ``tfs_sim_counts/``, ``tfs_sim_tubes.csv`` + - The raw experiment: one counts file per tube and the tube table. + * - ``tfs_sim_processed_growth.csv`` + - The growth table ``tfs-process-counts`` made from them, which the + model reads. * - ``tfs_sim_growth.csv`` - - Simulated ln(CFU) data (same format as ``tfs-process-counts`` output). + - The same table written directly by the simulator, with each row's + true values. * - ``tfs_sim_binding.csv`` - - Simulated binding curve observations. + - Simulated binding curves for the four genotypes in + ``hill_params.csv``. + * - ``tfs_sim_presplit.csv`` + - Simulated presplit abundances. + * - ``tfs_sim_library.csv`` + - Every genotype in the library and its origin. * - ``tfs_sim_parameters.csv`` - - Ground-truth per-genotype parameters (θ, dk_geno, etc.). + - True per-genotype parameters (Hill parameters, ``dk_geno``, + activity). + * - ``tfs_sim_genotype_theta.csv`` + - True *θ* for every genotype and concentration. + * - ``tfs_sim_growth_parameters.csv`` + - True growth *k* and *m* for each condition. + * - ``tfs_sim_transformation_lam.csv`` + - The simulated congression rate. + * - ``tfs_sim_input-config.yaml`` + - The simulate config as run. * - ``tfs_configure_config.yaml`` - - Model configuration read by all downstream steps. + - Model configuration read by every later step. + * - ``tfs_configure_priors.csv``, ``tfs_configure_guesses.csv`` + - Priors and starting values, updated by the pre-fit (the originals + are kept as ``.bak``). + * - ``tfs_configure_library.csv`` + - Library composition resolved from the simulate config. + * - ``tfs_configure_model_stats.csv``, ``tfs_configure_model_stats.json`` + - Parameter and observation census of the configured model. + * - ``tfs_prefit_*`` + - Pre-fit diagnostics: checkpoint, parameters, losses, convergence. * - ``tfs_fit_model_checkpoint.pkl`` - Fitted model checkpoint. + * - ``tfs_fit_model_params.npz`` + - Fitted guide parameters. + * - ``tfs_fit_model_losses.txt``, ``tfs_fit_model_convergence.csv`` + - Loss trace and one row per convergence window. + * - ``tfs_fit_model_premap_*`` + - The MAP warm-up's checkpoint, parameters, losses and convergence. + * - ``checkpoints/`` + - Numbered checkpoints written during the fit. * - ``tfs_posterior.h5`` - - Posterior samples (~5 GB; not committed to the repo). + - Posterior samples. * - ``tfs_params_*.csv`` - - Posterior summaries (quantiles) for each parameter group. + - Posterior quantiles for each parameter group. * - ``tfs_pred_theta.csv`` - - Predicted θ with posterior quantiles. + - Predicted *θ* with posterior quantiles. * - ``tfs_pred_growth.csv`` - - Predicted ln(CFU) with posterior quantiles. + - Predicted ln_cfu with posterior quantiles. * - ``summary/`` - - Diagnostic plots and statistics from ``tfs-summarize-fit``. + - Plots and statistics from ``tfs-summarize-fit``. Next steps ---------- -* **Understand each analysis step** — see :doc:`analysis` for the full CLI - reference covering every ``tfs-*`` command in the pipeline. -* **Interpret the diagnostic outputs** — see :doc:`summarize-fit` for a - guided walkthrough of every plot and statistic in ``out/summary/``. -* **Simulate a parameter sweep** — see :doc:`grid` for how to run a - Cartesian grid of configurations. -* **Use real data** — see :doc:`process-raw` for converting FASTQ reads - into the ``growth.csv`` format that ``tfs-configure-model`` accepts. +:doc:`summarize-fit` walks through every plot and statistic in +``out/summary/``. :doc:`simulation` documents the simulate config, so you +can change the library, the conditions or the noise. :doc:`grid` runs a +sweep of settings. :doc:`process-raw` turns real FASTQ files into the growth +table that ``tfs-configure-model`` reads. diff --git a/docs/source/screen-analysis.rst b/docs/source/screen-analysis.rst new file mode 100644 index 00000000..3fe0c773 --- /dev/null +++ b/docs/source/screen-analysis.rst @@ -0,0 +1,347 @@ +================== +Analyzing a screen +================== + +This page takes a real screen from per-tube read counts to one table of +per-genotype response curves with 95% intervals. It is the route used for +the current manuscript. :doc:`pipeline` lists every command the package +has; this page is the subset that a screen analysis needs, in order, with +the reasons for each choice. + +The result is a curve of *X* against IPTG for every genotype. *X* is an +occupancy-like variable gauged on wild type: wt's *X* is 1 at the lowest +IPTG concentration and 0 at the highest. Read `What X is`_ before using +the numbers. + +.. contents:: On this page + :local: + :depth: 1 + +What you need +============= + +From the experiment: + +- **One counts file per tube**, from ``tfs-process-fastq`` (see + :doc:`process-raw`). +- **The tube table**: one row per sequenced tube with its library, + replicate, pre-growth and selection conditions, times and IPTG + concentration. +- **OD600 readings** for every tube and the **OD600 calibration** for the + instrument and strain (``tfs-calibrate-od600``), plus the culture volume. +- **The library YAML**, the same file given to ``tfs-process-fastq``. +- **The measured co-transformation rate** lambda, as a mean and SD. The + fit requires it. For the 2026 screen it was 0.357 (likelihood-ratio 95% + CI 0.156-0.674), entered as 0.38 +- 0.15. + +For the fit: + +- **Loose growth priors**: one row per condition with a growth baseline + ``k`` and slope ``m``. The values only need to be the right order of + magnitude; ``examples/tfmodel/growth_priors_loose.csv`` is the file used + for the 2026 screen. +- **A GPU**, and tfscreen installed with GPU JAX: on a Linux cluster, + ``pip install -e ".[cuda12]"`` in a fresh environment with Python 3.12 + or newer, then check that ``python -c "import jax; + print(jax.devices())"`` lists a GPU (see :doc:`quickstart`). Save the + environment's ``pip freeze`` with the analysis, so its versions are on + record. The whole analysis takes about half a day of GPU time (see + `Cost`_). + +The next section turns the counts, tube table and OD600 readings into the +growth table the fit reads, checking the tube labels on the way. If a +tube turns out to be mislabeled, you will also write a corrections file +there. + +Make the growth table +===================== + +The fit reads one **growth table**, ``growth.csv``: every genotype's +reads and cell numbers in every tube, made by ``tfs-process-counts``. It +is the ``growth_df`` in the fit's YAML (`Configure the fit`_). Before +trusting it, check that each tube's label matches its contents. + +If you have already made and checked the growth table, for example +following :doc:`process-raw`, skip to `Configure the fit`_. + +These commands read every tube's counts for the whole library, which is +too much for a cluster's login node. Run them on a compute node with +plenty of memory, for example in an interactive session: + +.. code-block:: bash + + srun --account=YOUR_ACCOUNT --mem=32G --cpus-per-task=4 --time=02:00:00 --pty bash + +``tfs-process-counts`` used 4-6 GB per library and a few minutes on the +2026 screen; ``tfs-check-tubes`` holds the whole growth table in memory, +so ask for more than that. + +**1. Make a first growth table and check the tubes.** + +.. code-block:: bash + + tfs-process-counts tube_table.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix growth_unchecked + tfs-check-tubes growth_unchecked.csv --out_prefix tube_qc + +``tfs-check-tubes`` scores each tube's apparent IPTG concentration from +its genotype composition and flags tubes that are out of order in their +row of the design (one replicate at one time). Tube totals and read depth +cancel out of the score, so it tests the labels, not the OD600. + +**2. If no tube is flagged**, the first table is the one to use: rename +``growth_unchecked.csv`` to ``growth.csv`` and go on to +`Configure the fit`_. + +**3. If tubes are flagged**, read the whole row of each flagged tube in +``tube_qc.csv`` and decide what happened. A single swap is common and is +fixed with a ``swap``; a row whose order cannot be recovered is left out +with ``exclude``. Write each decision into a corrections file, a YAML +file kept with the experiment's data, recording who decided it, when, +why and the evidence (format: :doc:`process-raw`, "Correcting the tube +table: tfs-correct-tubes"; example: +``examples/process_raw/corrections.yaml``). + +**4. Apply the corrections and make the growth table again** from the +corrected tube table: + +.. code-block:: bash + + tfs-correct-tubes tube_table.csv corrections.yaml --out_prefix tube_table_corrected + tfs-process-counts tube_table_corrected.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix growth + +This writes ``growth.csv``, the table the fit uses. The original tube +table is never edited; ``tube_table_corrected_changes.csv`` lists every +change. + +For the 2026 screen one row is excluded: pheS+4CP, replicate 2, t_sel 82. +Its IPTG tubes were out of order in a way no single swap explains. + +Configure the fit +================= + +The fit is one command, ``tfs-fit-two-stage``, driven by one YAML file. +Its top-level keys are ``tfs-configure-model``'s arguments; the +``two_stage`` block holds the fit's own settings. Paths are relative to +the YAML. Start from ``examples/tfmodel/two_stage.yaml``: + +.. code-block:: yaml + + growth_df: growth.csv + library_config: library_config.yaml + theta_model: hill_relative + growth_shares_replicates: true + batch_size: 4096 + allow_missing_spikes: true + growth_priors: growth_priors_loose.csv + set_priors: + - sigma_fixed=0.17 + - theta_log_hill_n_hyper_scale_fixed=0.5 + skip_model_stats: true + + two_stage: + transformation_lambda: [0.38, 0.15] # the measured lambda, required + seed: 1 + num_posterior_samples: 500 + consistency: true + consistency_library: kanR + predict_epistasis: true + +Keep everything except the data paths and ``transformation_lambda`` as +shown. These are the settings the validation used. +``consistency_library`` only names which library's tubes the consistency +check refits alone (see `What the fit does`_); it is a convention, not a +modeling choice, and defaults to the first library alphabetically +(``kanR`` here). ``sigma_fixed`` holds the per-tube level offsets at an SD of 0.17 +(left free, they absorbed the population's growth on real data), and the +held log Hill-n spread keeps genotypes from hiding their response below +the lowest IPTG concentration. Set ``predict_epistasis: true`` if you want +epistasis on *X*. + +The command refuses settings it does not support: binding data, theta +noise models, the experimental joint-fit priors and a marker warp +(``m_warp``) in the growth priors. + +What the fit does +================= + +Each step writes its own directory under ``--out_dir`` and is skipped +when its outputs exist, so a stopped run resumes where it stopped. + +1. **stage1**: wt, the spiked controls and the single mutants (about 950 + genotypes), fitted with the congression mixture and the measured + lambda. This gives every condition's growth baseline *k* and slope *m*. + The full library cannot be used for this: each double mutant pulls *m* + a little steeper, and over 200,000 doubles the pulls compress *X* about + 1.45-fold. +2. **stage2**: the full library with every condition's *k* and *m* pinned + at stage 1's values, then the Laplace approximation of the posterior. +3. **predict**: stage 2's *X* from the posterior draws, and its epistasis + with ``predict_epistasis: true``. +4. **consistency**: stage 2 refitted on one library's tubes alone + (``consistency_library``, kanR here), compared genotype by genotype + with the fit on both markers. +5. **summary**: the results table, ``tfs_x.csv``. + +Two more steps, **jackknife** and **kshift**, try to add *k*'s +uncertainty to the intervals. They are experimental and off by default +(``kshift: false``): on simulations the shift they measured was a third +or less of the true one (``planning/studies/two-stage-validation/``, +run 2). + +Run it +====== + +Run it one of two ways, not both. + +**On a SLURM cluster** (the usual case), copy +``examples/tfmodel/two_stage_submit.sh`` and +``examples/tfmodel/two_stage_step.srun`` next to your YAML, edit the +account, partition and time in ``two_stage_step.srun`` for your cluster, +and run the submit script from the login node: + +.. code-block:: bash + + bash two_stage_submit.sh two_stage.yaml two_stage + +That is the only command to run. It submits one job per step, each +waiting on the steps it needs, and prints the job ids. Logs go to +``two_stage/logs/``. Every path the jobs use is made absolute, so it +works from any directory. If a job fails, fix the cause and run the +submit script again: finished steps are skipped. + +**On one machine with a GPU**, run everything in order with one command: + +.. code-block:: bash + + tfs-fit-two-stage two_stage.yaml --out_dir two_stage + +This is also what each cluster job calls, one step at a time +(``tfs-fit-two-stage two_stage.yaml --out_dir two_stage --step stage2`` +and so on). + +Cost +---- + +On one GPU, for a library of about 200,000 genotypes: stage 1 takes 15-30 +minutes, stage 2 two to five hours with its Laplace, and the consistency +refit, which runs beside prediction, about five hours. About half a day +in all. + +Read the results +================ + +``{out_dir}/tfs_x.csv`` has one row per genotype and IPTG concentration: + +``genotype``, ``num_mutations``, ``titrant_name``, ``titrant_conc`` + The row. +``q0.001`` ... ``q0.999`` + Quantiles of *X* (``q0.5`` the median, ``q0.025`` and ``q0.975`` the + 95% interval). +``k_shift`` + Blank unless the experimental ``kshift`` is on: then *X*'s shift per + SD of *k* at this row, already included in the quantiles. +``in_training_data`` + 1 if the genotype was measured at this concentration. +``theta_scale`` + ``X``. +``reads``, ``reads_kanR``, ``reads_pheS`` + The genotype's reads over all tubes, in total and per library. +``consistency_z``, ``inconsistent`` + How far the genotype's kanR-only *X* sits from its both-marker *X*, + in SDs after the overall map between the two fits, and whether that + exceeds 2 SD. Blank for wt and when the check did not run. + +Which genotypes to use +---------------------- + +Use genotypes with at least 1,000 reads. Below that, *X* is mostly set by +the population prior: the intervals still cover the truth, but the point +estimates carry little information about the genotype. On a simulation +of the 2026 screen, the double mutants' *X* against the truth, by read +depth: + +=========== =========== ====== ====== ============ ============ +reads genotypes slope RMSE 95% coverage median width +=========== =========== ====== ====== ============ ============ +<= 100 37,707 0.04 2.60 0.98 5.9 +100-1e3 89,022 0.39 0.54 0.90 1.18 +1e3-1e4 64,362 0.88 0.17 0.88 0.48 +1e4-1e5 15,283 1.02 0.07 0.90 0.20 +> 1e5 1,225 1.02 0.05 0.76 0.07 +=========== =========== ====== ====== ============ ============ + +The slope is the truth regressed on the fit; 1 means the fitted *X* is on +the true scale. The 2026 screen has about 77,000 double mutants above +1,000 reads. + +Report genotypes flagged ``inconsistent`` separately rather than dropping +them silently. On the 2026 screen about 10% of deep genotypes are flagged, +against about 2% on simulations with no marker problem, so most flags are +a real disagreement between the markers whose cause is not known. + +What X is +========= + +Growth alone fixes occupancy only up to an affine map: any change of +scale and offset in occupancy is absorbed by each condition's *k* and +*m*. So the fit does not estimate occupancy. It estimates *X*, which is +the in vivo occupancy put on a scale where wt reads 1 at the lowest IPTG +and 0 at the highest. If wt is not fully repressed or fully induced in the +cell, *X* is still 1 and 0 there. + +*X* is not in vitro binding. Compare the two after the fit, and expect +them to differ: + +- Growth curves are broader than binding curves (Hill n 0.7-0.95 in *X* + against 1.9-3.2 in binding at 20 uM protein). +- wt's growth midpoint (about 21 uM IPTG) is about 8 times its midpoint + in binding at 2 uM protein, and matches its midpoint at 20 uM. +- Order can invert: M42I switches at a fifth of wt's IPTG in growth and + needs 4-5 times wt's in binding. + +No single protein concentration or monotone map carries binding onto +growth for every genotype, so binding is not used to set *X*'s scale. + +Epistasis on *X* is defined only on the additive scale +(``tfs-predict-epistasis --scale add``), since *X* has no fixed bounds. + +What the intervals carry +======================== + +Each genotype's interval comes from the Laplace approximation of stage 2, +with every condition's *k* and *m* held at stage 1's values. On +simulations of this design, the deep genotypes' 95% intervals covered the +truth 0.88-0.91 of the time up to 1e5 reads, and 0.76-0.81 above 1e5 +(table above; the second simulation seed gave 0.91 and 0.81). Treat them +as somewhat narrow, most of all for the deepest genotypes. + +What they leave out: + +- **k's uncertainty.** Stage 1 estimates each condition's growth + baseline with an error that shifts every non-wt genotype's *X* by about + the same amount (one SD of *k* moved *X* by 0.03-0.04 on simulations). + It matters most where the per-genotype interval is narrowest, which is + why the deepest genotypes undercover. It cancels from differences + between genotypes, rankings and epistasis, which are better calibrated + than single curves. +- **Stage 1's small bias in m**, 2-6% steep on simulations. It stretches + *X* by about the same factor. Resampling cannot cover a bias. +- **The pheS gap.** In the pool, pheS selection is about half as strong + as wt's monoculture rate predicts. *X* is set mostly through kanR, so + this need not distort it, but it could if the cause depends on + genotype. The consistency flag is the check. +- **kanR's absolute slope.** The fit takes *m* from the tube totals. + Within-tube comparisons, which do not use the totals, give a kanR slope + about 2.5 times smaller. Binding at 2 uM protein supports the totals' + value. If the within-tube value were right, *X*'s scale would change. +- **Congression's per-genotype error.** Stage 2 does not fit the + congression mixture; for most genotypes the error is small next to the + interval. diff --git a/docs/source/simulation.rst b/docs/source/simulation.rst index 5ab2b0a2..ca2a2dd1 100644 --- a/docs/source/simulation.rst +++ b/docs/source/simulation.rst @@ -2,110 +2,523 @@ Simulation ========== -The ``tfscreen.simulate`` module generates synthetic high-throughput TF-library -selection experiments. Starting from a thermodynamic model of TF binding, it -builds a genotype library, samples per-genotype occupancy (*θ*) values, applies -the growth model to convert occupancy to fitness, and then simulates the -sequencing step. The result is a ``growth.csv`` that has exactly the same -format as the output of ``tfs-process-counts``, so simulated data can be fed -directly into ``tfs-configure-model`` for end-to-end benchmarking. - -Configuration File (simulate_config.yaml) ------------------------------------------- - -All simulation parameters are collected in a single YAML file. An +The ``tfscreen.simulate`` module generates synthetic high-throughput screens +of TF libraries. It builds a genotype library, draws each genotype's +operator occupancy (*θ*) curve, converts *θ* to growth rates, grows the +library through the selection experiment and sequences it. The simulated +growth table has the same format as the output of ``tfs-process-counts``, so +it goes straight into ``tfs-configure-model``. Because the ground truth is +known, a simulation is how we benchmark a model choice or an inference +method. :doc:`quickstart` runs one end to end. + +The simulation config +--------------------- + +One YAML file holds every simulation setting. The :download:`annotated example <../../examples/simulate/simulate_config.yaml>` -is provided in ``examples/simulate/``. The top-level sections are: - -**Library genetics** — same keys as the ``run_config.yaml`` used by -``tfs-process-fastq`` (``reading_frame``, ``wt_seq``, ``degen_sites``, -``tiles``, ``expected_5p``, ``expected_3p``, ``tile_combos``, -``spiked_seqs``). This means a single config file can drive both simulation -and raw-data processing. - -**Phenotype** — controls how genotype *θ* values are drawn: - -* ``theta_component`` — registered theta-model name (e.g. ``hill_geno``, - ``thermo.O2_C12_K5_U0_a.PK``). -* ``theta_rng_seed`` — JAX RNG seed for reproducible *θ* sampling. -* ``theta_priors`` (optional) — dict of hyperparameter overrides for the - chosen component. -* ``thermo_data`` (optional) — path to structural/thermodynamic data required - by ``PnnC`` and ``PddG`` theta components. - -**Conditions** — ``condition_blocks`` list; each entry specifies: -``library``, ``titrant_name``, ``titrant_conc`` list, ``condition_pre``, -``t_pre``, ``condition_sel``, and ``t_sel`` list. - -**Growth** — per-condition ``{m, b}`` parameters mapping *θ* to growth rate -(*k = b + A·m·θ*). ``dk_geno_hyper_*`` sets the pleiotropic growth-cost -distribution. ``activity_wt`` and ``activity_mut_scale`` control per-genotype -TF activity. - -**Experimental parameters** — ``transform_sizes``, ``library_mixture``, -``lib_assembly_skew_sigma``, ``transformation_poisson_lambda``, -``multi_plasmid_combine_fcn``, ``cfu0``, ``tube_noise_sigma``, -``total_num_reads``, ``prob_index_hop``, ``random_seed``. - -**Growth transition** (optional) — ``growth_transition`` list; one entry per -``condition_pre`` that has a detectable lag phase. Supported models: ``instant``, -``memory``. - -**Binding data** (optional) — ``binding_data`` block; if present, a simulated -binding CSV is also written. Required sub-keys: ``genotypes``, -``titrant_name``, ``titrant_conc``, ``noise``. +in ``examples/simulate/`` comments every key. ``tfs-simulate`` refuses a key +it does not know, so a misspelled key fails at once rather than being +ignored. Times are in minutes and growth rates are per minute throughout. + +Library genetics +^^^^^^^^^^^^^^^^ + +The keys ``reading_frame``, ``first_amplicon_residue``, ``wt_seq``, +``degen_sites``, ``tiles``, ``tile_combos``, ``spiked_seqs``, +``expected_5p`` and ``expected_3p`` describe the library. They are the same +keys ``tfs-process-fastq`` reads, and together with ``library_mixture`` they +are what ``tfs-configure-model --library_config`` reads. Keep one file per +experiment and hand that same file to ``tfs-simulate``, ``tfs-process-fastq`` +and ``tfs-configure-model --library_config``. Each tool ignores the keys it +does not use. Nothing checks that ``library_mixture`` is the same in all +three places, so a single file is the only guard against drift. + +Subsampling the library +^^^^^^^^^^^^^^^^^^^^^^^ + +A real double-mutant library has hundreds of thousands of genotypes, far +too many for some studies (a full Laplace posterior needs a dense Hessian +over every genotype's parameters). Four optional keys restrict the described +library to a subset. They belong with the library keys, because +``tfs-configure-model --library_config`` reads them too and builds its +library snapshot from the same subset. ``tfs-process-fastq`` ignores them. + +.. code-block:: yaml + + # keep every single and 1,000 random cross-tile doubles + library_subsample: + double-1-2: 1000 + library_subsample_seed: 20261010 + + # or keep exactly the genotypes listed in a file + library_genotypes: sampled_genotypes.txt + +``library_genotypes`` names a text file with one genotype per line (``#`` +comments allowed), relative to the YAML's own directory. Every genotype not +listed is dropped from every sub-library that encodes it, so the file lists +everything to keep, singles included. A listed genotype the library does +not encode is an error. ``library_subsample`` keeps ``N`` genotypes of a +sub-library, drawn at random, and leaves the sub-libraries it does not name +whole. It needs ``library_subsample_seed``, an integer. That seed is +separate from ``seed`` on purpose: ``tfs-configure-model`` never sees +``tfs-simulate --seed``, and both must draw the same genotypes. Each +sub-library draws from its own stream, and a smaller ``N`` keeps a subset +of what a larger one keeps, so a grid over ``N`` compares nested libraries. +Both keys can be given; the random draw is then made from the listed +genotypes. wt and every genotype a spiked sequence encodes are always kept, +in every sub-library that encodes them, and do not count toward ``N``. A +``double-x-y`` sub-library also encodes singles (and wt) wherever a site's +codon set includes the wt residue; those are drawn like any other member. + +``library_subsample_depth`` says what the experiment settings describe. +With ``full_library`` (the default) ``library_mixture``, ``transform_sizes``, +``total_num_reads`` and ``pcr_template_molecules`` describe the full +library, and every kept sequence keeps the pool share, transformants, reads +and PCR templates it has there: each sub-library's mixture weight and +transformants are multiplied by its kept fraction of sequences, and the +reads and templates by the kept share of the pool. ``cfu0`` is not changed, +so tube totals and OD600 stay those of the full experiment and each kept +genotype has 1 / (kept share) times the cells. A grid over ``N`` then varies +only the number of genotypes, not the depth per genotype. With +``as_written`` the settings apply to the subsampled library as given: the +subsample is the library. ``tfs-simulate`` prints a table of genotypes and +sequences kept per sub-library and the settings it used. + +``tfs_sim_library.csv``, the raw count files and every other output list +only the kept genotypes. Pass the same YAML to ``tfs-configure-model +--library_config``; its snapshot (``*_library.csv``) then holds the same +genotypes with the composition the simulation used, and its config records +the subsample keys under ``library.subsample``. ``tfs-setup-sim-grid`` +copies a ``library_genotypes`` file into the grid, and ``library_subsample`` +is an ordinary grid variable. + +Phenotypes +^^^^^^^^^^ + +``theta_component`` names the theta model that draws each genotype's *θ* +curve. ``hill_geno`` draws an independent Hill curve per genotype around a +wild-type reference. ``hill_mut`` draws per-mutation effects, adds them and, +when the library has double mutants, adds sparse pairwise epistasis from a +regularized horseshoe. The ``thermo.*`` keys are thermodynamic +partition-function models; ``PnnC`` and ``PddG`` variants also need +``thermo_data``, the structural ensemble (HDF5) or the per-mutation ΔΔG CSV. +An optional ``thermo_options`` block says how to read it: ``ddG_units`` +(``kT``, the default, ``kcal/mol`` or ``kJ/mol``), ``temperature_K``, +``column_map``, ``residue_offset`` and ``max_missing_fraction``, with the +same meaning as the ``--thermo_*`` flags of ``tfs-configure-model`` (see +:doc:`model-inputs`). Without it the CSV is read as kT, so the simulated +truth is unchanged. + +In a ``PddG`` or ``PnnC`` simulation the truth is drawn per structure: each +mutation's ΔΔG in each structure is the structural value plus Normal noise +with SD σ_s, one σ_s per structure, and the true constants are its +projection. That is exactly the distribution the models' prior on the +constants assumes. The simulator fixes σ_s at ``ddG_truth_sd`` instead of +drawing it from its prior, so the key is the dial for how far the truth +sits from the structural prior. It is in the +unit of ``thermo_options`` ``ddG_units`` and converted once at load, like +the CSV: with ``ddG_units: kcal/mol``, ``ddG_truth_sd: 0.5`` means +0.5 kcal/mol. Give one number for every structure or a mapping that names +each structure of the model, such as ``{H: 0.5, HO: 0.5, L: 1.0, LO: 1.0, +HE2: 0.5, LE2: 0.5}``. With 0 the truth is the CSV exactly. Without the key +σ_s is 1 kT in every structure. The noise is independent across mutations +and structures. The true σ_s goes to ``tfs_sim_ddG_sigma.csv`` (one row per +structure, in kT), which ``tfs-summarize-fit`` compares with the fit's +``ddG_sigma`` table. The setting changes only the simulated draw. A fit of +the result still centers the prior on the unperturbed CSV, which its +``ddG_prior_mean_{structure}`` tables report, and estimates σ_s under its +own prior. For ``PnnC`` the structural value +in a simulation is the untrained network's output, 0. Other theta +components refuse the key, as does ``phenotype_source: empirical``. In a +``tfs-setup-sim-grid`` grid it is an ordinary top-level variable, and a +mapping is given whole as one variant. + +``theta_sim_priors`` overrides the simulation priors of ``hill_geno`` and +``hill_mut`` (wild-type curve, perturbation widths, epistasis scale), and +``theta_priors`` overrides the hyperparameters of the other components. The +example config lists every key. ``theta_rescale`` (``passthrough``, the +default, or ``logit``) transforms *θ* before it enters the growth model, as +the fit-side component of the same name does. + +The ``growth`` block maps *θ* to a growth rate for every condition named in +``condition_blocks``. Each entry is a dict whose optional ``model`` key picks +the form. The default ``linear`` takes ``b`` and ``m`` and gives +*k = b + A·m·θ + dk_geno*. ``power`` takes ``b``, ``a`` and ``n``, and +``saturation`` takes ``kmin`` and ``kmax``. These match the fit's +``condition_growth`` components, and ``tfs-simulate`` writes the values as +ground truth under the fit's names (``growth_k``, ``growth_m`` and so on). + +Each genotype's pleiotropic growth effect is drawn as +``dk_geno = dk_geno_hyper_shift - exp(Normal(dk_geno_hyper_loc, +dk_geno_hyper_scale))``, with wild type at 0. The three ``dk_geno_hyper_*`` +keys are required unless ``dk_geno_zero: true``, which sets every +genotype's ``dk_geno`` to 0. + +TF activity *A* defaults to 1 for every genotype. With the default +``activity_component: fixed``, ``activity_wt`` sets wild type and +``activity_mut_scale`` > 0 draws each mutant's log *A* around it. +``activity_component`` can instead be ``hierarchical_geno`` or +``horseshoe_geno``, numpy versions of the fit's priors, tuned through +``activity_priors``. + +``phenotype_source: empirical`` replaces all of this with phenotypes +resampled from a distribution fit to real data. See +`Empirical phenotypes`_ below. + +Conditions +^^^^^^^^^^ + +``condition_blocks`` is a list. Each entry gives a ``library``, a +``titrant_name``, a ``titrant_conc`` list, a ``condition_pre`` and its +``t_pre``, and a ``condition_sel`` with a ``t_sel`` list. Every combination +of concentration and selection time is one tube. Every ``condition_pre`` and +``condition_sel`` must appear in ``growth``. + +To follow a real experiment's layout exactly, give ``design`` instead: the +path to a tube table, the format ``tfs-process-counts`` reads (``sample``, +``library``, ``replicate``, ``condition_pre``, ``t_pre``, ``condition_sel``, +``t_sel``, ``titrant_name``, ``titrant_conc``; other columns are ignored). +Each replicate gets exactly its own tubes, so irregular time points and +replicates sampled at different times carry over, and the replicates are +the design's (``--num_replicates`` is ignored). The raw experiment keeps the +design's tube names, so the simulated tube table has the real layout. A +design cannot be combined with ``condition_blocks`` or with +``od600.num_od_only_replicates``. Like every path in the config, it is read +relative to the directory ``tfs-simulate`` runs in. + +``growth_transition`` is an optional list with one entry per +``condition_pre``, describing the switch from pre-growth to selection. +The simulator supports ``instant`` (no lag), ``memory`` (``tau0``, ``k1``, +``k2``; a lag that depends on *θ*), ``baranyi`` (``tau_lag``, ``k_sharp``) +and ``two_pop`` (``k_trans``). Omit the block for an instant transition +everywhere. The fit has two more transition components, ``baranyi_k`` and +``baranyi_tau``, that the simulator does not have. + +Transformation and congression +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``transform_sizes`` gives the number of transformants for each entry in +``tile_combos`` plus ``spiked``. ``library_mixture`` gives the ratio in +which those sub-libraries are pooled. ``lib_assembly_skew_sigma`` spreads +the starting genotype frequencies log-normally; 0 makes them even. It is one +number, or one per sub-library (``{double-1-2: 2.0, single-1: 0.8, ...}``): +a pool of doubles is usually far less even than one of singles. + +Transformants are counted, not listed one by one. Cells carrying one plasmid +are drawn as a number of clones per genotype, and each genotype becomes one +weighted row, so a library can carry a realistic number of transformants +(hundreds per genotype, tens of millions in all) at the cost of one row per +genotype. Nothing downstream changes: founder sampling and demographic noise +add up exactly over identical clones. Cells with several plasmids stay one +row each; ``max_congressed_cells`` caps how many are kept, representing the +rest by a weighted sample of that many cells (an approximation that lumps +their noise). +``cfu0`` is the number of cells in each tube at the start of pre-growth: one +number, or one per library (``cfu0: {kanR: 2.6e7, pheS: 5.3e7}``) when the +libraries were grown up separately. A tube's simulated total scales with +it, which makes it easy to match a real experiment's densities. + +``transformation_poisson_lambda`` gives each cell a zero-truncated Poisson +number of plasmids; 0 or null gives exactly one. A cell with several +plasmids splits its abundance among them and has one growth rate, built from +cell-level physics. ``congression_theta_rule`` sets the cell's *θ*: +``homodimer`` (the default) combines the plasmids' *θ* odds weighted by +share, ``heterodimer`` sits closer to the mean logit, and ``max`` lets the +highest-*θ* plasmid set *θ* and activity. ``homodimer`` and ``heterodimer`` +need activity 1. ``congression_dk_rule`` sets the cell's ``dk_geno``: +``dilution`` (the default, the share-weighted mean), ``softmin`` with a +finite ``congression_dk_alpha``, or ``min``, where the worst plasmid sets +the cost. ``congression_dk_alpha`` is required by ``softmin`` and refused by +the other rules. The fit's ``mixture`` transformation uses the same rules. + +``tube_noise_sigma`` adds one growth-rate offset per tube, shared by every +genotype in it. + +Sequencing and sampling noise +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``total_num_reads`` is the total number of reads across all tubes of all +replicates, split evenly over the sequenced tubes. ``prob_index_hop`` is the +fraction of reads reassigned to a random genotype. +``unassigned_read_fraction`` (one number, or a map from library to a +number) sends that share of each tube's reads to ``__unknown__``, as reads +that call no library genotype; they count toward the tube's reads but carry +no genotype. Real libraries can lose a large share of their reads this way. +``seed`` makes the run reproducible; ``tfs-simulate --seed`` overrides it. + +Without further settings, read counts are Poisson given each tube's +composition. Real counts vary 5 to 18 times more than that at 100 to 3,000 +reads (``planning/studies/noise-anatomy/``). Five optional keys, all off by +default, add the physical sources one at a time. ``founder_sampling`` seeds +each tube with a Poisson number of cells from each transformant clone. +``demographic_growth`` then grows each clone from its founders with +birth-death noise and needs ``founder_sampling``. ``shared_transformation`` +draws every replicate from one library assembly and transformation, as from +one glycerol stock, instead of redrawing them per replicate. +``pcr_template_molecules`` draws each tube's reads from that many template +molecules, and ``pcr_amplification_cv`` gives each template's amplification +that coefficient of variation. The count variance grows by about +(reads per tube / templates)(1 + cv²), so matching real data needs templates +on the order of a tenth of the reads per tube. + +``condition_selector`` and ``library_selector`` name the columns that define +a growth condition and a library. The defaults are right for every config we +know of; leave them out. + +OD600 +^^^^^ + +An optional ``od600`` block gives every tube one OD600 reading. +``calibration`` is a calibration YAML from ``tfs-calibrate-od600`` (the one +in ``examples/od600/`` is synthetic). The simulator inverts it to get each +tube's true OD600 from its true total, then adds reading noise and applies +the detection threshold. ``tube_volume_mL`` (default 5.0) converts the +calibration's per-mL units to the tube. ``num_od_only_replicates`` adds +replicates that are read but not sequenced. With +``sample_cfu_from_od600: true`` the growth table gets each tube's total as +estimated from its reading, as in the lab, instead of the true total. + +A tube read below the detection threshold has only an upper bound on its +total, so the lab drops it, and ``tfs-process-counts`` refuses such a +reading rather than guess. ``tfs-simulate`` drops these tubes from every +route the same way: from the raw tube table and count files, and from the +direct growth table ``tfs_sim_growth.csv`` (also with ``--no_write_raw``). +It prints the dropped tubes. ``tfs_sim_od600.csv`` still lists every +reading with its ``od600_detectable`` flag, and the presplit table, which +samples the library before the split, is not affected. + +Binding data +^^^^^^^^^^^^ + +An optional ``binding_data`` block writes a simulated binding table for +``tfs-configure-model --binding_df``. ``titrant_name``, ``titrant_conc`` and +``noise`` describe the assay. Noise is Gaussian on ``theta_obs`` and is not +clipped to [0, 1], since the fit's binding likelihood is an unclipped +Normal. ``clip_theta_obs: true`` restores the old clipping. + +Two sub-blocks pick the measured genotypes. ``spiked_binding`` draws from +``spiked_seqs``, the clean monoclonal controls. ``library_binding`` draws +from the rest of the library, so its genotypes grow under congression like +any bulk genotype, which is what lets the fit learn the growth slope *m* +from data. Each sub-block takes ``choose_by``: ``stratified`` (curves spread +across the prior), ``random``, or the path to a CSV of measured Hill +parameters (``genotype``, ``theta_low``, ``theta_high``, ``log_hill_K``, +``hill_n``) that become those genotypes' true phenotypes. ``num`` sets how +many genotypes to pick and is forbidden with a file. It is optional for +``spiked_binding`` (default all spikes) and required for a non-file +``library_binding``. ``library_binding`` picks its genotypes after the +growth simulation, from those that survived with growth data. A file of +measured parameters needs a Hill theta component (``hill_geno`` or +``hill_mut``). + +Binding theta is the true theta at the binding concentrations, which need +not match the growth concentrations: the simulator evaluates each genotype's +drawn parameters there, so any theta component works. For the Hill +components, ``stratified`` and ``random`` give the chosen spikes new curves +drawn from the prior. For the thermodynamic components (``thermo.*``) every +genotype's theta follows from per-mutation equilibrium constants, so the +spikes keep their own curves and K truth, and ``stratified`` or ``random`` +only chooses which ``num`` spikes are measured: ``stratified`` takes the +spikes whose binding curves are most spread out. + +Presplit and base-growth data +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An optional ``presplit_data`` block writes a presplit table for +``tfs-configure-model --presplit_df``: one ``ln_cfu`` per replicate, +``condition_pre`` and genotype, sequenced before the culture is split, with +the true ``ln_cfu_0`` alongside. Its one optional key, ``noise``, adds +Gaussian noise on the ln scale. ``presplit_data:`` with nothing under it +turns it on with no extra noise. + +An optional ``base_growth_data`` block writes direct growth-rate +measurements for ``tfs-configure-model --base_growth_df``. ``k_ref`` (the +wild-type rate) is required. ``genotypes`` (default ``[wt]``) lists genotypes +already in the library, each measured at ``k_ref`` plus its own +``dk_geno``. ``rates`` overrides the true rate for listed genotypes, and +``noise`` is the measurement SD, also reported as ``rate_std``. Set +``noise`` above 0 if the table will be fit: ``rate_std`` is a likelihood +scale, and a value of 0 crashes the pre-fit and the fit the first time the +model is traced. tfs-simulate ------------ -Runs a full simulation from a config file and writes output CSVs. +``tfs-simulate`` runs a simulation from a config file: + +.. code-block:: bash + + tfs-simulate simulate_config.yaml --out_prefix sim/tfs_sim --num_replicates 2 --seed 1 -**Usage:** +It draws the ground-truth library and phenotypes once, then simulates +``--num_replicates`` independent replicates (default 2). Files are written as +``{out_prefix}_{name}.csv``; the default prefix is ``tfs_sim``, and a +directory part of the prefix is created if needed. ``tfs-simulate`` refuses +to overwrite existing outputs. See :doc:`cli` for every flag. + +Every run writes these files: + +``tfs_sim_growth.csv`` + The analysis-ready growth table, the same format as + ``tfs-process-counts`` output, with the read counts and tube totals the + count likelihood needs. It also carries each row's true ``theta``, + ``dk_geno``, ``activity``, ``k_pre``, ``k_sel`` and ``ln_cfu_0``. +``tfs_sim_library.csv`` + Every genotype in the library and its origin. +``tfs_sim_parameters.csv`` + Ground-truth per-genotype parameters (the theta model's parameters, + ``dk_geno``, activity). For the thermodynamic models these are the + per-genotype equilibrium constants, such as ``ln_K_op``, ``ln_K_HL`` + and ``ln_K_E``. +``tfs_sim_genotype_theta.csv`` + Ground-truth *θ* per genotype and titrant concentration. + ``tfs-summarize-fit`` finds this file and uses it as the reference for + out-of-sample *θ*. +``tfs_sim_growth_parameters.csv`` + Ground truth for each condition's growth parameters, keyed by + ``condition_rep``. +``tfs_sim_transformation_lam.csv`` + The configured congression rate, the ground truth for the fit's ``lam``. +``tfs_sim_input-config.yaml`` + The config as run, with ``--seed`` applied. +``tfs_sim_provenance.json`` + Version, git commit and command line of the run. + +The thermodynamic theta models also write ``tfs_sim_mutation_parameters.csv``, +one row per mutation with its true ``d_*`` effects and, for PddG and PnnC, +its true ``ddG_{structure}`` values. When the library has multi-mutants they +write ``tfs_sim_pair_parameters.csv`` as well, one row per mutation pair with +its true ``epi_*`` terms. The column names match the tables +``tfs-extract-params`` writes, so ``tfs-summarize-fit`` can score them. The +``ddG_{structure}`` columns follow the model's structure names, whatever order +the ΔΔG CSV or structure ensemble lists them in. PddG and PnnC also write +``tfs_sim_ddG_sigma.csv``, the true σ_s of each structure (``structure``, +``ddG_sigma``; see ``ddG_truth_sd`` above). + +The optional blocks add ``tfs_sim_binding.csv`` (and, with +``library_binding``, ``tfs_sim_library_binding.csv`` listing the in-library +genotypes chosen), ``tfs_sim_presplit.csv``, ``tfs_sim_base_growth.csv`` +with ``tfs_sim_k_ref.csv``, and ``tfs_sim_od600.csv``. + +The raw experiment +^^^^^^^^^^^^^^^^^^ + +``tfs-simulate`` also writes the experiment the way a lab hands it to the +pipeline, so a simulation is processed by exactly the commands real data go +through (see :doc:`process-raw`): + +``tfs_sim_counts/counts_tube0001.csv``, ... + One file per sequenced tube in ``tfs-process-fastq``'s format: every + library genotype with its read count, zeros included, and the + ``__unknown__`` row (0; the simulator assigns every read). Tubes are + named with fixed-width ids, so each name matches exactly one file. +``tfs_sim_tubes.csv`` + The tube table: ``sample``, ``library`` and the design columns, with no + ground truth. Without an ``od600`` block it also carries each tube's + total (``sample_cfu``, ``sample_cfu_std``). +``tfs_sim_tube_od600.csv``, ``tfs_sim_od600_calibration.yaml`` + With an ``od600`` block: one OD600 reading per tube and a copy of the + calibration. A tube read below the detection threshold is left out of + the tube table, as a lab would drop it, and the dropped tubes are + printed. ``tfs_sim_growth.csv`` leaves out the same tubes. + +``tfs-simulate`` prints the ``tfs-process-counts`` command that processes +these files. Its output matches ``tfs_sim_growth.csv`` row for row: the same +counts, tube totals and *ln_cfu*. The ln_cfu values differ slightly only +where a library genotype never appears in any tube, because a real counts +file lists it and its pseudocount enters the denominator. ``--no_write_raw`` +skips the raw files, which on a full-size library are one file per tube +with every genotype. ``--no_write_growth`` skips ``tfs_sim_growth.csv``, +which ``tfs-process-counts`` rebuilds from the raw files anyway. The +presplit table is still written directly. + +A simulation the size of a real screen (a few hundred thousand genotypes +in about a hundred tubes) is large: tens of GB of memory at its peak, an +hour or more of computing, and several GB of output, most of it the growth +table and the count files. + +Empirical phenotypes +-------------------- + +Prior-predictive phenotypes test whether a fit recovers the truth when the +truth looks like the model's prior. To test it on phenotypes shaped like a +real library, fit the real data to a phenotype-generating distribution and +resample the simulated library from it. The ground truth stays known. + +``tfs-build-empirical`` builds that distribution from a real growth table. +It configures a ``hill_geno`` model with linear growth, runs the pre-fit to +calibrate each condition's *k* and *m*, fits each genotype's growth curves +separately by maximum likelihood (Stage 1), then fits a multivariate normal +to those estimates with their estimation noise deconvolved (Stage 2): .. code-block:: bash - tfs-simulate [options] + tfs-build-empirical growth.csv --binding_df binding.csv \ + --library_config library_config.yaml --seed 42 --out_prefix emp -**Positional arguments:** +It writes ``emp_phenotype_model.json``, the distribution, and +``emp_stage1_fits.csv``, the per-genotype fits, plus the configure and +pre-fit files under ``emp_configure_*`` and ``emp_prefit_*``. The pre-fit is +the slow step. Once a calibration exists, pass it with +``--growth_calibration_file`` (the pre-fit priors CSV, or a CSV with +``condition_rep``, ``growth_k`` and ``growth_m``) to skip configuring and +pre-fitting. ``--binding_df``, ``--library_config`` and ``--seed`` are then +not needed. ``--num_workers -1`` runs the Stage 1 fits in parallel. -* ``config_file``: Path to the simulate YAML configuration file. -* ``output_dir``: Directory to write output files (created if absent). +The Stage 1 fits do not correct for congression, so the distribution carries +its small bias (about 0.05 ln units in the measured regime). -**Optional arguments:** +To simulate from the distribution, set two keys in an ordinary simulate +config: -* ``--output_prefix``: Prefix for all output filenames (default: ``tfscreen_``). -* ``--num_replicates``: Number of independent experimental replicates to - simulate (default: 2). +.. code-block:: yaml -**Outputs** (all written to ``output_dir``): + phenotype_source: empirical + empirical: + phenotype_model: /abs/path/to/emp_phenotype_model.json -* ``{prefix}library.csv`` — all genotypes in the library. -* ``{prefix}phenotype.csv`` — ground-truth growth rates per genotype. -* ``{prefix}genotype_theta.csv`` — ground-truth *θ* per genotype and - titrant concentration. -* ``{prefix}growth.csv`` — analysis-ready ln(CFU) data (same format as - ``tfs-process-counts`` output). -* ``{prefix}binding.csv`` — simulated binding curve data (only written when - ``binding_data`` is present in the config). +Every genotype then gets a resampled Hill curve and ``dk_geno``; wild type +keeps its own Stage 1 fit with ``dk_geno`` 0. ``theta_component`` is forced +to ``hill_geno``, activity is forced to 1, and ``theta_priors`` and +``theta_sim_priors`` are ignored. The ``growth`` block must hold the same +calibrated *k* and *m* the build used, or the simulated growth will not +match the distribution. ``binding_data`` still chooses which genotypes are +measured, and their curves come from the resampled phenotypes. The +:download:`empirical example config <../../examples/simulate-empirical/simulate_config.yaml>` +shows a complete file. -tfs-setup-sim-grid ------------------- +Prior-predictive datasets +------------------------- -Creates a directory grid for sweeping simulate parameters (see -:doc:`grid` for a full description of the grid format). Each subdirectory -receives its own ``tfs_sim_config.yaml`` derived from a base config, plus -an optional rendered shell script. +``tfs-sample-prior`` draws synthetic datasets from a configured model's own +prior rather than from the simulator. It takes the YAML written by +``tfs-configure-model``: .. code-block:: bash - tfs-setup-sim-grid simulate_grid.yaml --out_prefix my_sim_grid + tfs-sample-prior tfs_configure_config.yaml --num_datasets 10 --seed 0 --out_prefix tfs_prior -An :download:`annotated example grid YAML <../../examples/simulate/simulate_grid.yaml>` -is provided in ``examples/simulate/``. After running the command, launch all -simulations with: +Each dataset gives ``tfs_prior_NNN_growth.csv``, the configured growth table +with ``ln_cfu`` replaced by a prior draw, and +``tfs_prior_NNN_ground_truth.h5``, the latent values that generated it in +the format of a ``tfs-sample-posterior`` file. Observation noise from +``ln_cfu_std`` is added unless ``--no_noise`` is given. Fitting these +datasets checks the inference against the model it assumes, the basis of +simulation-based calibration. ``tfs-simulate`` is the stronger test, since +its physics is not the fit's model. + +Simulation grids +---------------- + +``tfs-setup-sim-grid`` sets up a sweep of simulate settings, one directory +per combination, each with its own ``tfs_sim_config.yaml`` and a rendered +run script: .. code-block:: bash - for d in my_sim_grid/*/; do - cd "$d" && bash run.sh && cd - - done + tfs-setup-sim-grid simulate_grid.yaml --out_dir my_sim_grid + +The :download:`example grid YAML <../../examples/simulate/simulate_grid.yaml>` +and its ``run.sh`` template are in ``examples/simulate/``. :doc:`grid` +describes the grid format, and ``tfs-summarize-calibration`` pools the +posterior calibration across a finished simulate-and-fit grid. diff --git a/docs/source/summarize-fit.rst b/docs/source/summarize-fit.rst index 25315f70..7cafdd86 100644 --- a/docs/source/summarize-fit.rst +++ b/docs/source/summarize-fit.rst @@ -2,58 +2,84 @@ tfs-summarize-fit ================= -``tfs-summarize-fit`` collects the outputs of a completed model-fitting run, -computes prediction quality statistics, and writes a set of diagnostic plots -and tables to a ``summary/`` subdirectory. +``tfs-summarize-fit`` collects the outputs of a finished fit, computes +prediction statistics and writes diagnostic plots and tables. It is the quick +look at a run, and the place where simulated runs are scored against their +ground truth. -.. note:: +.. code-block:: bash - **CSV files are the authoritative outputs.** Every PDF written by this - command has a matching CSV containing the exact data used to generate it. - Use the CSVs for downstream quantitative analysis; treat the PDFs as - human-readable summaries. + tfs-summarize-fit out/ -Usage ------ +The one positional argument is the run directory. Outputs go to +``{run_dir}/summary/`` with the prefix ``tfs_summarize`` unless +``--out_prefix`` says otherwise. ``--ref_theta_file`` names a theta reference +table; see below. The full flag list is in :doc:`cli`. -.. code-block:: bash +Every PDF has a matching CSV holding the exact data behind it. Use the CSVs +for any quantitative analysis and treat the PDFs as summaries. + +What the command reads +---------------------- + +The command finds its inputs by file-name suffix inside ``run_dir``. When +several files match, it warns and uses the first in alphabetical order. Each +missing input switches off only the outputs that need it. + +``*_config.yaml`` + The model config from ``tfs-configure-model``. Other YAML files in the + directory, such as a simulate config, are ignored. The config also points + to the binding data, used as training theta, and to the guesses CSV. + +``*_pred_theta.csv`` + Theta predictions from ``tfs-predict-theta``. - tfs-summarize-fit [options] +``*_pred_growth.csv`` + Growth predictions from ``tfs-predict-growth``. Optional. -**Positional arguments:** +``*_losses.txt`` + The loss history from ``tfs-fit-model``. The pre-MAP and pre-fit loss + files (``*_premap_losses.txt``, ``*_prefit_losses.txt``) are skipped. + Optional. -* ``run_dir`` — directory containing the completed model-fit outputs (i.e. - the ``out/`` directory produced by ``run.sh``). +``*_posterior.h5`` or ``*_params.npz`` + The source for the growth trajectories, with the posterior preferred. + ``tfs-sample-posterior`` writes ``{out_prefix}.h5``, so the default name + ``tfs_posterior.h5`` matches but a custom prefix such as ``run1.h5`` + does not. The MAP ``*_params.npz`` from ``tfs-fit-model`` is the + fallback. Optional. -**Optional arguments:** +``*_params_*.csv`` + Parameter tables from ``tfs-extract-params``. Compared to truth on + simulated runs. Optional. -* ``--ref_theta_file`` — CSV with columns ``genotype``, ``titrant_name``, - ``titrant_conc``, and a ``theta_obs`` or ``theta`` column containing - known θ values for evaluating out-of-sample predictions. When omitted, - the script automatically looks for ``*_sim_genotype_theta.csv`` in - ``run_dir`` (the file written by ``tfs-simulate``). -* ``--out_prefix`` — prefix for all output files. Defaults to - ``{run_dir}/summary/tfs_summarize``. +Simulated runs also have ground truth from ``tfs-simulate``: -What the script needs ---------------------- +``*_sim_genotype_theta.csv`` + True theta for every genotype and titrant point. Used as the theta test + reference unless ``--ref_theta_file`` names another table. A reference + table needs ``genotype``, ``titrant_name``, ``titrant_conc`` and a + ``theta_obs`` or ``theta`` column. -The script scans ``run_dir`` for: +``*_sim_parameters.csv`` + True per-genotype parameters. -* ``*_config.yaml`` — model configuration (required). -* ``*_pred_theta.csv`` — theta predictions from ``tfs-predict-theta`` (required). -* ``*_sim_genotype_theta.csv`` — ground-truth θ from ``tfs-simulate`` - (used for test statistics; absent for real-data runs). -* ``*_pred_growth.csv`` — growth predictions from ``tfs-predict-growth`` - (optional; enables growth statistics and plots). -* ``*_losses.txt`` — training loss history from ``tfs-fit-model`` - (optional; enables the loss-curve plot). -* ``*_params_*.csv`` — parameter summaries from ``tfs-extract-params`` - (optional; enables parameter-recovery plots when ``*_sim_parameters.csv`` - is also present). +``*_sim_mutation_parameters.csv`` and ``*_sim_pair_parameters.csv`` + True per-mutation and per-pair parameters of the thermodynamic models. -Output file reference ---------------------- +``*_sim_growth_parameters.csv`` + True per-condition growth parameters. + +``*_sim_k_ref.csv`` + The true base growth rate ``k_ref``. + +``*_sim_transformation_lam.csv`` + The true congression rate ``lam`` of the ``mixture`` transformation. + +Output files +------------ + +All names below use the default prefix ``tfs_summarize``. .. list-table:: :header-rows: 1 @@ -62,273 +88,382 @@ Output file reference * - File - Contents * - ``tfs_summarize_fit_summary.json`` - - Nested statistics dict (see :ref:`fit-summary-json`). - * - ``tfs_summarize_theta_corr.pdf`` / ``.csv`` - - Two-panel θ correlation: training (left) and test (right). + - Statistics and run metadata (see :ref:`fit-summary-json`). + * - ``tfs_summarize_theta_corr.pdf`` + - Two-panel theta correlation: training on the left, test on the right. * - ``tfs_summarize_theta_corr_training.csv`` - - Joined (ref, predicted) θ pairs for training genotypes. + - Training theta predictions joined to the binding observations, which + are in the ``ref`` column. * - ``tfs_summarize_theta_corr_test.csv`` - - Same for test genotypes. + - Theta predictions joined to the reference table, which is in the + ``ref`` column. * - ``tfs_summarize_growth_corr.pdf`` / ``.csv`` - - Observed vs predicted ln(CFU) across all training points. + - Observed against predicted ln(CFU). The CSV is a copy of + ``*_pred_growth.csv`` with ``ln_cfu`` and ``ln_cfu_std`` renamed to + ``ref`` and ``ref_std``. * - ``tfs_summarize_{genotype}_theta_fits.pdf`` / ``.csv`` - - Per-genotype θ curve overlaid with binding observations. + - Predicted theta curve over the binding observations, one pair per + binding genotype. * - ``tfs_summarize_{genotype}_trajectory.pdf`` / ``.csv`` - - Per-genotype predicted ln(CFU) vs time across all conditions. + - Predicted ln(CFU) against time in every condition. * - ``tfs_summarize_losses.pdf`` - - Training loss curve. - * - ``tfs_summarize_theta_training_calibration.pdf`` - - Calibration plots for training θ (PIT histogram + coverage curve). - * - ``tfs_summarize_theta_test_calibration.pdf`` - - Calibration plots for test θ. - * - ``tfs_summarize_growth_calibration.pdf`` - - Calibration plots for growth predictions. + - The loss history. + * - ``tfs_summarize_{name}_calibration.pdf``, ``_pit.csv``, + ``_calibration_curve.csv`` + - Interval calibration for ``theta_training``, ``theta_test``, + ``growth`` and each ``params_*`` table with a reference. * - ``tfs_summarize_params_{name}.pdf`` / ``.csv`` - - Per-parameter scatter of simulated vs inferred values (simulated - data only). - * - ``tfs_summarize_params_{name}_calibration.pdf`` - - Calibration plots for each inferred parameter (simulated data only). + - A parameter table with the true value added as ``ref``. Simulated + runs only. + * - ``tfs_summarize_tube_offsets.pdf`` / ``.csv``, + ``tfs_summarize_tube_offset_trends.csv`` + - The per-tube sample offsets in prior SDs and their trends with titrant + and time, one row per condition (see `Tube offsets`_). Fits with tube + offsets only. + +Slashes and spaces in genotype names become underscores in file names, so +``M42I/K84L`` gives ``tfs_summarize_M42I_K84L_trajectory.pdf``. + +The theta fit plots need binding data with a ``theta_std`` column. The +trajectory plots need growth data and a posterior or params file. With binding +data, trajectories are drawn for the binding genotypes. Without it, they are +drawn for wt, the spiked genotypes and 10 other genotypes picked at random +with a fixed seed, so a rerun draws the same ones. + +Calibration needs at least two ``q`` columns. A MAP checkpoint passed +to the predict commands gives only ``q0.5``, so its predictions get +correlation statistics but no calibration outputs. + +Relative-X fits +--------------- + +A ``hill_relative`` fit predicts the wt-relative growth variable X rather +than theta (see :doc:`model`). For such a fit, ``metadata.theta_scale`` in the +JSON is ``X``; it is ``theta`` otherwise. Simulated truth is mapped onto the +X scale before comparison. The mapping uses the gauge concentrations recorded +in the config, wt's true theta at those concentrations from the reference +table, and each genotype's simulated activity from ``*_sim_parameters.csv``. +The true ``growth_k`` and ``growth_m`` are mapped the same way: k becomes +wt's growth at the high gauge concentration and m its change between the two. +That mapping needs a single titrant. With several, the growth truth is left +blank. When the truth cannot be mapped, for example because the reference has +no wt value at a gauge concentration, the theta test comparison is skipped +with a warning rather than made on the wrong scale. Reading the outputs ------------------- .. _fit-summary-json: -fit_summary.json — at a glance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The summary JSON +~~~~~~~~~~~~~~~~ -The JSON file gives a rapid numerical overview of the run. Open it in any -text editor or load it with ``json.load``. +``tfs_summarize_fit_summary.json`` has four top-level keys. ``metadata`` +describes the run. ``theta`` holds ``training`` and ``test`` statistics, and +``growth`` holds ``training`` statistics. ``tube_offsets`` holds the +tube-offset summary (see `Tube offsets`_). A block that could not be +computed, or a fit without tube offsets, is ``null``. The example below, with illustrative values, is a simulated +growth-only run: there is no binding data, so ``theta.training`` is null. .. code-block:: json { "metadata": { - "n_parameters": 5005, - "n_theta_training_points": 32, + "run_dir": "/home/user/runs/run_0001/out", + "ref_theta_file": "/home/user/runs/run_0001/out/tfs_sim_genotype_theta.csv", + "timestamp": "2026-10-01T14:02:11.512340", + "n_parameters": 412, + "n_theta_training_points": null, "n_theta_test_points": 3752, "n_growth_training_points": 54180, - "final_loss": 7281255.0 + "final_loss": 251873.4, + "theta_scale": "X" }, "theta": { - "training": { "pearson_r": 0.999, "r_squared": 0.998, "rmse": 0.016 }, - "test": { "pearson_r": 0.936, "r_squared": 0.876, "rmse": 0.140 } + "training": null, + "test": { + "pct_success": 1.0, + "rmse": 0.071, + "normalized_rmse": 0.058, + "pearson_r": 0.968, + "spearman_r": 0.951, + "r_squared": 0.937, + "mean_error": -0.004, + "residual_corr": -0.12, + "residual_corr_p_value": 1.3e-13, + "bp_p_value": 0.002 + } }, "growth": { - "training": { "pearson_r": 0.981, "r_squared": 0.963, "rmse": 0.577 } - } + "training": { + "pct_success": 1.0, + "rmse": 0.41, + "normalized_rmse": 0.031, + "pearson_r": 0.991, + "spearman_r": 0.987, + "r_squared": 0.982, + "mean_error": 0.002, + "residual_corr": -0.05, + "residual_corr_p_value": 1.1e-31, + "bp_p_value": 0.0 + } + }, + "tube_offsets": null } -Key fields: - -* ``n_parameters`` — total number of free parameters (from the guesses CSV). - Compare to ``n_growth_training_points`` to check for over-parameterisation. -* ``final_loss`` — the converged SVI ELBO (negative; more negative = better). - Use this to compare runs with identical data but different components or seeds. -* ``theta.training`` — statistics comparing the model's θ predictions against - the direct **binding observations** used as training data. These will - always be near-perfect because the model is conditioned on these values. -* ``theta.test`` — statistics comparing θ predictions against the full - ground-truth θ grid from ``tfs-simulate``. This is the real generalization - diagnostic: the vast majority of test points were never directly observed. -* ``growth.training`` — statistics comparing predicted ln(CFU) against the - observed growth data. - -Useful derived checks: - -* ``theta.test.r_squared > 0.85`` is a reasonable threshold for a well-fitting - model on a library of this size and noise level. -* A large gap between ``theta.training.rmse`` and ``theta.test.rmse`` signals - that the model fits the spiked genotypes well but struggles to generalize - to the broader library. -* ``residual_corr`` and its p-value test for autocorrelation in the residuals. - A significant value indicates systematic structure the model is not capturing. - -Training-loss curve (tfs_summarize_losses.pdf) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The loss curve shows the SVI ELBO at each recorded epoch. A healthy run -shows a smooth decrease that levels off to a plateau; a run that has not -converged will still be declining at the final epoch. If the loss spikes -upward mid-run, the learning rate or the prior scales may need adjustment. - -Theta correlation (tfs_summarize_theta_corr.pdf) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The metadata keys: + +* ``run_dir`` and ``ref_theta_file`` are the resolved absolute paths. + ``ref_theta_file`` is null when no reference was found. +* ``timestamp`` is when the summary ran. +* ``n_parameters`` is the number of rows in the config's guesses CSV, a rough + size of the model. +* ``n_theta_training_points``, ``n_theta_test_points`` and + ``n_growth_training_points`` count the points behind each statistics block. +* ``final_loss`` is the last entry of ``*_losses.txt``. Each entry is the + median loss over one convergence window, not a single step. The loss is the + negative ELBO for SVI and the negative log joint density for MAP. Lower is + better, and it is usually positive. Compare it only between runs on the same + data and model. +* ``theta_scale`` is ``X`` for a relative fit and ``theta`` otherwise. + +Each statistics block compares the ``q0.5`` prediction with its reference: + +* ``pearson_r``, ``spearman_r`` and ``r_squared`` measure agreement. + Spearman tests whether genotypes are ranked correctly. +* ``rmse`` is the root mean squared error. ``normalized_rmse`` divides it by + the 2.5 to 97.5 percentile range of the reference, so it reads as error + relative to the signal. +* ``mean_error`` is the average of prediction minus reference, the bias. +* ``residual_corr`` and ``residual_corr_p_value`` test whether the error + depends on the true value. A significant correlation means systematic + structure, often shrinkage of the extremes toward the middle. +* ``bp_p_value`` is the Breusch-Pagan test. A small value means the error + variance changes with the true value. +* ``pct_success`` is the fraction of predictions that are not NaN. +* Coverage of the reference by the posterior intervals is in the + calibration outputs below, not in these blocks. + +The three blocks mean different things. ``theta.training`` compares +predictions with the binding observations the model was fit to, so it should +be close to perfect. A poor value points to a data-loading or configuration +problem. ``theta.test`` compares predictions with the full reference grid, +which on a simulated run covers every genotype at every concentration, nearly +all of them never measured directly. That is the real test of the fit. A large +gap between training and test error means the model fits the anchors but does +not carry that accuracy to the library. ``growth.training`` compares +predicted with observed ln(CFU) for every observed point. + +Tube offsets +~~~~~~~~~~~~ + +A ``level`` tube offset is meant to absorb tube noise: an error in a tube's +total or its composition, independent from tube to tube and about the size of +the OD600 scatter. On the dev data the MAP's better optimum used the offsets +for something else. Within each selection condition they followed IPTG, from ++2 to -4 ln units across 0 to 1 mM, and stayed constant over time, carrying +growth the model could not express (see :doc:`fitting`, "Staged MAP"). This +check makes that visible. + +It reads ``*_sample_offset_offset.csv`` (``*_sample_offset_delta_k.csv`` for +``normal`` offsets) from ``tfs-extract-params``. The prior SD comes from +``*_sample_offset_sigma.csv`` when the SD was learned, or from +``sigma_fixed`` in the priors file when it was held. + +* ``tfs_summarize_tube_offsets.csv`` is the offsets table with ``offset`` + (the ``q0.5``) and ``z``, the offset in prior SDs. +* ``tfs_summarize_tube_offset_trends.csv`` has one row per condition + (``library``, ``condition_pre``, ``condition_sel``, ``titrant_name``): + the mean and SD of the offsets, in ln units and in prior SDs; the Spearman + correlation of the offset with titrant concentration (``rho_titrant``, + ``p_titrant``) and with selection time (``rho_time``, ``p_time``); + ``r2_titrant``, the share of the offsets' variance explained by the mean + at each concentration, which catches a non-monotone pattern; and + Benjamini-Hochberg q values over every condition and both trends. + ``structured`` is true when either q is below 0.05. +* ``tfs_summarize_tube_offsets.pdf`` plots the offsets against titrant, one + panel per selection condition, colored by time. +* The JSON's ``tube_offsets`` block holds ``n_tubes``, ``sigma`` and its + source, the offsets' ``sd``, ``sd_over_sigma``, the median, 95th + percentile and maximum of ``|z|``, the ``range``, and ``structured`` with + the list of structured conditions. + +Noise offsets have ``sd_over_sigma`` near 1 and no structured condition. A +structured fit is not wrong by construction, since a real tube effect can +trend with IPTG, but its offsets are carrying signal the growth model +leaves out, and its *k*, *m* and curves should not be read until that is +understood. + +Loss history +~~~~~~~~~~~~ + +``tfs_summarize_losses.pdf`` plots each window's median loss against epoch. +A healthy run falls and levels off. A run still falling at the end did not +converge. ``*_convergence.csv`` from ``tfs-fit-model`` records why the run stopped; see +:doc:`fitting`. + +Theta correlation +~~~~~~~~~~~~~~~~~ .. figure:: _static/saa_theta_corr.png :alt: Two-panel theta correlation plot :width: 100% - **Left panel** — training θ: the model's posterior-median prediction - vs the direct binding observations used as input. Near-perfect agreement - is expected and is not by itself a measure of model quality. **Right - panel** — test θ: predictions vs the full ground-truth θ grid from - simulation, covering all library genotypes at all concentrations. Points - far from the diagonal indicate genotypes or concentrations where the - mutation-additivity assumption breaks down. - -The two panels serve different purposes: + Left: training theta, the posterior median against the binding + observations used as input. Right: test theta, predictions against the + simulated ground truth for every library genotype at every concentration. -* **Training panel** confirms that the model correctly ingest the binding - data — any large deviation here points to a data-loading or configuration - error. -* **Test panel** is the generalization diagnostic. The model must predict θ - for thousands of genotypes that were never directly measured; it does so by - combining per-mutation effects inferred from the spiked controls and the - growth data. Scatter around the diagonal reflects the irreducible noise in - that extrapolation. +The training panel confirms that the model took in the binding data. The test +panel is the generalization check. The model must predict theta for thousands +of genotypes never measured directly, so scatter around the diagonal reflects +how well growth alone pins each genotype down. Points far from the diagonal +mark genotypes or concentrations the model cannot resolve. On a relative fit +both axes are on the X scale. -Per-genotype theta fits (tfs_summarize_{genotype}_theta_fits.pdf) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Per-genotype theta fits +~~~~~~~~~~~~~~~~~~~~~~~ .. figure:: _static/saa_wt_theta_fits.png :alt: Wild-type theta fit :width: 60% - Wild-type θ curve: posterior-median prediction (line) overlaid with the - binding observations (circles). The x-axis is on a log scale. A sigmoid - transition from high θ at low [IPTG] to low θ at high [IPTG] is the - expected shape for a repressor inactivated by its inducer. + Wild-type theta: the posterior median prediction as a line over the binding + observations as points, on a log concentration axis. A repressor released + by its inducer goes from high theta at low IPTG to low theta at high IPTG. -One plot is produced per genotype that appears in both the binding data and -the theta predictions. For spiked genotypes with measured binding curves -these plots confirm that the Hill model adequately describes the shape of the -induction curve. +One plot is drawn per genotype found in both the binding data and the theta +predictions. These plots check that the theta model describes the shape of +each measured induction curve. -Growth trajectories (tfs_summarize_{genotype}_trajectory.pdf) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Growth trajectories +~~~~~~~~~~~~~~~~~~~ -The trajectory plots show predicted ln(CFU) versus time for every growth -condition, with the observed data points (two replicates) overlaid on the -posterior-median prediction and its 95% credible interval. +The trajectory plots show predicted ln(CFU) against time for every condition, +with the posterior median, its 95% interval and the observed points. Each +genotype gets one page with one panel per condition. A dashed vertical line +marks the switch from pre-growth to selection. -One page is produced per genotype; each page contains one panel per -condition (library × selection × titrant concentration). The dashed -vertical line marks the transition from pre-growth to selection medium. +Things to look for: -Typical patterns to look for: - -* **Pre-growth panels** (``kanR-kan``, ``pheS-4CP`` without selection): - all genotypes should grow at similar rates; large deviations suggest a - problematic ``dk_geno`` estimate. -* **Selection panels at extreme IPTG**: the two markers should respond in - opposite directions as θ moves from ~1 to ~0 with increasing [IPTG]. -* **Replicate agreement**: the two colored lines should be close together; - large replicate-to-replicate variation indicates high tube noise or a +* In pre-growth, genotypes should grow at similar rates. A large deviation + suggests a poor ``dk_geno`` estimate. +* At the extremes of IPTG the two markers should move in opposite directions + as theta goes from about 1 to about 0. +* Replicates should agree. Large disagreement points to tube noise or a confounded sample. -Growth correlation (tfs_summarize_growth_corr.pdf) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Growth correlation +~~~~~~~~~~~~~~~~~~ .. figure:: _static/saa_growth_corr.png :alt: Growth prediction correlation :width: 65% - Observed vs predicted ln(CFU) across all genotypes, conditions, and - time-points in the training data (54,180 points in the example run). - Points cluster tightly along the diagonal except at low ln(CFU) values - (~3–7), where sequencing noise dominates and the model's predictions - spread somewhat. + Observed against predicted ln(CFU) for every genotype, condition and time + point in the training data. Points spread more at low ln(CFU), where + sequencing noise dominates. -A tight cluster along the diagonal confirms that the growth model -accurately reproduces the data. Systematic vertical or horizontal bands -point to conditions or time-points where the model consistently over- or -under-predicts; these are usually caused by incorrect ``{m, b}`` priors for -those conditions or by tube-noise events in specific samples. +A tight cluster along the diagonal means the growth model reproduces the data. +Vertical or horizontal bands mark conditions or time points the model over- or +under-predicts throughout. Those usually come from poor per-condition growth +priors or from bad tubes. .. _calibration-plots: -Calibration plots (tfs_summarize_*_calibration.pdf) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Calibration +~~~~~~~~~~~ .. figure:: _static/saa_growth_calibration.png :alt: Growth calibration plots :width: 100% - Growth-prediction calibration. **Left** — PIT histogram; the dashed - red line shows the ideal uniform distribution. **Right** — calibration - curve; the dashed black line is perfect calibration. - -Calibration plots assess whether the model's **posterior uncertainty -intervals** are appropriately sized — neither too narrow (overconfident) -nor too wide (underconfident). - -**PIT histogram** (left panel) - -The Probability Integral Transform (PIT) value for each observation is the -fraction of the posterior predictive distribution that falls below the -observed value. For a perfectly calibrated model, PIT values are -uniformly distributed on [0, 1] and the histogram should be flat. - -* **U-shaped histogram** (spikes near 0 and 1, flat middle): many - observations fall outside the predicted intervals → posterior is - **overconfident** (intervals too narrow). -* **Hump-shaped histogram** (peak in the middle, low edges): observations - cluster inside the predicted intervals → posterior is **underconfident** - (intervals too wide). - -**Calibration curve** (right panel) - -For each nominal coverage level *α* (x-axis), the calibration curve shows -the empirical fraction of observations that fall within the model's *α* -credible interval (y-axis). - -* **Below the diagonal**: empirical coverage < nominal → **overconfident**. -* **Above the diagonal**: empirical coverage > nominal → **underconfident**. -* **On the diagonal**: perfectly calibrated. - -The example growth-calibration plot shows U-shaped PIT and a curve below -the diagonal. This slight overconfidence in growth predictions is typical -of SVI: the mean-field variational approximation tends to underestimate -posterior variance. It does not indicate a model misspecification; the -point predictions (correlation, R²) remain accurate. - -Similar calibration plots are produced for theta (training and test) and for -each parameter group when ground-truth values are available. - -Parameter recovery (tfs_summarize_params_{name}.pdf) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Growth-prediction calibration. Left: PIT histogram, with the dashed red + line at the ideal uniform. Right: calibration curve, with the dashed black + line at perfect calibration. + +Calibration asks whether the posterior intervals have the right width. The PIT +value of an observation is the fraction of its predictive distribution below +the observed value, interpolated from the ``q`` columns. For a +calibrated model the PIT values are uniform on [0, 1] and the histogram is +flat. A U shape, with spikes near 0 and 1, means many observations fall +outside their intervals: the posterior is overconfident. A hump in the middle +means the intervals are too wide. + +The calibration curve plots, for each nominal coverage level, the fraction of +observations inside that interval. Below the diagonal is overconfident and +above is underconfident. ``_pit.csv`` holds the PIT values (``true_val``, +``pit``) and ``_calibration_curve.csv`` the curve (``nominal``, +``empirical``). + +The example shows a U-shaped PIT and a curve below the diagonal: the growth +intervals are somewhat too narrow, while the point predictions stay accurate. +Variational guides tend to understate posterior variance, so check calibration +before trusting interval widths. Theta calibration on a simulated run is the +more important check, since theta is what downstream analysis uses. + +Parameter recovery +~~~~~~~~~~~~~~~~~~ .. figure:: _static/saa_params_log_hill_K.png :alt: log_hill_K parameter recovery :width: 65% - Simulated vs inferred ``log_hill_K`` (log₁₀ of the Hill dissociation - constant) for all library genotypes. Each point is one genotype; the - dashed line is perfect recovery. Good agreement across the dynamic range - confirms that the model correctly identifies which genotypes have shifted - binding affinities. - -Parameter-recovery plots are produced only when the run directory contains -a ``*_sim_parameters.csv`` file (i.e. when running on simulated data). They -compare the posterior-median inferred value for each parameter against the -ground truth used during simulation. - -One plot is produced per parameter group. Versions are produced for: - -* **Direct parameters** — per-genotype values (e.g. ``log_hill_K``, - ``theta_high``, ``theta_low``, ``hill_n``). -* **Diff parameters** (``_d_`` infix) — per-mutation effect sizes relative to - wild-type (e.g. ``d_log_hill_K``, ``d_logit_low``). -* **Epistasis parameters** (``_epi_`` infix) — pairwise epistatic deviations - from additivity for double-mutant genotypes. - -Each parameter group also has a companion calibration plot -(``*_calibration.pdf``) assessing whether the inferred uncertainty correctly -covers the true value. - -Interpreting parameter recovery -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* Tight scatter around the diagonal for all parameters → model recovers the - ground truth; the experimental design has sufficient signal. -* Tight recovery for direct parameters but wide scatter for diff/epi - parameters → the model identifies which genotypes differ from wild-type - but cannot resolve the per-mutation contributions as precisely. -* Systematic bias (all points shifted to one side of the diagonal) → - mis-specified priors, an incorrect model component, or a calibration issue - in the pre-fit step. -* Good recovery for ``log_hill_K`` but poor for ``hill_n`` or ``theta_high`` - → the mid-range of the induction curve is well-constrained but the shape - of the saturation plateaux is not (common when the growth data covers only - a restricted θ range for most genotypes). + Simulated against inferred ``log_hill_K``, the natural log of the Hill + constant, for every library genotype. Each point is one genotype and the + dashed line is perfect recovery. + +Parameter recovery runs only on simulated runs. Each ``*_params_*.csv`` table +from ``tfs-extract-params`` is matched to its truth, written with a ``ref`` +column as ``tfs_summarize_params_{name}.csv``, plotted against ``q0.5`` and +checked for calibration. + +* Per-genotype tables such as ``log_hill_K``, ``hill_n``, ``theta_low``, + ``theta_high`` and ``dk_geno`` take their truth from + ``*_sim_parameters.csv``. Tables on a log or logit scale are matched by + transforming the truth. +* Per-mutation tables (``d_`` prefix) and pair tables (``epi_`` prefix) are + joined to ``*_sim_mutation_parameters.csv`` and + ``*_sim_pair_parameters.csv`` when those files have the column, as they do + for the thermodynamic models. Otherwise, as for ``hill_mut``, a ``d_`` table + is compared with the true value minus wt's and an ``epi_`` table with the + additive epistasis of the true values. +* The per-structure ``ddG_{structure}`` tables of the PddG and PnnC models + are joined to the ``ddG_{structure}`` columns of + ``*_sim_mutation_parameters.csv`` on ``mutation``. Each + ``ddG_prior_mean_{structure}`` table, the structural prior mean, is + compared with the same truth to show how well the prior alone predicts it. + It is a point, so it gets no calibration; read its ``rmse`` and + ``pearson_r`` in ``tfs-summarize-calibration``. A fit without ΔΔG, such as + PK on a PddG simulation, writes no ddG tables. Then one note is printed in + place of a warning per structure, and its ``d_`` tables are scored against + the same truth. In a model without an unfolded state (``U0``) the + per-structure ΔΔG shares one shift across structures that the data do not + determine; the fit takes it from the prior. For those fits the + ``ddG_{structure}`` summary is still written, but with the truth in a + ``ref_unidentified`` column and ``per_state_identified`` set to False, and + it gets no plot or calibration. A note says so. Score U0 fits on their + ``d_`` tables. +* ``params_ddG_sigma``, the per-structure SD of the PddG and PnnC ΔΔG about + their prior means, is joined to ``*_sim_ddG_sigma.csv`` on ``structure``. + It gets a CSV and a calibration check but no plot. +* ``params_growth_k``, ``params_growth_m`` and the other growth-model tables + are joined to ``*_sim_growth_parameters.csv`` on ``condition_rep``. +* ``params_k_ref`` and ``params_lam`` are single values compared with + ``*_sim_k_ref.csv`` and ``*_sim_transformation_lam.csv``. They get a CSV + and a calibration check but no plot. + +A table whose truth cannot be resolved is skipped. A truth column with no +matching parameter table produces a warning. + +How to read recovery: + +* Tight scatter around the diagonal means the design has the signal to + recover the parameter. +* Good recovery of per-genotype values with wide scatter in ``d_`` or + ``epi_`` values means the model sees which genotypes differ but cannot split + the difference into per-mutation parts as precisely. +* A shift of all points to one side points to a misspecified prior, a wrong + component or a problem in the pre-fit calibration. +* Good ``log_hill_K`` with poor ``hill_n`` or ``theta_high`` means the + midpoint of the curve is well constrained and its plateaus are not. That is + common when most genotypes span only part of the theta range. diff --git a/examples/od600/make_example_data.py b/examples/od600/make_example_data.py new file mode 100644 index 00000000..96e43bc1 --- /dev/null +++ b/examples/od600/make_example_data.py @@ -0,0 +1,46 @@ +""" +Write synthetic inputs for tfs-calibrate-od600 (replicates.csv and +plate_counts.csv), to show the formats and exercise the tool. + +The numbers are made up, not any instrument's: a culture with +cfu/mL = 8e8 OD + 2e8 OD^2, a reader whose readings sit on a 0.088 floor +with 1.5% noise, 1:3 dilution series read ten times, and 16 cultures +plated at 1e5 dilution (five 1:10 steps, 0.1 mL spread). Each lab +measures its own calibration; do not use these for real data. + + python make_example_data.py + tfs-calibrate-od600 replicates.csv plate_counts.csv --out_prefix od600_calibration +""" + +import numpy as np +import pandas as pd + +rng = np.random.default_rng(20260927) + + +def cfu_per_mL(od): + return 8e8 * od + 2e8 * od ** 2 + + +# Ten readings of each of six 1:3 dilutions of an OD600 ~0.6 culture. +floor, rel_sd = 0.088, 0.015 +dilution = 1.0 / 3.0 ** np.arange(6) +rows = [] +for d in dilution: + true = floor + 0.52 * d + for _ in range(10): + rows.append({"dilution": d, + "od600": round(true * (1 + rel_sd * rng.normal()), 4)}) +pd.DataFrame(rows).to_csv("replicates.csv", index=False) + +# Sixteen cultures across OD600 0.12-0.6, each read once and plated. +od_true = np.linspace(0.12, 0.6, 16) +od_read = od_true * (1 + rel_sd * rng.normal(size=od_true.size)) +fold, volume = 1e5, 0.1 +colonies = rng.poisson(cfu_per_mL(od_true) * volume / fold) +pd.DataFrame({"od600": np.round(od_read, 4), + "colonies": colonies, + "dilution": fold, + "plated_volume_mL": volume, + "num_dilutions": 5, + "plating_steps": 1}).to_csv("plate_counts.csv", index=False) diff --git a/examples/od600/od600_calibration.yaml b/examples/od600/od600_calibration.yaml new file mode 100644 index 00000000..b3d74b98 --- /dev/null +++ b/examples/od600/od600_calibration.yaml @@ -0,0 +1,33 @@ +# OD600-to-CFU calibration, written by tfs-calibrate-od600. +# cfu/mL = sum_i coefficients[i] * OD600^i. covariance is the +# coefficients' (shared by every tube calibrated with this file); +# reading_rel_sd is one reading's SD as a fraction of the reading. +# Specific to one plate reader, plate, volume and strain. +kind: od600_to_cfu_per_mL +degree: 2 +coefficients: +- -3232113.9094636133 +- 854172076.6876991 +- 41938427.08096051 +covariance: +- - 227467162769893.9 + - -1599191946265586.5 + - 2296356870091127.0 +- - -1599191946265586.5 + - 1.2197351004649222e+16 + - -1.828231333112995e+16 +- - 2296356870091127.0 + - -1.828231333112995e+16 + - 2.857868475371398e+16 +chi2_reduced: 0.39237605605574044 +reading_rel_sd: 0.01886048606352703 +detection_threshold: 0.09292 +calibrated_od600_range: +- 0.1192 +- 0.5997 +num_plate_counts: 16 +pipette_rel_error: 0.02 +source: + replicate_file: replicates.csv + plate_count_file: plate_counts.csv +written: '2026-09-27' diff --git a/examples/od600/plate_counts.csv b/examples/od600/plate_counts.csv new file mode 100644 index 00000000..62835a3f --- /dev/null +++ b/examples/od600/plate_counts.csv @@ -0,0 +1,17 @@ +od600,colonies,dilution,plated_volume_mL,num_dilutions,plating_steps +0.1192,97,100000.0,0.1,5,1 +0.1477,131,100000.0,0.1,5,1 +0.1842,151,100000.0,0.1,5,1 +0.2146,186,100000.0,0.1,5,1 +0.2448,207,100000.0,0.1,5,1 +0.2795,240,100000.0,0.1,5,1 +0.3126,239,100000.0,0.1,5,1 +0.3424,317,100000.0,0.1,5,1 +0.3782,344,100000.0,0.1,5,1 +0.4067,347,100000.0,0.1,5,1 +0.4452,379,100000.0,0.1,5,1 +0.4669,411,100000.0,0.1,5,1 +0.504,438,100000.0,0.1,5,1 +0.5335,483,100000.0,0.1,5,1 +0.5735,479,100000.0,0.1,5,1 +0.5997,527,100000.0,0.1,5,1 diff --git a/examples/od600/replicates.csv b/examples/od600/replicates.csv new file mode 100644 index 00000000..0c9f558f --- /dev/null +++ b/examples/od600/replicates.csv @@ -0,0 +1,61 @@ +dilution,od600 +1.0,0.5893 +1.0,0.6039 +1.0,0.6042 +1.0,0.6119 +1.0,0.6056 +1.0,0.604 +1.0,0.588 +1.0,0.6141 +1.0,0.6065 +1.0,0.6303 +0.3333333333333333,0.2582 +0.3333333333333333,0.2655 +0.3333333333333333,0.2713 +0.3333333333333333,0.2583 +0.3333333333333333,0.2624 +0.3333333333333333,0.2627 +0.3333333333333333,0.2592 +0.3333333333333333,0.2647 +0.3333333333333333,0.2646 +0.3333333333333333,0.2723 +0.1111111111111111,0.1426 +0.1111111111111111,0.1474 +0.1111111111111111,0.1491 +0.1111111111111111,0.1465 +0.1111111111111111,0.1456 +0.1111111111111111,0.1504 +0.1111111111111111,0.144 +0.1111111111111111,0.1456 +0.1111111111111111,0.1467 +0.1111111111111111,0.1442 +0.037037037037037035,0.1074 +0.037037037037037035,0.1042 +0.037037037037037035,0.1085 +0.037037037037037035,0.1075 +0.037037037037037035,0.1044 +0.037037037037037035,0.1078 +0.037037037037037035,0.1085 +0.037037037037037035,0.1063 +0.037037037037037035,0.1064 +0.037037037037037035,0.1086 +0.012345679012345678,0.0931 +0.012345679012345678,0.0972 +0.012345679012345678,0.0975 +0.012345679012345678,0.0942 +0.012345679012345678,0.093 +0.012345679012345678,0.0967 +0.012345679012345678,0.0946 +0.012345679012345678,0.0947 +0.012345679012345678,0.0948 +0.012345679012345678,0.0953 +0.00411522633744856,0.0895 +0.00411522633744856,0.0941 +0.00411522633744856,0.0889 +0.00411522633744856,0.0899 +0.00411522633744856,0.0897 +0.00411522633744856,0.0925 +0.00411522633744856,0.0907 +0.00411522633744856,0.0904 +0.00411522633744856,0.0903 +0.00411522633744856,0.0913 diff --git a/examples/process_raw/README.md b/examples/process_raw/README.md new file mode 100644 index 00000000..01695707 --- /dev/null +++ b/examples/process_raw/README.md @@ -0,0 +1,45 @@ +# Raw-data example + +A small synthetic experiment in the formats `tfs-process-counts` reads. The +numbers are made up. They show the formats, not a realistic screen. + +- `library_config.yaml`: the library description. The same file goes to + `tfs-process-fastq` and to `tfs-configure-model --library_config`. +- `make_example_data.py`: writes the three inputs below from a fixed seed. +- `tube_table.csv`: one row per sequenced tube. Two libraries (kanR and + pheS), each with a selection and a control condition, two IPTG + concentrations and three time points, for 24 tubes. +- `od600.csv`: one OD600 reading per tube. +- `counts/`: one count file per tube, in the format `tfs-process-fastq` + writes, including the `__unknown__` row. + +- `corrections.yaml`: example corrections for `tfs-correct-tubes` (one + swap, one relabel, one exclusion). The tubes and reasons are made up. + +The OD600 calibration is in `../od600/`. To build the growth table: + +```bash +python make_example_data.py +tfs-process-counts tube_table.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file ../od600/od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix growth +``` + +To apply the corrections first and build the growth table from the +corrected tube table: + +```bash +tfs-correct-tubes tube_table.csv corrections.yaml --out_prefix tubes_corrected +tfs-process-counts tubes_corrected.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file ../od600/od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix growth +``` + +`growth.csv` is ready for +`tfs-configure-model --growth_df growth.csv --library_config library_config.yaml`. + +See `docs/source/process-raw.rst` for the formats and the steps. diff --git a/examples/process_raw/corrections.yaml b/examples/process_raw/corrections.yaml new file mode 100644 index 00000000..cb17af2c --- /dev/null +++ b/examples/process_raw/corrections.yaml @@ -0,0 +1,34 @@ +# Corrections to tube_table.csv, applied by tfs-correct-tubes. +# +# The tubes and reasons are made up: they show the format. A real +# corrections file lives with the experiment's data, under version control, +# next to the tube table it corrects. + +description: Example corrections to the synthetic tube table. + +corrections: + - id: pheS-t60-iptg-swap + action: swap + tubes: [pheS_pheSp4CP_0mM_t60, pheS_pheSp4CP_1mM_t60] + reason: > + Composition QC: the tube labeled 0 mM IPTG looks like a 1 mM tube and + the one labeled 1 mM like a 0 mM tube. The person who ran the screen + says a swap is probable. + by: A. Researcher + date: 2026-10-15 + evidence: Apparent-IPTG score from a composition QC table. + + - id: kanR-t120-time + action: relabel + tubes: [kanR_kanRpkan_0mM_t120] + set: {t_sel: 125} + reason: The bench log says this tube was pulled at 125 minutes. + by: A. Researcher + date: 2026-10-15 + + - id: kanR-control-t0-failed + action: exclude + tubes: [kanR_kanRmkan_1mM_t0] + reason: Library prep failed; the tube has almost no reads. + by: A. Researcher + date: 2026-10-15 diff --git a/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t0.csv b/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t0.csv new file mode 100644 index 00000000..8a16ec82 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2028 +K84L,700 +M42I/K84L,578 +H29T/L63V,1668 +V30L/L63R,758 +V30Y/L63S,1094 +V30R/L63N,1995 +V30T/L62S,1160 +V30P/L62D,728 +H29V/L62F,1435 +H29N/L63G,1025 +H29N/L63D,926 +V30F/L62F,1738 +V30T/L63R,886 +H29G/L63D,501 +H29C/L62A,958 +H29L/L62C,1238 +V30P/L62S,1779 +L63F,1041 +V30R/L62Y,2698 +H29L/L63V,2607 +H29N/L62F,628 +H29S/L62A,748 +H29Y/L63Y,1403 +H29D/L62C,1242 +H29Y/L63T,4043 +V30H/L63N,838 +H29A/L63S,1811 +V30C/L63R,2020 +V30D/L63S,1628 +L62D,966 +V30D/L62I,1420 +H29P/L62V,3146 +__unknown__,2566 diff --git a/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t120.csv b/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t120.csv new file mode 100644 index 00000000..ce3bcf5e --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1924 +K84L,424 +M42I/K84L,389 +H29T/L63V,1099 +V30L/L63R,659 +V30Y/L63S,242 +V30R/L63N,2668 +V30T/L62S,1992 +V30P/L62D,466 +H29V/L62F,2284 +H29N/L63G,601 +H29N/L63D,451 +V30F/L62F,1624 +V30T/L63R,579 +H29G/L63D,385 +H29C/L62A,1148 +H29L/L62C,786 +V30P/L62S,2750 +L63F,660 +V30R/L62Y,2436 +H29L/L63V,2056 +H29N/L62F,405 +H29S/L62A,515 +H29Y/L63Y,1157 +H29D/L62C,738 +H29Y/L63T,4680 +V30H/L63N,729 +H29A/L63S,2098 +V30C/L63R,3845 +V30D/L63S,1849 +L62D,1098 +V30D/L62I,2137 +H29P/L62V,2659 +__unknown__,2467 diff --git a/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t60.csv b/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t60.csv new file mode 100644 index 00000000..f2e78244 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRmkan_0mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1969 +K84L,548 +M42I/K84L,518 +H29T/L63V,1356 +V30L/L63R,689 +V30Y/L63S,510 +V30R/L63N,2322 +V30T/L62S,1548 +V30P/L62D,581 +H29V/L62F,1836 +H29N/L63G,815 +H29N/L63D,681 +V30F/L62F,1713 +V30T/L63R,719 +H29G/L63D,447 +H29C/L62A,1034 +H29L/L62C,1067 +V30P/L62S,2203 +L63F,815 +V30R/L62Y,2625 +H29L/L63V,2394 +H29N/L62F,536 +H29S/L62A,636 +H29Y/L63Y,1277 +H29D/L62C,1008 +H29Y/L63T,4441 +V30H/L63N,782 +H29A/L63S,2054 +V30C/L63R,2910 +V30D/L63S,1759 +L62D,1009 +V30D/L62I,1801 +H29P/L62V,2931 +__unknown__,2466 diff --git a/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t0.csv b/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t0.csv new file mode 100644 index 00000000..40a21020 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2091 +K84L,738 +M42I/K84L,518 +H29T/L63V,1670 +V30L/L63R,751 +V30Y/L63S,1109 +V30R/L63N,2050 +V30T/L62S,1244 +V30P/L62D,727 +H29V/L62F,1392 +H29N/L63G,1048 +H29N/L63D,938 +V30F/L62F,1799 +V30T/L63R,859 +H29G/L63D,497 +H29C/L62A,896 +H29L/L62C,1219 +V30P/L62S,1776 +L63F,1080 +V30R/L62Y,2687 +H29L/L63V,2447 +H29N/L62F,653 +H29S/L62A,778 +H29Y/L63Y,1332 +H29D/L62C,1289 +H29Y/L63T,4115 +V30H/L63N,803 +H29A/L63S,1839 +V30C/L63R,2096 +V30D/L63S,1728 +L62D,919 +V30D/L62I,1396 +H29P/L62V,2998 +__unknown__,2518 diff --git a/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t120.csv b/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t120.csv new file mode 100644 index 00000000..6c58e939 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2017 +K84L,417 +M42I/K84L,390 +H29T/L63V,1130 +V30L/L63R,668 +V30Y/L63S,262 +V30R/L63N,2666 +V30T/L62S,1956 +V30P/L62D,404 +H29V/L62F,2348 +H29N/L63G,620 +H29N/L63D,395 +V30F/L62F,1664 +V30T/L63R,560 +H29G/L63D,373 +H29C/L62A,1177 +H29L/L62C,805 +V30P/L62S,2758 +L63F,602 +V30R/L62Y,2415 +H29L/L63V,2106 +H29N/L62F,391 +H29S/L62A,515 +H29Y/L63Y,1163 +H29D/L62C,727 +H29Y/L63T,4572 +V30H/L63N,677 +H29A/L63S,2218 +V30C/L63R,3912 +V30D/L63S,1874 +L62D,1089 +V30D/L62I,2182 +H29P/L62V,2579 +__unknown__,2368 diff --git a/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t60.csv b/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t60.csv new file mode 100644 index 00000000..c560c5ba --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRmkan_1mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2099 +K84L,521 +M42I/K84L,465 +H29T/L63V,1336 +V30L/L63R,758 +V30Y/L63S,567 +V30R/L63N,2249 +V30T/L62S,1592 +V30P/L62D,580 +H29V/L62F,1802 +H29N/L63G,797 +H29N/L63D,668 +V30F/L62F,1777 +V30T/L63R,688 +H29G/L63D,468 +H29C/L62A,1011 +H29L/L62C,946 +V30P/L62S,2266 +L63F,836 +V30R/L62Y,2616 +H29L/L63V,2313 +H29N/L62F,539 +H29S/L62A,651 +H29Y/L63Y,1314 +H29D/L62C,990 +H29Y/L63T,4448 +V30H/L63N,830 +H29A/L63S,2026 +V30C/L63R,2834 +V30D/L63S,1732 +L62D,1015 +V30D/L62I,1821 +H29P/L62V,2981 +__unknown__,2464 diff --git a/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t0.csv b/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t0.csv new file mode 100644 index 00000000..20f2bbf1 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2004 +K84L,734 +M42I/K84L,560 +H29T/L63V,1661 +V30L/L63R,753 +V30Y/L63S,1073 +V30R/L63N,2076 +V30T/L62S,1246 +V30P/L62D,733 +H29V/L62F,1343 +H29N/L63G,1046 +H29N/L63D,1006 +V30F/L62F,1779 +V30T/L63R,906 +H29G/L63D,487 +H29C/L62A,964 +H29L/L62C,1270 +V30P/L62S,1789 +L63F,1149 +V30R/L62Y,2664 +H29L/L63V,2466 +H29N/L62F,661 +H29S/L62A,760 +H29Y/L63Y,1311 +H29D/L62C,1218 +H29Y/L63T,4149 +V30H/L63N,763 +H29A/L63S,1759 +V30C/L63R,2067 +V30D/L63S,1730 +L62D,940 +V30D/L62I,1448 +H29P/L62V,2979 +__unknown__,2506 diff --git a/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t120.csv b/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t120.csv new file mode 100644 index 00000000..902407c4 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1974 +K84L,409 +M42I/K84L,372 +H29T/L63V,1136 +V30L/L63R,694 +V30Y/L63S,223 +V30R/L63N,2574 +V30T/L62S,1989 +V30P/L62D,423 +H29V/L62F,2285 +H29N/L63G,670 +H29N/L63D,445 +V30F/L62F,1750 +V30T/L63R,516 +H29G/L63D,367 +H29C/L62A,1129 +H29L/L62C,751 +V30P/L62S,2767 +L63F,624 +V30R/L62Y,2472 +H29L/L63V,2059 +H29N/L62F,419 +H29S/L62A,479 +H29Y/L63Y,1215 +H29D/L62C,724 +H29Y/L63T,4515 +V30H/L63N,719 +H29A/L63S,2134 +V30C/L63R,3873 +V30D/L63S,1838 +L62D,1091 +V30D/L62I,2175 +H29P/L62V,2623 +__unknown__,2566 diff --git a/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t60.csv b/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t60.csv new file mode 100644 index 00000000..d1f45488 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRpkan_0mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2035 +K84L,528 +M42I/K84L,484 +H29T/L63V,1360 +V30L/L63R,720 +V30Y/L63S,501 +V30R/L63N,2441 +V30T/L62S,1559 +V30P/L62D,564 +H29V/L62F,1836 +H29N/L63G,793 +H29N/L63D,697 +V30F/L62F,1709 +V30T/L63R,676 +H29G/L63D,427 +H29C/L62A,1108 +H29L/L62C,1005 +V30P/L62S,2315 +L63F,848 +V30R/L62Y,2542 +H29L/L63V,2274 +H29N/L62F,545 +H29S/L62A,622 +H29Y/L63Y,1212 +H29D/L62C,1010 +H29Y/L63T,4411 +V30H/L63N,733 +H29A/L63S,2081 +V30C/L63R,2890 +V30D/L63S,1845 +L62D,979 +V30D/L62I,1851 +H29P/L62V,2897 +__unknown__,2502 diff --git a/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t0.csv b/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t0.csv new file mode 100644 index 00000000..7bc56422 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2003 +K84L,729 +M42I/K84L,596 +H29T/L63V,1642 +V30L/L63R,783 +V30Y/L63S,1036 +V30R/L63N,2069 +V30T/L62S,1210 +V30P/L62D,724 +H29V/L62F,1434 +H29N/L63G,1017 +H29N/L63D,949 +V30F/L62F,1764 +V30T/L63R,922 +H29G/L63D,463 +H29C/L62A,978 +H29L/L62C,1180 +V30P/L62S,1770 +L63F,1098 +V30R/L62Y,2615 +H29L/L63V,2539 +H29N/L62F,628 +H29S/L62A,818 +H29Y/L63Y,1302 +H29D/L62C,1227 +H29Y/L63T,4176 +V30H/L63N,830 +H29A/L63S,1867 +V30C/L63R,2061 +V30D/L63S,1731 +L62D,886 +V30D/L62I,1481 +H29P/L62V,2984 +__unknown__,2488 diff --git a/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t120.csv b/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t120.csv new file mode 100644 index 00000000..2edda5a6 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1883 +K84L,448 +M42I/K84L,384 +H29T/L63V,1115 +V30L/L63R,666 +V30Y/L63S,242 +V30R/L63N,2639 +V30T/L62S,1829 +V30P/L62D,442 +H29V/L62F,2350 +H29N/L63G,638 +H29N/L63D,476 +V30F/L62F,1687 +V30T/L63R,562 +H29G/L63D,360 +H29C/L62A,1133 +H29L/L62C,727 +V30P/L62S,2725 +L63F,628 +V30R/L62Y,2318 +H29L/L63V,2120 +H29N/L62F,400 +H29S/L62A,495 +H29Y/L63Y,1168 +H29D/L62C,775 +H29Y/L63T,4636 +V30H/L63N,702 +H29A/L63S,2152 +V30C/L63R,3957 +V30D/L63S,1820 +L62D,1143 +V30D/L62I,2149 +H29P/L62V,2664 +__unknown__,2567 diff --git a/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t60.csv b/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t60.csv new file mode 100644 index 00000000..64068367 --- /dev/null +++ b/examples/process_raw/counts/counts_kanR_kanRpkan_1mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2053 +K84L,551 +M42I/K84L,505 +H29T/L63V,1360 +V30L/L63R,684 +V30Y/L63S,481 +V30R/L63N,2328 +V30T/L62S,1599 +V30P/L62D,592 +H29V/L62F,1890 +H29N/L63G,855 +H29N/L63D,642 +V30F/L62F,1713 +V30T/L63R,710 +H29G/L63D,444 +H29C/L62A,995 +H29L/L62C,1033 +V30P/L62S,2227 +L63F,828 +V30R/L62Y,2642 +H29L/L63V,2332 +H29N/L62F,522 +H29S/L62A,620 +H29Y/L63Y,1338 +H29D/L62C,1009 +H29Y/L63T,4346 +V30H/L63N,789 +H29A/L63S,2027 +V30C/L63R,2787 +V30D/L63S,1790 +L62D,1001 +V30D/L62I,1867 +H29P/L62V,2979 +__unknown__,2461 diff --git a/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t0.csv b/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t0.csv new file mode 100644 index 00000000..bafcca5f --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2057 +K84L,733 +M42I/K84L,578 +H29T/L63V,1545 +V30L/L63R,743 +V30Y/L63S,1068 +V30R/L63N,2063 +V30T/L62S,1230 +V30P/L62D,771 +H29V/L62F,1405 +H29N/L63G,992 +H29N/L63D,931 +V30F/L62F,1766 +V30T/L63R,930 +H29G/L63D,503 +H29C/L62A,916 +H29L/L62C,1314 +V30P/L62S,1748 +L63F,1069 +V30R/L62Y,2598 +H29L/L63V,2595 +H29N/L62F,637 +H29S/L62A,804 +H29Y/L63Y,1371 +H29D/L62C,1210 +H29Y/L63T,4168 +V30H/L63N,830 +H29A/L63S,1864 +V30C/L63R,2049 +V30D/L63S,1719 +L62D,913 +V30D/L62I,1430 +H29P/L62V,3017 +__unknown__,2433 diff --git a/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t120.csv b/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t120.csv new file mode 100644 index 00000000..57b7fdbf --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1922 +K84L,416 +M42I/K84L,341 +H29T/L63V,1181 +V30L/L63R,686 +V30Y/L63S,256 +V30R/L63N,2535 +V30T/L62S,1869 +V30P/L62D,432 +H29V/L62F,2377 +H29N/L63G,662 +H29N/L63D,440 +V30F/L62F,1674 +V30T/L63R,577 +H29G/L63D,345 +H29C/L62A,1181 +H29L/L62C,713 +V30P/L62S,2740 +L63F,684 +V30R/L62Y,2408 +H29L/L63V,2102 +H29N/L62F,419 +H29S/L62A,516 +H29Y/L63Y,1187 +H29D/L62C,749 +H29Y/L63T,4462 +V30H/L63N,672 +H29A/L63S,2132 +V30C/L63R,3848 +V30D/L63S,1830 +L62D,1106 +V30D/L62I,2240 +H29P/L62V,2725 +__unknown__,2573 diff --git a/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t60.csv b/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t60.csv new file mode 100644 index 00000000..4351b36d --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSm4CP_0mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2075 +K84L,603 +M42I/K84L,484 +H29T/L63V,1389 +V30L/L63R,725 +V30Y/L63S,504 +V30R/L63N,2402 +V30T/L62S,1545 +V30P/L62D,543 +H29V/L62F,1886 +H29N/L63G,814 +H29N/L63D,660 +V30F/L62F,1767 +V30T/L63R,756 +H29G/L63D,439 +H29C/L62A,1041 +H29L/L62C,978 +V30P/L62S,2245 +L63F,894 +V30R/L62Y,2492 +H29L/L63V,2310 +H29N/L62F,567 +H29S/L62A,640 +H29Y/L63Y,1274 +H29D/L62C,972 +H29Y/L63T,4392 +V30H/L63N,775 +H29A/L63S,2011 +V30C/L63R,2833 +V30D/L63S,1824 +L62D,1003 +V30D/L62I,1814 +H29P/L62V,2902 +__unknown__,2441 diff --git a/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t0.csv b/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t0.csv new file mode 100644 index 00000000..95c75095 --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2033 +K84L,721 +M42I/K84L,564 +H29T/L63V,1604 +V30L/L63R,735 +V30Y/L63S,1092 +V30R/L63N,2046 +V30T/L62S,1207 +V30P/L62D,744 +H29V/L62F,1385 +H29N/L63G,1002 +H29N/L63D,993 +V30F/L62F,1752 +V30T/L63R,880 +H29G/L63D,531 +H29C/L62A,961 +H29L/L62C,1277 +V30P/L62S,1739 +L63F,1061 +V30R/L62Y,2602 +H29L/L63V,2570 +H29N/L62F,657 +H29S/L62A,745 +H29Y/L63Y,1325 +H29D/L62C,1270 +H29Y/L63T,4157 +V30H/L63N,864 +H29A/L63S,1762 +V30C/L63R,2053 +V30D/L63S,1745 +L62D,905 +V30D/L62I,1434 +H29P/L62V,3066 +__unknown__,2518 diff --git a/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t120.csv b/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t120.csv new file mode 100644 index 00000000..149b21c9 --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1903 +K84L,416 +M42I/K84L,360 +H29T/L63V,1145 +V30L/L63R,633 +V30Y/L63S,236 +V30R/L63N,2619 +V30T/L62S,1884 +V30P/L62D,464 +H29V/L62F,2354 +H29N/L63G,627 +H29N/L63D,426 +V30F/L62F,1714 +V30T/L63R,522 +H29G/L63D,367 +H29C/L62A,1151 +H29L/L62C,779 +V30P/L62S,2786 +L63F,661 +V30R/L62Y,2439 +H29L/L63V,2132 +H29N/L62F,358 +H29S/L62A,488 +H29Y/L63Y,1244 +H29D/L62C,737 +H29Y/L63T,4509 +V30H/L63N,656 +H29A/L63S,2116 +V30C/L63R,3930 +V30D/L63S,1860 +L62D,1098 +V30D/L62I,2166 +H29P/L62V,2673 +__unknown__,2547 diff --git a/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t60.csv b/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t60.csv new file mode 100644 index 00000000..fc08a4ee --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSm4CP_1mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2047 +K84L,508 +M42I/K84L,514 +H29T/L63V,1399 +V30L/L63R,780 +V30Y/L63S,498 +V30R/L63N,2360 +V30T/L62S,1548 +V30P/L62D,631 +H29V/L62F,1810 +H29N/L63G,786 +H29N/L63D,690 +V30F/L62F,1771 +V30T/L63R,726 +H29G/L63D,464 +H29C/L62A,1001 +H29L/L62C,985 +V30P/L62S,2267 +L63F,861 +V30R/L62Y,2658 +H29L/L63V,2349 +H29N/L62F,520 +H29S/L62A,636 +H29Y/L63Y,1241 +H29D/L62C,979 +H29Y/L63T,4330 +V30H/L63N,739 +H29A/L63S,2051 +V30C/L63R,2932 +V30D/L63S,1810 +L62D,1055 +V30D/L62I,1768 +H29P/L62V,2864 +__unknown__,2422 diff --git a/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t0.csv b/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t0.csv new file mode 100644 index 00000000..2fe19a41 --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2041 +K84L,733 +M42I/K84L,521 +H29T/L63V,1612 +V30L/L63R,794 +V30Y/L63S,1057 +V30R/L63N,2112 +V30T/L62S,1217 +V30P/L62D,749 +H29V/L62F,1522 +H29N/L63G,1018 +H29N/L63D,933 +V30F/L62F,1701 +V30T/L63R,929 +H29G/L63D,498 +H29C/L62A,888 +H29L/L62C,1262 +V30P/L62S,1728 +L63F,1131 +V30R/L62Y,2618 +H29L/L63V,2571 +H29N/L62F,667 +H29S/L62A,736 +H29Y/L63Y,1376 +H29D/L62C,1242 +H29Y/L63T,4131 +V30H/L63N,766 +H29A/L63S,1803 +V30C/L63R,2085 +V30D/L63S,1646 +L62D,931 +V30D/L62I,1432 +H29P/L62V,3152 +__unknown__,2398 diff --git a/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t120.csv b/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t120.csv new file mode 100644 index 00000000..e50e05d7 --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1997 +K84L,405 +M42I/K84L,428 +H29T/L63V,1156 +V30L/L63R,653 +V30Y/L63S,246 +V30R/L63N,2680 +V30T/L62S,1924 +V30P/L62D,437 +H29V/L62F,2238 +H29N/L63G,657 +H29N/L63D,425 +V30F/L62F,1656 +V30T/L63R,578 +H29G/L63D,328 +H29C/L62A,1152 +H29L/L62C,734 +V30P/L62S,2794 +L63F,624 +V30R/L62Y,2504 +H29L/L63V,2128 +H29N/L62F,400 +H29S/L62A,485 +H29Y/L63Y,1219 +H29D/L62C,677 +H29Y/L63T,4497 +V30H/L63N,673 +H29A/L63S,2158 +V30C/L63R,3887 +V30D/L63S,1867 +L62D,1051 +V30D/L62I,2192 +H29P/L62V,2722 +__unknown__,2428 diff --git a/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t60.csv b/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t60.csv new file mode 100644 index 00000000..18a37077 --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSp4CP_0mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2016 +K84L,540 +M42I/K84L,471 +H29T/L63V,1427 +V30L/L63R,736 +V30Y/L63S,497 +V30R/L63N,2400 +V30T/L62S,1535 +V30P/L62D,568 +H29V/L62F,1832 +H29N/L63G,763 +H29N/L63D,667 +V30F/L62F,1682 +V30T/L63R,701 +H29G/L63D,442 +H29C/L62A,1024 +H29L/L62C,978 +V30P/L62S,2269 +L63F,816 +V30R/L62Y,2593 +H29L/L63V,2398 +H29N/L62F,526 +H29S/L62A,655 +H29Y/L63Y,1298 +H29D/L62C,955 +H29Y/L63T,4427 +V30H/L63N,794 +H29A/L63S,2085 +V30C/L63R,2923 +V30D/L63S,1820 +L62D,908 +V30D/L62I,1812 +H29P/L62V,2890 +__unknown__,2552 diff --git a/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t0.csv b/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t0.csv new file mode 100644 index 00000000..103cf1dc --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t0.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2008 +K84L,695 +M42I/K84L,562 +H29T/L63V,1590 +V30L/L63R,780 +V30Y/L63S,1040 +V30R/L63N,2081 +V30T/L62S,1212 +V30P/L62D,731 +H29V/L62F,1459 +H29N/L63G,954 +H29N/L63D,957 +V30F/L62F,1772 +V30T/L63R,975 +H29G/L63D,465 +H29C/L62A,883 +H29L/L62C,1196 +V30P/L62S,1759 +L63F,1099 +V30R/L62Y,2578 +H29L/L63V,2607 +H29N/L62F,656 +H29S/L62A,748 +H29Y/L63Y,1344 +H29D/L62C,1324 +H29Y/L63T,4117 +V30H/L63N,846 +H29A/L63S,1848 +V30C/L63R,2079 +V30D/L63S,1620 +L62D,917 +V30D/L62I,1473 +H29P/L62V,3064 +__unknown__,2561 diff --git a/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t120.csv b/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t120.csv new file mode 100644 index 00000000..a64e0e43 --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t120.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,1968 +K84L,406 +M42I/K84L,400 +H29T/L63V,1131 +V30L/L63R,691 +V30Y/L63S,284 +V30R/L63N,2723 +V30T/L62S,1946 +V30P/L62D,466 +H29V/L62F,2261 +H29N/L63G,614 +H29N/L63D,489 +V30F/L62F,1589 +V30T/L63R,518 +H29G/L63D,400 +H29C/L62A,1199 +H29L/L62C,749 +V30P/L62S,2676 +L63F,635 +V30R/L62Y,2387 +H29L/L63V,2076 +H29N/L62F,391 +H29S/L62A,475 +H29Y/L63Y,1186 +H29D/L62C,744 +H29Y/L63T,4578 +V30H/L63N,677 +H29A/L63S,2196 +V30C/L63R,3855 +V30D/L63S,1890 +L62D,1077 +V30D/L62I,2170 +H29P/L62V,2706 +__unknown__,2447 diff --git a/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t60.csv b/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t60.csv new file mode 100644 index 00000000..8f1e9c97 --- /dev/null +++ b/examples/process_raw/counts/counts_pheS_pheSp4CP_1mM_t60.csv @@ -0,0 +1,35 @@ +genotype,counts +wt,2054 +K84L,576 +M42I/K84L,524 +H29T/L63V,1314 +V30L/L63R,729 +V30Y/L63S,508 +V30R/L63N,2327 +V30T/L62S,1482 +V30P/L62D,580 +H29V/L62F,1839 +H29N/L63G,782 +H29N/L63D,668 +V30F/L62F,1753 +V30T/L63R,771 +H29G/L63D,450 +H29C/L62A,1026 +H29L/L62C,971 +V30P/L62S,2290 +L63F,855 +V30R/L62Y,2646 +H29L/L63V,2359 +H29N/L62F,513 +H29S/L62A,612 +H29Y/L63Y,1308 +H29D/L62C,1020 +H29Y/L63T,4410 +V30H/L63N,763 +H29A/L63S,2050 +V30C/L63R,2816 +V30D/L63S,1745 +L62D,999 +V30D/L62I,1801 +H29P/L62V,2904 +__unknown__,2555 diff --git a/examples/process_raw/library_config.yaml b/examples/process_raw/library_config.yaml index 00e38435..b85c2c89 100644 --- a/examples/process_raw/library_config.yaml +++ b/examples/process_raw/library_config.yaml @@ -1,9 +1,10 @@ # --------------------------------------------------------------------------- -# Library configuration for tfs-process-fastq +# Library configuration for tfs-process-fastq and tfs-configure-model # -# Pass this file as the run_config argument: -# tfs-process-fastq R1.fastq.gz R2.fastq.gz out/ library_config.yaml +# Pass this one file to both: +# tfs-process-fastq library_config.yaml R1.fastq.gz R2.fastq.gz --out_dir counts +# tfs-configure-model --growth_df growth.csv --library_config library_config.yaml # # If you are running both tfs-simulate and tfs-process-fastq on the same # experiment, you can pass the simulate config directly to tfs-process-fastq @@ -40,7 +41,7 @@ tile_combos: ["single-1", "single-2", "double-1-2"] # Control sequences added as spikes. Must be the same length as wt_seq. spiked_seqs: -- gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgatggcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctggcggcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # k84l +- gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgatggcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctgcacgcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # k84l - gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgattgcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctgcacgcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # m42i/k84l # --------------------------------------------------------------------------- diff --git a/examples/process_raw/make_example_data.py b/examples/process_raw/make_example_data.py new file mode 100644 index 00000000..e73d9964 --- /dev/null +++ b/examples/process_raw/make_example_data.py @@ -0,0 +1,82 @@ +""" +Write a synthetic experiment in the formats tfs-process-counts reads. + + python make_example_data.py + tfs-process-counts tube_table.csv counts \ + --od600_file od600.csv \ + --od600_calibration_file ../od600/od600_calibration.yaml \ + --tube_volume_mL 5 --out_prefix growth + +It writes: + +- ``tube_table.csv``: one row per sequenced tube. ``sample`` names the tube + (and its count file); ``library`` is the transformed library it came from + (kanR and pheS are transformed and grown up separately); the rest is the + design: ``replicate``, ``condition_pre``, ``t_pre``, ``condition_sel``, + ``t_sel`` (minutes), ``titrant_name``, ``titrant_conc`` (mM). +- ``od600.csv``: one OD600 reading per tube (``sample``, ``od600``). +- ``counts/counts_.csv``: ``genotype``, ``counts``, the format + tfs-process-fastq writes, including the ``__unknown__`` row of reads that + matched no library genotype. + +The numbers are made up: a few dozen genotypes of the library in +``library_config.yaml`` growing at random rates, read at about 50,000 reads +per tube. They show the formats; they are not a realistic screen. +""" + +import os + +import numpy as np +import pandas as pd + +from tfscreen.genetics import library_composition_table + +HERE = os.path.dirname(os.path.abspath(__file__)) +rng = np.random.default_rng(20261003) + +# Genotypes: wt, the spikes and a random handful of library members. +lib = library_composition_table(os.path.join(HERE, "library_config.yaml")) +spikes = list(lib.loc[lib["in_spiked_origin"], "genotype"]) +others = [g for g in lib["genotype"] if g != "wt" and g not in spikes] +genotypes = ["wt"] + spikes + list(rng.choice(others, size=30, replace=False)) +rate_shift = dict(zip(genotypes, rng.normal(0.0, 0.004, len(genotypes)))) +rate_shift["wt"] = 0.0 + +# Design: two libraries, each with a selection and a control condition, two +# IPTG concentrations and three time points, one replicate. +rows = [] +for library, marker, agent in (("kanR", "kanR", "kan"), ("pheS", "pheS", "4CP")): + for sel in (f"{marker}+{agent}", f"{marker}-{agent}"): + for conc in (0.0, 1.0): + for t_sel in (0, 60, 120): + name = f"{library}_{sel.replace('+', 'p').replace('-', 'm')}" \ + f"_{conc:g}mM_t{t_sel}" + rows.append({"sample": name, "library": library, + "replicate": 1, + "condition_pre": f"{marker}-{agent}", "t_pre": 30, + "condition_sel": sel, "t_sel": t_sel, + "titrant_name": "iptg", "titrant_conc": conc}) +tubes = pd.DataFrame(rows) +tubes.to_csv(os.path.join(HERE, "tube_table.csv"), index=False) + +# OD600 grows with time; every reading inside the example calibration's range. +od = 0.15 * np.exp(0.006 * tubes["t_sel"].to_numpy()) \ + * rng.lognormal(0.0, 0.03, len(tubes)) +pd.DataFrame({"sample": tubes["sample"], "od600": np.round(od, 4)}).to_csv( + os.path.join(HERE, "od600.csv"), index=False) + +# Counts: each genotype's share grows by its own rate offset; reads are a +# multinomial draw with 5% of reads matching no genotype. +start = rng.dirichlet(np.full(len(genotypes), 5.0)) +counts_dir = os.path.join(HERE, "counts") +os.makedirs(counts_dir, exist_ok=True) +shift = np.array([rate_shift[g] for g in genotypes]) +for _, tube in tubes.iterrows(): + w = start * np.exp(shift * (tube["t_pre"] + tube["t_sel"])) + w = 0.95 * w / w.sum() + reads = rng.multinomial(50_000, np.append(w, 0.05)) + pd.DataFrame({"genotype": genotypes + ["__unknown__"], + "counts": reads}).to_csv( + os.path.join(counts_dir, f"counts_{tube['sample']}.csv"), index=False) + +print(f"Wrote tube_table.csv, od600.csv and {len(tubes)} count files.") diff --git a/examples/process_raw/od600.csv b/examples/process_raw/od600.csv new file mode 100644 index 00000000..13c677c6 --- /dev/null +++ b/examples/process_raw/od600.csv @@ -0,0 +1,25 @@ +sample,od600 +kanR_kanRpkan_0mM_t0,0.1506 +kanR_kanRpkan_0mM_t60,0.2014 +kanR_kanRpkan_0mM_t120,0.3086 +kanR_kanRpkan_1mM_t0,0.1469 +kanR_kanRpkan_1mM_t60,0.2177 +kanR_kanRpkan_1mM_t120,0.2987 +kanR_kanRmkan_0mM_t0,0.1571 +kanR_kanRmkan_0mM_t60,0.2153 +kanR_kanRmkan_0mM_t120,0.3007 +kanR_kanRmkan_1mM_t0,0.1508 +kanR_kanRmkan_1mM_t60,0.221 +kanR_kanRmkan_1mM_t120,0.3062 +pheS_pheSp4CP_0mM_t0,0.1561 +pheS_pheSp4CP_0mM_t60,0.2153 +pheS_pheSp4CP_0mM_t120,0.3113 +pheS_pheSp4CP_1mM_t0,0.1494 +pheS_pheSp4CP_1mM_t60,0.2108 +pheS_pheSp4CP_1mM_t120,0.3081 +pheS_pheSm4CP_0mM_t0,0.1524 +pheS_pheSm4CP_0mM_t60,0.2115 +pheS_pheSm4CP_0mM_t120,0.2986 +pheS_pheSm4CP_1mM_t0,0.1587 +pheS_pheSm4CP_1mM_t60,0.2158 +pheS_pheSm4CP_1mM_t120,0.3085 diff --git a/examples/process_raw/tube_table.csv b/examples/process_raw/tube_table.csv new file mode 100644 index 00000000..f92bdb23 --- /dev/null +++ b/examples/process_raw/tube_table.csv @@ -0,0 +1,25 @@ +sample,library,replicate,condition_pre,t_pre,condition_sel,t_sel,titrant_name,titrant_conc +kanR_kanRpkan_0mM_t0,kanR,1,kanR-kan,30,kanR+kan,0,iptg,0.0 +kanR_kanRpkan_0mM_t60,kanR,1,kanR-kan,30,kanR+kan,60,iptg,0.0 +kanR_kanRpkan_0mM_t120,kanR,1,kanR-kan,30,kanR+kan,120,iptg,0.0 +kanR_kanRpkan_1mM_t0,kanR,1,kanR-kan,30,kanR+kan,0,iptg,1.0 +kanR_kanRpkan_1mM_t60,kanR,1,kanR-kan,30,kanR+kan,60,iptg,1.0 +kanR_kanRpkan_1mM_t120,kanR,1,kanR-kan,30,kanR+kan,120,iptg,1.0 +kanR_kanRmkan_0mM_t0,kanR,1,kanR-kan,30,kanR-kan,0,iptg,0.0 +kanR_kanRmkan_0mM_t60,kanR,1,kanR-kan,30,kanR-kan,60,iptg,0.0 +kanR_kanRmkan_0mM_t120,kanR,1,kanR-kan,30,kanR-kan,120,iptg,0.0 +kanR_kanRmkan_1mM_t0,kanR,1,kanR-kan,30,kanR-kan,0,iptg,1.0 +kanR_kanRmkan_1mM_t60,kanR,1,kanR-kan,30,kanR-kan,60,iptg,1.0 +kanR_kanRmkan_1mM_t120,kanR,1,kanR-kan,30,kanR-kan,120,iptg,1.0 +pheS_pheSp4CP_0mM_t0,pheS,1,pheS-4CP,30,pheS+4CP,0,iptg,0.0 +pheS_pheSp4CP_0mM_t60,pheS,1,pheS-4CP,30,pheS+4CP,60,iptg,0.0 +pheS_pheSp4CP_0mM_t120,pheS,1,pheS-4CP,30,pheS+4CP,120,iptg,0.0 +pheS_pheSp4CP_1mM_t0,pheS,1,pheS-4CP,30,pheS+4CP,0,iptg,1.0 +pheS_pheSp4CP_1mM_t60,pheS,1,pheS-4CP,30,pheS+4CP,60,iptg,1.0 +pheS_pheSp4CP_1mM_t120,pheS,1,pheS-4CP,30,pheS+4CP,120,iptg,1.0 +pheS_pheSm4CP_0mM_t0,pheS,1,pheS-4CP,30,pheS-4CP,0,iptg,0.0 +pheS_pheSm4CP_0mM_t60,pheS,1,pheS-4CP,30,pheS-4CP,60,iptg,0.0 +pheS_pheSm4CP_0mM_t120,pheS,1,pheS-4CP,30,pheS-4CP,120,iptg,0.0 +pheS_pheSm4CP_1mM_t0,pheS,1,pheS-4CP,30,pheS-4CP,0,iptg,1.0 +pheS_pheSm4CP_1mM_t60,pheS,1,pheS-4CP,30,pheS-4CP,60,iptg,1.0 +pheS_pheSm4CP_1mM_t120,pheS,1,pheS-4CP,30,pheS-4CP,120,iptg,1.0 diff --git a/examples/simulate-and-analyze/README.md b/examples/simulate-and-analyze/README.md index c3765eb3..d89fb772 100644 --- a/examples/simulate-and-analyze/README.md +++ b/examples/simulate-and-analyze/README.md @@ -1,33 +1,37 @@ # simulate-and-analyze -A complete end-to-end example: simulate a synthetic TF-library screen and fit -the hierarchical Bayesian model to the simulated data. +A complete example: simulate a small TF-library screen, then fit the +hierarchical Bayesian model to the simulated data and compare the fit with +the known truth. ## Contents | File | Purpose | |------|---------| -| `simulate_config.yaml` | Simulation parameters (library genetics, conditions, binding data) | -| `hill_params.csv` | Per-genotype Hill binding parameters used to generate ground-truth θ | -| `run.sh` | Pipeline script (simulate → configure → calibrate → fit → predict → summarise) | +| `simulate_config.yaml` | Simulation settings (library genetics, conditions, growth, binding and presplit data); also the library description for `tfs-configure-model --library_config` | +| `hill_params.csv` | Hill parameters for wt and three spiked single mutants, used as their true θ curves and binding data | +| `run.sh` | The pipeline: simulate, configure, pre-fit, fit, sample the posterior, extract, predict, summarize | ## Running +Run from this directory, since the config names `hill_params.csv` by a +relative path: + ```bash -bash run.sh simulate_config.yaml out/ 1 +bash run.sh simulate_config.yaml out 1 ``` -The three arguments are the config file, the output directory (created -automatically), and the random seed. Seed `1` reproduces the documented -example outputs. +The arguments are the simulate config, the output directory (created if +needed) and the random seed (default 1). -Running time is roughly 20–60 minutes on a laptop CPU or a few minutes on a -GPU. On a cluster, comment out the `XLA_FLAGS` line in `run.sh` and -uncomment `module load cuda/...` instead. +The fit is the slow step: expect tens of minutes on a laptop CPU and a few +minutes on a GPU. On a cluster, comment out the `XLA_FLAGS` line in `run.sh` +and uncomment `module load cuda/...`; the `#SBATCH` lines let you submit it +with `sbatch`. ## Documentation -See the [quickstart](https://tfscreen.readthedocs.io/en/latest/quickstart.html) -for a guided walkthrough and the -[tfs-summarize-fit reference](https://tfscreen.readthedocs.io/en/latest/summarize-fit.html) -for a detailed guide to interpreting the diagnostic outputs in `out/summary/`. +The [quickstart](https://tfscreen.readthedocs.io/en/latest/quickstart.html) +walks through every step and every output file. The +[tfs-summarize-fit guide](https://tfscreen.readthedocs.io/en/latest/summarize-fit.html) +explains the diagnostic outputs in `out/summary/`. diff --git a/examples/simulate-and-analyze/run.sh b/examples/simulate-and-analyze/run.sh index 09ff0d9b..8fa59d64 100644 --- a/examples/simulate-and-analyze/run.sh +++ b/examples/simulate-and-analyze/run.sh @@ -38,9 +38,12 @@ fi # 1. Simulate library # --------------------------------------------------------------------------- # tfs-simulate reads the config YAML and writes simulated growth data, -# binding curves, and ground-truth parameter CSVs to run_dir. +# binding curves, and ground-truth parameter CSVs as run_dir/tfs_sim_*. It +# also writes the experiment in the raw formats a lab's data come in: one +# counts file per tube (tfs_sim_counts/) and the tube table +# (tfs_sim_tubes.csv). echo ">>> Simulate library" -tfs-simulate "${config_file}" "${run_dir}" --seed "${seed}" +tfs-simulate "${config_file}" --out_prefix "${run_dir}/tfs_sim" --seed "${seed}" # The simulate config also describes the library (genetics keys + # library_mixture), so tfs-configure-model reads the same file. Resolve it to @@ -49,20 +52,38 @@ library_config="$(cd "$(dirname "${config_file}")" && pwd)/$(basename "${config_ cd "${run_dir}" +# --------------------------------------------------------------------------- +# 1b. Process the raw counts, exactly as for real data +# --------------------------------------------------------------------------- +# The growth table the model reads comes from the per-tube count files and the +# tube table, through the same command a lab runs. (This config has no od600 +# block, so the tube table carries each tube's total; with one, add the +# --od600_file, --od600_calibration_file and --tube_volume_mL that +# tfs-simulate prints.) The presplit table is still written directly by +# tfs-simulate. +echo ">>> Process counts" +tfs-process-counts tfs_sim_tubes.csv tfs_sim_counts \ + --out_prefix tfs_sim_processed_growth --no_verbose + # --------------------------------------------------------------------------- # 2. Configure model # --------------------------------------------------------------------------- -# tfs-configure-model validates the data, maps labels to indices, selects -# model components, and writes tfs_configure_config.yaml + priors/guesses CSVs. -# Edit the flags here to change which model components are used. +# tfs-configure-model validates the data, selects model components, and +# writes tfs_configure_config.yaml, the priors/guesses/library CSVs and a +# parameter census (tfs_configure_model_stats.*). Edit the flags here to +# change which model components are used. The data were simulated with +# hill_mut theta and epistasis, one plasmid per cell +# (transformation_poisson_lambda: 0) and an instant growth transition, so the +# model below matches the simulation; growth_likelihood counts with a level +# tube offset is the recommended default. echo ">>> Configure model" tfs-configure-model \ - tfs_sim_binding.csv \ - --growth_df tfs_sim_growth.csv \ + --binding_df tfs_sim_binding.csv \ + --growth_df tfs_sim_processed_growth.csv \ --presplit_df tfs_sim_presplit.csv \ --condition_growth_model linear \ --growth_transition_model instant \ - --ln_cfu0_model hierarchical_factored \ + --ln_cfu0_model hierarchical \ --dk_geno_model hierarchical_geno \ --activity_model fixed \ --theta_model hill_mut \ @@ -70,7 +91,9 @@ tfs-configure-model \ --theta_rescale_model passthrough \ --theta_growth_noise_model logit_normal \ --theta_binding_noise_model zero \ - --growth_noise_model normal_kt \ + --growth_likelihood counts \ + --sample_offset_model level \ + --growth_noise_model zero \ --library_config "${library_config}" \ --growth_shares_replicates \ --epistasis @@ -78,60 +101,68 @@ tfs-configure-model \ # --------------------------------------------------------------------------- # 3. Pre-fit calibration # --------------------------------------------------------------------------- -# A fast MAP fit on a simplified model calibrates the growth-linking-function -# priors (m and b) before the full inference run. Updates priors.csv and -# guesses.csv in place. +# A MAP fit on a simplified model, using the genotypes that have binding +# data, calibrates each condition's growth baseline k and slope m. It writes +# them into tfs_configure_priors.csv and tfs_configure_guesses.csv in place +# (keeping .bak copies) and leaves its own diagnostics as tfs_prefit_*. echo ">>> Pre-fit calibration" tfs-prefit-calibration tfs_configure_config.yaml --seed "${seed}" # --------------------------------------------------------------------------- # 4. Fit model (SVI) # --------------------------------------------------------------------------- -# Main hierarchical Bayesian inference. SVI produces a full approximate -# posterior; for a point estimate only, use --analysis_method map instead. +# Main hierarchical Bayesian inference. SVI starts from a MAP warm-up +# (tfs_fit_model_premap_*) and fits the component guide, an approximate +# posterior. Writes tfs_fit_model_checkpoint.pkl, _params.npz, _losses.txt +# and _convergence.csv. --analysis_method map gives the MAP point instead; +# tfs-sample-posterior then builds a Laplace posterior from it. echo ">>> Fit model" tfs-fit-model \ tfs_configure_config.yaml \ --seed "${seed}" \ - --analysis_method svi \ - --convergence_tolerance 0.0001 + --analysis_method svi # --------------------------------------------------------------------------- # 5. Sample posterior # --------------------------------------------------------------------------- -# Draw posterior samples from the SVI variational distribution and write -# them to an HDF5 file used by the prediction steps below. +# Draw posterior samples from the fitted guide and write them to +# tfs_posterior.h5 for the steps below. --skip_growth_observations leaves out +# the stored per-observation growth sites (growth_pred, growth_obs). They are +# most of the file (several GB here at the default 10000 samples), and +# nothing below reads them: tfs-predict-growth and tfs-summarize-fit +# recompute growth from the parameter samples. echo ">>> Sample posterior" -tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --skip_growth_observations # --------------------------------------------------------------------------- # 6. Extract parameter estimates # --------------------------------------------------------------------------- -# Summarise the posterior into per-parameter CSV files (quantiles, means). +# Summarize the posterior into per-parameter CSV files (tfs_params_*.csv, +# one column per quantile). echo ">>> Extract parameter estimates" tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 # --------------------------------------------------------------------------- # 7. Predict theta # --------------------------------------------------------------------------- -# Predict operator occupancy θ as a function of titrant concentration for -# every genotype in the training data. +# Predict operator occupancy θ at every (genotype, titrant concentration) in +# the training data. Writes tfs_pred_theta.csv. echo ">>> Predict theta" tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 # --------------------------------------------------------------------------- # 8. Predict growth # --------------------------------------------------------------------------- -# Predict ln(CFU) with posterior uncertainty for every training observation. +# Predict ln(CFU) with posterior uncertainty for every training observation, +# from 500 of the posterior samples. Writes tfs_pred_growth.csv. echo ">>> Predict growth" -tfs-predict-growth tfs_configure_config.yaml tfs_posterior.h5 --num_marginal_samples=500 +tfs-predict-growth tfs_configure_config.yaml tfs_posterior.h5 --num_marginal_samples 500 # --------------------------------------------------------------------------- -# 9. Summarise fit +# 9. Summarize fit # --------------------------------------------------------------------------- -# Collects all outputs, computes statistics, and writes diagnostic PDFs and -# CSVs to the summary/ subdirectory. -echo ">>> Summarise fit" +# Compares the predictions with the simulated truth (tfs_sim_genotype_theta.csv) +# and writes diagnostic PDFs and CSVs to the summary/ subdirectory. +echo ">>> Summarize fit" tfs-summarize-fit . - -cd .. diff --git a/examples/simulate-and-analyze/simulate_config.yaml b/examples/simulate-and-analyze/simulate_config.yaml index 895c2024..72539c3d 100644 --- a/examples/simulate-and-analyze/simulate_config.yaml +++ b/examples/simulate-and-analyze/simulate_config.yaml @@ -179,12 +179,20 @@ lib_assembly_skew_sigma: 1.25 # Poisson with this lambda. Set to 0 or null for exactly one plasmid per cell. transformation_poisson_lambda: 0 -# When a cell carries multiple plasmids, growth rates are combined via this -# function. Recommended: 'min' for repressors of antibiotic resistance (tightest -# binder dominates); 'max' for inducers. Default: 'mean' for all libraries. -multi_plasmid_combine_fcn: - kanR: "min" - pheS: "max" +# How a cell carrying several plasmids gets its growth rate. The cell's theta +# comes from congression_theta_rule, at each condition: 'homodimer' +# (logit theta_cell = log of the share-weighted mean of the plasmids' +# theta odds), 'heterodimer' (closer to the mean logit) or 'max' (the +# highest-theta plasmid sets theta and TF activity). 'homodimer' and +# 'heterodimer' need activity 1 (activity_mut_scale 0). Its dk_geno comes +# from congression_dk_rule, the soft-min family +# dk_cell = -(1/alpha) log sum_g x_g exp(-alpha dk_g): 'dilution' +# (alpha -> 0, the share-weighted mean), 'softmin' (finite alpha, set by +# congression_dk_alpha, in units of 1/dk) or 'min' (alpha -> inf, the worst +# variant sets the cost). Growth is then computed once from these cell-level +# values. All three keys are optional; these are the defaults (no alpha). +congression_theta_rule: homodimer +congression_dk_rule: dilution # Colony-forming units per mL at the start of pre-growth. cfu0: 6.5e7 diff --git a/examples/simulate-empirical/simulate_config.yaml b/examples/simulate-empirical/simulate_config.yaml index 8b8afb02..f7b0ea62 100644 --- a/examples/simulate-empirical/simulate_config.yaml +++ b/examples/simulate-empirical/simulate_config.yaml @@ -10,13 +10,13 @@ # # Workflow: # # 1. real data -> empirical generating distribution (writes emp_phenotype_model.json) -# tfs-build-empirical growth.csv 42 \ -# --binding_file binding.csv --library_config library_config.yaml \ +# tfs-build-empirical growth.csv --seed 42 \ +# --binding_df binding.csv --library_config library_config.yaml \ # --out_prefix emp -# # 2. simulate from it -# tfs-simulate simulate_config.yaml out_dir/ +# # 2. simulate from it (writes out/tfs_sim_*.csv) +# tfs-simulate simulate_config.yaml --out_prefix out/tfs_sim # -# See the "Empirical phenotype pipeline" section of CLAUDE.md for the full story. +# See docs/source/simulation.rst ("Empirical phenotypes") for the full story. # --------------------------------------------------------------------------- # Empirical phenotype source (the only empirical-specific keys) @@ -59,7 +59,7 @@ expected_5p: cgcgtggtgaaccag expected_3p: gtagaacgaagcggc tile_combos: ["single-1", "single-2", "double-1-2"] spiked_seqs: -- gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgatggcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctggcggcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # k84l +- gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgatggcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctgcacgcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # k84l - gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgattgcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctgcacgcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # m42i/k84l # --------------------------------------------------------------------------- @@ -103,11 +103,13 @@ condition_blocks: # Growth rate parameters # --------------------------------------------------------------------------- -# Linear growth linkage per condition: k = m * theta + b. +# Linear growth linkage per condition: k = b + A * m * theta + dk_geno +# (A = 1 here). # # IMPORTANT: for a self-consistent empirical simulation these MUST be the same -# calibrated values the phenotype fit used -- i.e. the per-condition values the -# pre-fit wrote into _configure_priors.csv. For each condition_rep: +# calibrated values the phenotype fit used, i.e. the per-condition values the +# pre-fit wrote into _configure_priors.csv (or the values in the file you +# passed as --growth_calibration_file). For each condition_rep: # b = value of parameter growth.condition_growth.k_loc (baseline) # m = value of parameter growth.condition_growth.m_loc (slope) # (The numbers below are placeholders -- replace them with your calibration.) @@ -144,9 +146,8 @@ lib_assembly_skew_sigma: 1.25 # attenuation your empirical distribution is (currently) uncorrected for, so # keep it matched to the real library. transformation_poisson_lambda: 1.5 -multi_plasmid_combine_fcn: - kanR: "min" - pheS: "max" +congression_theta_rule: homodimer # needs activity 1 (the empirical path forces it) +congression_dk_rule: dilution cfu0: 65_000_000.0 tube_noise_sigma: 0.002 diff --git a/examples/simulate/run.sh b/examples/simulate/run.sh index 7271fb41..b67e9092 100644 --- a/examples/simulate/run.sh +++ b/examples/simulate/run.sh @@ -8,7 +8,7 @@ set -e # # NOTE: tfs-setup-sim-grid already applied the simulate: overrides and wrote # tfs_sim_config.yaml in this directory. Per-run config choices (noise level, -# thermodynamic model, random seed, etc.) live there; they do not appear here. +# thermodynamic model, seed, etc.) live there; they do not appear here. # --------------------------------------------------------------------------- NUM_REPLICATES={{ num_replicates }} @@ -18,6 +18,9 @@ NUM_REPLICATES={{ num_replicates }} # --------------------------------------------------------------------------- echo ">>> Running simulation" -tfs-simulate tfs_sim_config.yaml . \ - --output_prefix sim_ \ +# Writes tfs_sim_growth.csv, tfs_sim_genotype_theta.csv, ... in this +# directory. Keep the tfs_sim prefix: tfs-summarize-fit finds the ground +# truth as *_sim_genotype_theta.csv. +tfs-simulate tfs_sim_config.yaml \ + --out_prefix tfs_sim \ --num_replicates ${NUM_REPLICATES} diff --git a/examples/simulate/simulate_config.yaml b/examples/simulate/simulate_config.yaml index 78c193de..0defe041 100644 --- a/examples/simulate/simulate_config.yaml +++ b/examples/simulate/simulate_config.yaml @@ -3,6 +3,10 @@ # Library genetic information # --------------------------------------------------------------------------- +# These keys, plus library_mixture below, describe the library. Hand this same +# file to tfs-process-fastq and to tfs-configure-model --library_config; each +# tool ignores the keys it does not use. + # Reading frame 0, 1, 2. No reverse (negative) strand allowed. reading_frame: 0 @@ -35,15 +39,29 @@ tile_combos: ["single-1", "single-2", "double-1-2"] # Control sequences (e.g. known variants) added as spikes. Must be the same # length as wt_seq. spiked_seqs: -- gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgatggcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctggcggcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # k84l +- gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgatggcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctgcacgcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # k84l - gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgattgcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctgcacgcgccgtcgcaaattgtcgcggcgattctgtctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg # m42i/k84l +# Optional: simulate (and fit) only a subset of the described genotypes. +# library_subsample keeps N random genotypes of a sub-library (others whole) +# and needs its own integer seed; library_genotypes names a file listing +# every genotype to keep, relative to this file. wt and spiked genotypes are +# always kept. With library_subsample_depth: full_library (the default) the +# mixture, transform sizes, reads and PCR templates below describe the full +# library and each kept sequence keeps its share; as_written applies them to +# the subsample as given. tfs-configure-model --library_config reads these +# keys too. See docs/source/simulation.rst, "Subsampling the library". +# library_subsample: +# double-1-2: 200 +# library_subsample_seed: 1 +# library_genotypes: genotypes.txt +# library_subsample_depth: full_library + # --------------------------------------------------------------------------- # Phenotype calculation # --------------------------------------------------------------------------- -# Registered theta component name. Any key in model_registry["theta"] is -# valid except "_simple" (calibration-only). +# Theta component: hill_geno, hill_mut or a thermo.* partition-function key. # # Thermodynamic components (physics-based operator-occupancy model): # "thermo.O2_C12_K5_U0_a.PK" — dimer with operator architecture O2, 12-mer @@ -63,9 +81,11 @@ spiked_seqs: # exact reference curve. # # Perturbation-based components ignore theta_priors and instead use -# theta_sim_priors (see below). They do not call define_model; tfs-simulate -# will fall back to prior-predictive sampling only for components that lack a -# simulate() function. +# theta_sim_priors (see below). Components without their own simulation +# routine draw theta from the component's prior instead. +# +# theta_rescale (optional): 'passthrough' (default) or 'logit', applied to +# theta before it enters the growth model, as the fit-side component. theta_component: "thermo.O2_C12_K5_U0_a.PK" # Optional: path to structural/thermodynamic data file required by PnnC and @@ -74,6 +94,17 @@ theta_component: "thermo.O2_C12_K5_U0_a.PK" # .csv → per-mutation per-structure ddG prior means (PddG) # thermo_data: null +# Optional: how to read thermo_data (see docs/source/model-inputs.rst). +# thermo_options: +# ddG_units: kcal/mol # kT (default), kcal/mol or kJ/mol +# residue_offset: 0 + +# Optional (PddG/PnnC only): SD of the simulated per-structure ddG truth around +# the thermo_data values, in ddG_units. A number for every structure, or +# {structure: sd} naming each one. 0 makes the truth the table exactly. +# Omit for 1 kT per structure. The true SDs go to tfs_sim_ddG_sigma.csv. +# ddG_truth_sd: 0.5 + # Optional: overrides for the theta component's default model hyperparameters. # Any key returned by the component's get_hyperparameters() can be overridden. # Used by thermodynamic/hierarchical components; ignored when theta_component @@ -136,6 +167,12 @@ theta_component: "thermo.O2_C12_K5_U0_a.PK" # Each block defines one set of growth conditions. The Cartesian product of # titrant_conc × t_sel gives one sample per entry. condition_pre and # condition_sel must also appear as keys in the 'growth' section below. +# +# To copy a real experiment's layout instead (irregular times, replicates +# sampled differently), replace condition_blocks with +# design: tube_table.csv +# a tube table in the format tfs-process-counts reads; each replicate then +# gets exactly its own tubes, under the table's names. condition_blocks: - library: "kanR" titrant_name: "iptg" @@ -165,9 +202,23 @@ condition_blocks: # Growth rate parameters # --------------------------------------------------------------------------- -# Linear mapping from operator occupancy (theta) to growth rate k for each -# condition: k = m * theta + b. Every condition_pre and condition_sel that -# appears in condition_blocks must have an entry here, and vice versa. +# Mapping from operator occupancy (theta) to growth rate for each condition, +# in units of 1/minute (times are in minutes). The default linear model is +# k = b + A * m * theta + dk_geno +# An entry can instead set model: power (keys b, a, n) or +# model: saturation (keys kmin, kmax). Every condition_pre and condition_sel +# that appears in condition_blocks must have an entry here, and vice versa. +# A linear entry may add warp_a (> 0) and warp_x0 (default 0.5): growth then +# follows a logistic warp f(theta), f(0) = 0 and f(1) = 1, for a marker whose +# growth is a switch-like (large warp_a) or graded function of occupancy; the +# fit's m_warp, truth written as warp_log_a = log(warp_a) and warp_x0. +# kanR+kan: {m: -0.009933, b: 0.0, warp_a: 8.0, warp_x0: 0.5} +# +# growth_titrant_scale (s, default 1): growth sees each genotype's curve at +# s times the nominal titrant concentration, binding at the nominal one (the +# fit's theta_growth_log_conc_shift = log s, hill_geno). Truth in +# {out_prefix}_growth_titrant_scale.csv. +# growth_titrant_scale: 0.5 growth: kanR+kan: {m: -0.009933, b: 0.000000} kanR-kan: {m: 0.000808, b: 0.004670} @@ -190,7 +241,10 @@ dk_geno_hyper_shift: 0.02 # by inference convention. Set activity_mut_scale > 0 to draw each mutant # genotype's log(A) independently from Normal(log(activity_wt), activity_mut_scale). # activity_mut_scale=0 (default) gives all genotypes identical activity, matching -# the 'fixed' activity inference component. +# the 'fixed' activity inference component. activity_component (default +# 'fixed') can instead be 'hierarchical_geno' or 'horseshoe_geno', numpy +# versions of the fit's priors, tuned by an activity_priors dict; both ignore +# activity_wt and activity_mut_scale. activity_wt: 1.0 activity_mut_scale: 0.0 @@ -214,29 +268,45 @@ library_mixture: spiked: 4 # Sigma of the log-normal used to draw initial genotype frequencies. 0 = flat -# distribution; 1–2 is realistic for a typical transformation. +# distribution; 1–2 is realistic for a typical transformation. One number, or +# one per tile combo (e.g. {double-1-2: 2.0, single-1: 0.8, single-2: 0.8, +# spiked: 0.5}; a real pool of doubles was much less even than its singles). lib_assembly_skew_sigma: 1.25 +# Optional: cap on the number of multi-plasmid (congressed) cells simulated one +# by one; more are represented by a weighted sample of this many. Single- +# plasmid cells are always merged into one row per genotype, so transform_sizes +# can be realistic (tens of millions) without this cap only when congression is +# off. Omit for no cap. +# max_congressed_cells: 2_000_000 + # Each bacterium receives a number of plasmids drawn from a zero-truncated # Poisson with this lambda. Set to 0 or null for exactly one plasmid per cell. transformation_poisson_lambda: 1.5 -# When a cell carries multiple plasmids, growth rates are combined via this -# function. Recommended: 'min' for repressors of antibiotic resistance (tightest -# binder dominates); 'max' for inducers. Default: 'mean' for all libraries. -multi_plasmid_combine_fcn: - kanR: "min" - pheS: "max" - -# Colony-forming units per mL at the start of pre-growth. +# How a cell carrying several plasmids gets its growth rate. The cell's theta +# comes from congression_theta_rule, at each condition: 'homodimer' +# (logit theta_cell = log of the share-weighted mean of the plasmids' +# theta odds), 'heterodimer' (closer to the mean logit) or 'max' (the +# highest-theta plasmid sets theta and TF activity). 'homodimer' and +# 'heterodimer' need activity 1 (activity_mut_scale 0). Its dk_geno comes +# from congression_dk_rule, the soft-min family +# dk_cell = -(1/alpha) log sum_g x_g exp(-alpha dk_g): 'dilution' +# (alpha -> 0, the share-weighted mean), 'softmin' (finite alpha, set by +# congression_dk_alpha, in units of 1/dk) or 'min' (alpha -> inf, the worst +# variant sets the cost). Growth is then computed once from these cell-level +# values. All three keys are optional; these are the defaults (no alpha). +congression_theta_rule: homodimer +congression_dk_rule: dilution + +# Cells in each tube at the start of pre-growth. cfu0: 65_000_000.0 -# Per-tube environmental growth-rate noise (hr⁻¹). Each physical tube gets its -# own delta_k ~ Normal(0, tube_noise_sigma); the kt contribution per tube is -# delta_k * (t_pre + t_sel), shared by all genotypes in that tube. A value of -# ~0.001–0.002 hr⁻¹ is typical for well-controlled liquid cultures. -# Set to 0 or null to disable. +# Per-tube environmental growth-rate noise, in growth-rate units (1/minute +# here). Each physical tube gets its own delta_k ~ Normal(0, tube_noise_sigma); +# the kt contribution per tube is delta_k * (t_pre + t_sel), shared by all +# genotypes in that tube. Set to 0 or null to disable. tube_noise_sigma: 0.002 # --------------------------------------------------------------------------- @@ -251,6 +321,9 @@ tube_noise_sigma: 0.002 # tau0 = baseline lag (minutes) # k1 = occupancy-dependence of lag (higher = more effect) # k2 = curvature constant (> 0; prevents division by zero) +# "baranyi" model: rate switches smoothly from k_pre to k_sel around +# tau_lag (minutes) with sharpness k_sharp (> 0) +# "two_pop" model: cells switch irreversibly to k_sel at rate k_trans (> 0) growth_transition: - condition_pre: "pheS-4CP" model: "memory" @@ -268,27 +341,88 @@ growth_transition: # Data collection parameters # --------------------------------------------------------------------------- -# Total sequencing reads across all samples. +# Total sequencing reads across all samples (all replicates together), split +# evenly over the sequenced tubes. total_num_reads: 25_000_000 +# Optional: share of each tube's reads that call no library genotype and are +# reported as __unknown__ (one number, or one per library). Real libraries +# can lose a large share this way, e.g. to structural variants. +# unassigned_read_fraction: {kanR: 0.66, pheS: 0.2} + # Fraction of reads re-assigned to a randomly chosen genotype (index hopping). # Set to 0 or null to disable. prob_index_hop: 0.05 -# Random seed for reproducibility. Set to null for a random seed each run. +# Random seed for reproducibility. Set to null for a random seed each run; +# tfs-simulate --seed overrides it. seed: null +# --------------------------------------------------------------------------- +# Sampling noise (optional; all off when omitted) +# --------------------------------------------------------------------------- + +# Real read counts vary 5-18x more than Poisson at 100-3,000 reads +# (planning/studies/noise-anatomy/). These switches add the physical sources; +# each can be turned on alone to measure what it does to a fit. +# +# founder_sampling: each tube is seeded with a Poisson number of cells from +# each transformant clone (mean cfu0 * frequency), drawn anew for every +# tube, so a rare genotype's starting abundance differs between tubes. +# demographic_growth: given its founders, a clone's cells at harvest are +# Gamma(founders, exp(kt)) when it grows and Binomial(founders, exp(kt)) +# when it shrinks. Needs founder_sampling. +# shared_transformation: every replicate draws its cells from one library +# assembly and transformation (one glycerol stock), as in the real +# protocol, instead of redrawing them per replicate. +# pcr_template_molecules: reads are drawn from this many template molecules +# per tube. The count variance grows by about +# (reads per tube / templates) * (1 + pcr_amplification_cv^2). +# pcr_amplification_cv: CV of each template's amplification (PCR +# jackpotting). Needs pcr_template_molecules. +# +# founder_sampling: true +# demographic_growth: true +# shared_transformation: true +# pcr_template_molecules: 2_000_000 +# pcr_amplification_cv: 1.0 + +# One OD600 reading per tube, from its true total through an OD600-to-CFU +# calibration (the one in ../od600/ is synthetic; a lab makes its own with +# tfs-calibrate-od600). Writes tfs_sim_od600.csv with every tube of every +# replicate. +# calibration: calibration YAML written by tfs-calibrate-od600 +# tube_volume_mL: tube volume; the calibration is per mL (5.0) +# num_od_only_replicates: extra replicates with OD600 but no reads (0) +# sample_cfu_from_od600: give the pipeline each tube's total as estimated +# from its OD600, as the lab does, instead of the +# true total (false) +# +# od600: +# calibration: ../od600/od600_calibration.yaml +# tube_volume_mL: 5.0 +# num_od_only_replicates: 1 +# sample_cfu_from_od600: true + # --------------------------------------------------------------------------- # Binding data simulation (optional) # --------------------------------------------------------------------------- -# If present, tfs-simulate also writes a binding CSV with per-genotype theta -# curves. titrant_name must match e_name in observable_calc_kwargs. +# If present, tfs-simulate also writes a binding CSV (tfs_sim_binding.csv) +# for tfs-configure-model --binding_df. noise is the Gaussian SD on +# theta_obs, not clipped to [0, 1] (clip_theta_obs: true restores clipping). +# choose_by can also be the path to a CSV of Hill parameters (genotype, +# theta_low, theta_high, log_hill_K, hill_n) that become those genotypes' +# true phenotypes; it needs a Hill theta component. # # binding_data: # titrant_name: iptg # titrant_conc: [0, 0.0001, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0] # noise: 0.02 +# # One offset per binding genotype added to its whole curve, +# # Normal(0, genotype_offset_sd) (the fit's genotype_offset); truth in +# # {out_prefix}_binding_offsets.csv. Default 0. +# genotype_offset_sd: 0.1 # # Clean, monoclonal controls (pool = spiked_seqs). # # choose_by: | stratified | random ; num optional (no file) # spiked_binding: @@ -308,8 +442,7 @@ seed: null # reference-condition growth-rate "measurements" for a subset of genotypes, # tied to the per-genotype dk_geno already assigned above and a shared # reference rate k_ref. Feeds tfs-configure-model's --base_growth_df, which -# anchors the hierarchical model's dk_geno/condition_growth identifiability -# (see CLAUDE.md). +# anchors the hierarchical model's dk_geno/condition_growth identifiability. # # k_ref (required): the reference (wt) growth rate, hr^-1. # genotypes (optional, default ["wt"]): genotypes to include. Every genotype @@ -353,8 +486,8 @@ seed: null # presplit_data: # # ---- What the presplit CSV represents ------------------------------------- -# This simulates the sequencing aliquot taken at t = -t_pre, just before -# the culture is split into titrant-concentration conditions. Because it is +# This simulates the sequencing sample taken at t = -t_pre, just before +# the culture is split into tubes. Because it is # taken before any pre-selection growth, each ln_cfu value directly # constrains ln_cfu0 in the hierarchical model. # diff --git a/examples/simulate/simulate_grid.yaml b/examples/simulate/simulate_grid.yaml index a842023b..c025c1d2 100644 --- a/examples/simulate/simulate_grid.yaml +++ b/examples/simulate/simulate_grid.yaml @@ -2,12 +2,12 @@ # Example grid YAML for tfs-setup-sim-grid # # Usage: -# tfs-setup-sim-grid simulate_grid.yaml --out_prefix my_sim_grid +# tfs-setup-sim-grid simulate_grid.yaml --out_dir my_sim_grid # -# This creates one subdirectory per combination, writes a tfs_sim_config.yaml -# in each (base config + overrides), and renders run.sh with the template -# variables. Run all simulations with: -# for d in my_sim_grid/run_*/; do cd "$d" && bash run.sh && cd -; done +# This creates one subdirectory per combination (run__), +# writes a tfs_sim_config.yaml in each (base config + overrides), and renders +# run.sh with the template variables. Run all simulations with: +# for d in my_sim_grid/run_*/; do (cd "$d" && bash run.sh); done # --------------------------------------------------------------------------- # Base simulate config. Each run inherits all settings from this file; the @@ -27,8 +27,9 @@ output_file: run.sh # in the same examples/simulate/ directory # simulate blocks # # Variables here override top-level keys in the base config. They are NOT -# injected into the Jinja2 template. Paths are resolved relative to this -# YAML and re-expressed relative to each subdirectory in the written config. +# injected into the Jinja2 template. Relative file paths are resolved against +# this YAML; every input file is copied into /inputs/ and each run +# refers to it as ../inputs/, so the grid directory can be moved. # # Multi-key variants always move together and are never split by the Cartesian # product — use this to keep parameters that are only meaningful in combination. @@ -44,7 +45,8 @@ simulate: # --- Theta component (thermodynamic model) -------------------------------- # - # Any key in model_registry["theta"] except "_simple" is valid. + # hill_geno, hill_mut or a thermo.* partition-function key (PnnC and PddG + # variants also need thermo_data). # Uncomment additional variants to compare thermodynamic models. - name: thermodynamic_model @@ -61,7 +63,8 @@ simulate: # --- Random seed --------------------------------------------------------- # - # Varying the seed with the same other settings tests reproducibility. + # 'seed' is the simulate config's seed key. Varying it with the other + # settings fixed gives independent simulated data sets. - name: seed variants: diff --git a/examples/tfmodel/grid.yaml b/examples/tfmodel/grid.yaml index 21775e41..d1fb7a0c 100644 --- a/examples/tfmodel/grid.yaml +++ b/examples/tfmodel/grid.yaml @@ -2,12 +2,16 @@ # Example grid YAML for tfs-setup-grid # # Usage: -# tfs-setup-grid grid.yaml --out_prefix my_grid +# tfs-setup-grid grid.yaml --out_dir my_grid # -# This creates one subdirectory per combination, calls tfs-configure-model -# inside each, and renders run.srun with the template variables. +# This creates one subdirectory per combination, named +# run__ (run_0001_linear__instant__hill_geno__seed0, ...), +# calls tfs-configure-model inside each, and renders run.srun with the +# template variables. The data paths below are placeholders: point them at +# your own tfs-process-counts / binding tables and library YAML first, since +# setup fails on a missing input file. # Run all jobs with: -# for d in my_grid/run_*/; do cd "$d" && sbatch run.srun && cd -; done +# for d in my_grid/run_*/; do (cd "$d" && sbatch run.srun); done # --------------------------------------------------------------------------- # Directory name for each run. Variables from both configure_model and @@ -24,7 +28,8 @@ output_file: run.srun # # Variables here are forwarded to tfs-configure-model. They are NOT # injected into the Jinja2 template. Paths are resolved relative to this -# YAML and re-expressed relative to each subdirectory in the written config. +# YAML; each input file is copied into /inputs/ and the written +# config refers to the copy, so the grid directory can be moved as a unit. # --------------------------------------------------------------------------- configure_model: @@ -37,16 +42,20 @@ configure_model: growth_df: ../data/growth.csv # The library description -- the same YAML used for tfs-process-fastq. # Required whenever growth_df is given. - library_config: ../data/run_config.yaml + library_config: ../data/library_config.yaml - name: fixed_components variants: - - activity: horseshoe_geno + # The mixture's default congression_theta_rule, homodimer, needs + # activity 'fixed' (activity 1 for every genotype); a learned activity + # (hierarchical_geno, horseshoe_geno, ...) needs + # congression_theta_rule: max. + - activity: fixed dk_geno: hierarchical_geno - transformation: empirical + transformation: mixture # (mean, std) of the experimentally measured congression lambda, in - # linear space -- required whenever transformation is 'empirical' - # or 'logit_norm' (transformation and transformation_lambda always travel + # linear space -- required whenever transformation is 'mixture' + # (transformation and transformation_lambda always travel # together, hence the joint variant here). transformation_lambda: [0.3572, 0.13] theta_rescale: passthrough @@ -55,9 +64,10 @@ configure_model: # # 'auto: condition_growth' expands to one variant per entry in the # model_registry["condition_growth"] registry (currently: linear, - # power, saturation). Incompatible combinations (e.g. - # power + logit theta_rescale) are caught by tfs-configure-model, - # skipped automatically, and logged in grid_summary.json. + # power, saturation). Incompatible combinations are caught by + # tfs-configure-model, skipped automatically, and logged in + # grid_summary.json. 'auto: theta' would include the private '_simple' + # component used only by the pre-fit; list theta variants by hand. - name: condition_growth auto: condition_growth @@ -104,8 +114,8 @@ template: - seed: 0 - seed: 1 - # Path to the genotypes file for tfs-predict-theta. - # Use an absolute path, or a path relative to each run subdirectory. + # Path to the genotypes file for tfs-predict-theta (absolute, or relative + # to this YAML). It is copied into /inputs/ like any input. - name: predict_genotypes variants: - predict_genotypes_file: /path/to/predict_genotypes.txt diff --git a/examples/tfmodel/growth_priors_loose.csv b/examples/tfmodel/growth_priors_loose.csv new file mode 100644 index 00000000..4cfb4e48 --- /dev/null +++ b/examples/tfmodel/growth_priors_loose.csv @@ -0,0 +1,5 @@ +condition_rep,k_loc,k_scale,m_loc,m_scale +kanR+kan,0.015,0.01,0.0,0.01 +kanR-kan,0.015,0.01,0.0,0.01 +pheS+4CP,0.015,0.01,0.0,0.01 +pheS-4CP,0.015,0.01,0.0,0.01 diff --git a/examples/tfmodel/run.srun b/examples/tfmodel/run.srun index ea86e6e5..04ebfbeb 100644 --- a/examples/tfmodel/run.srun +++ b/examples/tfmodel/run.srun @@ -33,16 +33,17 @@ RUN_SEED={{ seed }} PREDICT_GENOTYPES_FILE="{{ predict_genotypes_file }}" # --------------------------------------------------------------------------- -# Run prefit calibration +# Pre-fit calibration: pins each condition's growth k and m in the priors CSV # --------------------------------------------------------------------------- echo ">>> Prefit calibration" tfs-prefit-calibration tfs_configure_config.yaml \ - --seed ${RUN_SEED} \ - --convergence_tolerance 0.00001 + --seed ${RUN_SEED} # --------------------------------------------------------------------------- -# MAP fit +# MAP fit. Writes tfs_fit_model_checkpoint.pkl, _params.npz, _losses.txt and +# _convergence.csv. Step-size cuts and the stop rule use the defaults +# (tfs-fit-model --help lists them). # --------------------------------------------------------------------------- echo ">>> MAP fit" @@ -50,23 +51,22 @@ tfs-fit-model \ tfs_configure_config.yaml \ --seed ${RUN_SEED} \ --analysis_method map \ - --adam_step_size 1e-6 \ - --convergence_check_interval 100 \ - --convergence_window 50 \ - --checkpoint_interval 100 \ - --max_num_epochs 100000000 \ - --pre_map_num_epoch 100000 \ - --convergence_tolerance 0.0005 \ - --patience 5 + --checkpoint_interval 100 # --------------------------------------------------------------------------- -# Sample posterior +# Sample posterior. A MAP checkpoint becomes a Laplace posterior: 'auto' +# takes the full Laplace up to --laplace_max_params MAP parameters and the +# arrowhead Laplace above. --skip_growth_observations leaves the stored +# per-observation growth sites out of the file (most of its size); +# tfs-predict-growth recomputes growth from the parameter samples. # --------------------------------------------------------------------------- echo ">>> Sample posterior" tfs-sample-posterior \ tfs_configure_config.yaml \ tfs_fit_model_checkpoint.pkl \ + --laplace auto \ + --skip_growth_observations \ --sampling_batch_size 10 \ --num_posterior_samples 1000 \ --seed ${RUN_SEED} @@ -93,5 +93,7 @@ tfs-predict-theta \ echo ">>> Categorize response" tfs-cat-response \ - tfs_theta_pred.csv \ + tfs_pred_theta.csv \ + titrant_conc \ + --group_by titrant_name \ --num_workers -1 diff --git a/examples/tfmodel/two_stage.yaml b/examples/tfmodel/two_stage.yaml new file mode 100644 index 00000000..0427ac77 --- /dev/null +++ b/examples/tfmodel/two_stage.yaml @@ -0,0 +1,47 @@ +# Example configuration for tfs-fit-two-stage (planning/two-stage-fit.md). +# +# tfs-fit-two-stage two_stage.yaml --out_dir two_stage +# +# Top-level keys are tfs-configure-model's arguments, under the same names; +# file paths are relative to this file. The two-stage fit needs the +# growth-only relative model (theta_model: hill_relative, linear growth) +# with one k and m per condition (growth_shares_replicates: true). It sets +# the transformation itself: stage 1 fits the congression mixture, stage 2 +# does not. + +# --- Data --- +growth_df: tfs_growth.csv # tfs-process-counts output +library_config: library_config.yaml # the library YAML used to call the counts + +# --- Model (as for tfs-configure-model) --- +theta_model: hill_relative +growth_shares_replicates: true +batch_size: 4096 +allow_missing_spikes: true +growth_priors: growth_priors_loose.csv # loose per-condition k/m priors for stage 1 +set_priors: + - sigma_fixed=0.17 + - theta_log_hill_n_hyper_scale_fixed=0.5 +skip_model_stats: true + +# --- Two-stage settings --- +two_stage: + # Congression lambda prior (mean, std), required: the measured value. For + # the dev-data screen, 0.357 with a likelihood-ratio 95% CI of 0.156-0.674, + # about 0.38 +- 0.15 as a mean and SD. + transformation_lambda: [0.38, 0.15] + seed: 1 + num_posterior_samples: 500 + consistency: true # stage 2 on one library alone; --no_consistency skips + consistency_library: kanR # default: the first library alphabetically + predict_epistasis: false + # Experimental: k's uncertainty in X (jackknife + k-shift refits). Off: + # on simulations the shift came out a third or less of the true one + # (planning/studies/two-stage-validation/, run 2). + kshift: false + jackknife_subsamples: 30 # stage 1 refits for k's uncertainty + jackknife_drop_frac: 0.2 # share of tubes dropped per refit + kshift_step_size: 1.0e-5 # the k-shift refits' starting step size + kshift_control: true # a third refit with k unchanged; kshift/drift.csv + fit: {} # extra tfs-fit-model arguments + posterior: {} # extra tfs-sample-posterior arguments diff --git a/examples/tfmodel/two_stage_step.srun b/examples/tfmodel/two_stage_step.srun new file mode 100644 index 00000000..ab65487e --- /dev/null +++ b/examples/tfmodel/two_stage_step.srun @@ -0,0 +1,38 @@ +#!/bin/bash -l +#SBATCH --account=YOUR_ACCOUNT +#SBATCH --partition=gpu +#SBATCH --time=00-06:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# One step of tfs-fit-two-stage, submitted by two_stage_submit.sh, which +# sets the job name, log files, dependencies and (for the jackknife) the +# array. Edit the account, partition and time above for your cluster, and +# the module line below for its CUDA. +# +# CFG the two-stage YAML (absolute path) +# OUT the output directory (absolute path) +# STEP stage1 | stage2 | predict | consistency | summary, and with +# two_stage.kshift jackknife | kshift_predict +# (jackknife runs task SLURM_ARRAY_TASK_ID) + +module load cuda/12.4.1 +set -e +CFG="${CFG:?set CFG}" +OUT="${OUT:?set OUT}" +STEP="${STEP:?set STEP}" +echo ">>> ${STEP} ${SLURM_ARRAY_TASK_ID:-} $(date)" +case "${STEP}" in + jackknife) + tfs-fit-two-stage "${CFG}" --out_dir "${OUT}" \ + --jackknife_task "${SLURM_ARRAY_TASK_ID:?run as an array}" ;; + kshift_predict) + tfs-fit-two-stage "${CFG}" --out_dir "${OUT}" --step kshift + tfs-fit-two-stage "${CFG}" --out_dir "${OUT}" --step predict ;; + *) + tfs-fit-two-stage "${CFG}" --out_dir "${OUT}" --step "${STEP}" ;; +esac +echo ">>> Done ${STEP} ${SLURM_ARRAY_TASK_ID:-} $(date)" diff --git a/examples/tfmodel/two_stage_submit.sh b/examples/tfmodel/two_stage_submit.sh new file mode 100644 index 00000000..cba9ba8f --- /dev/null +++ b/examples/tfmodel/two_stage_submit.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Submit tfs-fit-two-stage as chained SLURM jobs: +# +# bash two_stage_submit.sh two_stage.yaml out_dir +# +# stage1; then stage2; then predict and consistency (after stage2); then +# the summary table (after both). With two_stage.kshift true (experimental) +# the jackknife array runs beside stage2 and kshift runs before predict. Logs go to out_dir/logs/. Every path is made +# absolute, so it runs from any directory. Each step skips finished work, +# so submitting the chain again resumes it. Set JACKKNIFE_PARALLEL to run +# more or fewer jackknife refits at once (default 10). +set -e +CFG="$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +OUT="$(mkdir -p "$2" && cd "$2" && pwd)" +HERE="$(cd "$(dirname "$0")" && pwd)" +SRUN="${HERE}/two_stage_step.srun" +N="$(python -c "import yaml,sys; c=yaml.safe_load(open(sys.argv[1])); print((c.get('two_stage') or {}).get('jackknife_subsamples', 30))" "${CFG}")" +KSHIFT="$(python -c "import yaml,sys; c=yaml.safe_load(open(sys.argv[1])); print(bool((c.get('two_stage') or {}).get('kshift', False)))" "${CFG}")" +mkdir -p "${OUT}/logs" +sub() { # name dependency extra-args... + local name=$1 dep=$2; shift 2 + sbatch --parsable --job-name="ts_${name}" \ + --output="${OUT}/logs/${name}_%A_%a.out" --error="${OUT}/logs/${name}_%A_%a.err" \ + ${dep:+--dependency=afterok:${dep}} \ + --export=ALL,CFG="${CFG}",OUT="${OUT}",STEP="${name}" "$@" "${SRUN}" +} +a=$(sub stage1 "") +c=$(sub stage2 "${a}") +if [ "${KSHIFT}" = "True" ]; then + b=$(sub jackknife "${a}" --array=1-${N}%${JACKKNIFE_PARALLEL:-10}) + d=$(sub kshift_predict "${b}:${c}") +else + d=$(sub predict "${c}") +fi +e=$(sub consistency "${c}") +f=$(sub summary "${d}:${e}") +echo "stage1 ${a}; stage2 ${c}${b:+; jackknife ${b} (1-${N})}; predict ${d}; consistency ${e}; summary ${f}" diff --git a/planning/.gitignore b/planning/.gitignore new file mode 100644 index 00000000..ca57e91d --- /dev/null +++ b/planning/.gitignore @@ -0,0 +1,2 @@ +history +studies/fetch.sh diff --git a/planning/README.md b/planning/README.md new file mode 100644 index 00000000..4ed71505 --- /dev/null +++ b/planning/README.md @@ -0,0 +1,92 @@ +# planning/ + +Three kinds of things live here, kept together so they can link to each +other: + +- **Ideas** (`status: idea`): tasks that matter but are deliberately out of + scope for current work. Filing an idea here means "we decided not to do + this now," not "we will do this." +- **Plans** (`status: active`): the few agreed multi-step plans being worked + now. Keeping them next to the ideas they spin off means an idea's `related` + link and a plan's step list can point at each other. +- **Studies** (`studies/`): the numerical or simulation work that answers a + question a plan or idea raised. See "Studies" below. + +Ideas and plans are one Markdown file each at the top level; the status in the +header tells them apart. (This directory was called `future/` until +2026-09-23.) + +## Why a directory and not issues + +GitHub issues are the usual home for a backlog, but they live outside the +repository: they aren't versioned with the code they describe, can't be read +offline, and can't be read by tools working in a checkout. These files are +plain Markdown with a small YAML header, so both people and scripts can list +and filter them. + +## File format + +Name files with short kebab-case slugs (`estimate-dk-alpha-by-varying-lambda.md`). +Start each file with this header: + +```yaml +--- +title: One-line name of the idea +status: idea # idea | active | promoted | dropped | done +filed: 2026-09-13 # date filed (YYYY-MM-DD) +area: tfmodel # tfmodel | simulate | analysis | process_raw | experiment | docs +revisit_when: >- + The condition under which this becomes worth doing. +related: # repo paths or other planning/ files; may be empty + - src/tfscreen/simulate/selection_experiment.py +--- +``` + +Then the body, with these sections (an `active` plan may use its own +structure, such as assumptions, decisions and ordered steps): + +- **Context:** what we were doing when the idea came up. +- **Idea:** what to do. +- **Why not now:** the reason it was deferred. +- **What it would take:** rough scope, dependencies, open questions. + +## Lifecycle + +- `idea`: filed and waiting. +- `active`: an agreed plan being worked now. Keep its step list current. +- `done`: an active plan that is finished. Add a line saying where the result + lives. +- `promoted`: work started. Add a line saying where it went (branch, changelog + entry, plan). Keep the file as a record. +- `dropped`: decided against. Add a line saying why. Keep the file so the idea + isn't re-proposed without that context. + +To list open ideas: + +```bash +grep -l "^status: idea" planning/*.md +``` + +## Studies + +A study answers one question that a plan or idea raised, and its result feeds +a decision. Each study gets its own directory, `studies//`, holding at +least: + +- `README.md` with: + - **Question** and the **decision it fed**, linking the plan step. + - **How to run** it, from a scratch directory. + - **Inputs**: repository paths only, with fixed seeds. + - **Commit** the recorded results were produced on. + - **Results**: the numbers the decision rests on. +- the script(s) that produce those results. + +Rules: + +- The plan records the decision and a short summary; the study README keeps + the full numbers. Link both ways. +- Generated outputs (CSVs, figures) are not committed unless small and cited; + the README's results are the record. +- Once a plan cites a study, freeze it. If the question changes, write a new + study rather than rewriting the old one, so the citation stays valid. +- `dev/` stays untracked scratch space. Anything a plan cites moves here. diff --git a/planning/absolute-abundance-measurements.md b/planning/absolute-abundance-measurements.md new file mode 100644 index 00000000..61d3a681 --- /dev/null +++ b/planning/absolute-abundance-measurements.md @@ -0,0 +1,116 @@ +--- +title: Measurements that fix each tube's total population, especially under strong selection +status: idea +filed: 2026-09-26 +area: experiment +revisit_when: >- + Now for the few bench experiments the current dataset can still use + (they feed roadmap steps 5-6); again when the next screen is designed + (earlier timepoints, longer time base). +related: + - planning/analysis-roadmap.md + - docs/source/process-raw.rst + - src/tfscreen/tfmodel/generative/observe/base_growth.py +--- + +**Context:** The analysis roadmap (`planning/analysis-roadmap.md`, "Reads and +totals") builds each tube's total population `N_s` into the fit as a smooth +curve observed through OD600. Reads measure only frequencies, so the total is +the sole source of the absolute scale, and with it the condition growth rates +`k_c` and, in the relative fit, `m_c`. Two gaps (user, 2026-09-26): + +- **OD600 has no signal under the strongest selection.** Tubes that barely + grow (particularly kan) stay below the reader's detection threshold + (OD 0.096) at the timepoints we have. Censoring turns such a reading into + an upper bound, not a measurement. +- **The next screen should sample earlier**, to resolve growth over a longer + time base, and early tubes are below the threshold too. + +What already constrains a blind tube: + +- **The start.** The culture is grown to a presplit OD600, then diluted by + the same factor into every tube, so every tube of a replicate starts from + a known total (calibration plus pipetting error), shared by all of them. +- **Later, visible tubes of the same curve.** A smooth curve through a known + start and later detectable points interpolates the early blind ones. This + covers the future design if late points stay visible. +- **Other columns of the same condition.** `k_c` and `m_c` are shared by all + IPTG columns of a condition and each genotype's X is a smooth Hill curve + across columns, so a total that is visible in some columns constrains the + others through the reads. This fails when every column of a condition is + blind at a timepoint (then `k_c` trades freely with the curves), and it + leans on the Hill shape exactly where the gauge needs it most (wt at 0 and + 1 mM). + +**Idea:** add measurements of the absolute scale that do not depend on +OD600, chosen by what each one pins. + +*For the current dataset (a few targeted experiments):* + +1. **wt monoculture growth rates in each selection condition at the gauge + concentrations** (0 and 1 mM IPTG), in the screen's tube format and + volume, started at the screen's split density (known from presplit OD + and dilution; aminoglycoside killing depends on inoculum density, so + matching it matters), counted by plating over the screen's time window. + Three replicates. In the relative fit this measures `k_c` (wt at 1 mM) + and `m_c` (wt at 0 minus wt at 1 mM) directly: the gauge stops depending + on the blind tubes. Add one or two controls: a non-binding repressor + (IPTG-independent X, the internal standard of roadmap C4) and, if one + exists, a super-repressor. Needs a new observer (roadmap step 6): the + clean growth rate `k_c + dk_g + m_c X_g(c)` of a named genotype in a + named condition and concentration. Today's `base_growth` observes only + `k_ref + dk_geno` in one reference condition. Caveat: a monoculture + lacks the library's other cells (density, any cross-protection), which + the matched starting density only partly addresses. + **Extended after study 0a (2026-09-26):** run the five genotypes with + in vitro binding data (wt, M42I, H74A, K84L, M42I/H74A) across all eight + IPTG concentrations in kan and 4CP. That measures the growth-binding + map directly, with no library and no totals problem. 0a found the + library data inconsistent with one linear map at the in vitro + concentration scale (kan responds at roughly 30-100x lower IPTG; 4CP + does not track binding), but could not rule out errors in the + per-column totals. +2. **OD600 against plate counts under selection**, a few screen-like tubes + in the strongest conditions at densities the reader can see. Tests + whether the unstressed calibration transfers to stressed cells: dead + cells and filaments scatter light without forming colonies (roadmap + D11). Roughly ten to fifteen plates. +3. **qPCR of total plasmid in the extracted DNA, if any remains.** Relative + plasmid amount per sample, calibrated against the samples whose OD600 was + visible, gives relative totals for the blind ones. Limited by extraction + yield and copy number varying between samples, so it is a weak + constraint, but it reaches where OD600 cannot. Only if the DNA exists. + +*For the next screen (built into the design):* + +4. **A cell spike-in counting standard.** At each pull, before spin-down, + pipette a fixed volume of a quantified frozen stock of cells carrying a + plasmid with the same amplicon and a unique sequence absent from the + library (for example a synonymous-codon wt, the same trick planned for + telling spikes from bulk). Then `N_s = N_std * reads_library / + reads_std`, at any density sequencing can see, with no plates and one + pipetting step per tube. It is co-extracted and co-amplified with the + library, so extraction and PCR differences cancel; the stock is + quantified once, carefully, by plating. Choose `N_std` so the standard + takes about 1-5% of reads. It also cross-checks OD600 wherever both + exist. In the model it is one more "genotype" with a known, non-growing + abundance per tube, and it fixes `P_s` directly. +5. **Flow cytometry with counting beads** (and a live/dead stain), as an + alternative or check: absolute counts well below OD600's floor, and it + separates live from dead cells. Costs instrument time per tube. +6. Keep OD600 for the visible tubes and the presplit; it is free and the + calibration exists. + +**Why not now:** Filed while revising the roadmap; the user will choose which +bench experiments are feasible for the current analysis (a few, not many). + +**What it would take:** + +- Bench time for 1-3 (1 is the most valuable for the relative fit). +- Model: the monoculture growth observer (roadmap step 6); a spike-in + standard is a small addition to the count likelihood (roadmap step 7). +- Simulator: monoculture rates and a spike-in standard as optional outputs, + to size how much each helps before running it (roadmap step 4). +- Open question for 1: which conditions are blind at which timepoints in the + current data (roadmap study 0c answers it), so the experiment covers + exactly those. diff --git a/planning/analysis-roadmap-summary.md b/planning/analysis-roadmap-summary.md new file mode 100644 index 00000000..d138632d --- /dev/null +++ b/planning/analysis-roadmap-summary.md @@ -0,0 +1,477 @@ +--- +title: What the analysis roadmap branch did and found +status: done +filed: 2026-10-02 +area: tfmodel +revisit_when: >- + A record. Revisit when picking up any item under "Where to go from here", + or before a methods section is written from it. +related: + - planning/analysis-roadmap.md + - planning/studies/count-likelihood/README.md + - planning/studies/relative-fit/README.md + - planning/studies/svi-overconfidence/README.md + - planning/studies/noise-anatomy/README.md + - planning/studies/od600-population/README.md + - planning/studies/growth-binding-map/README.md + - planning/studies/real-data-fit/README.md + - planning/experiment-pipeline.md +--- + +Branch `claude/physics-improvements-roadmap-03a273`, 2026-09-26 to +2026-10-02. The plan is `planning/analysis-roadmap.md`; this file is the +outcome. Part 1 is written as a methods narrative, with the reasons and the +decisive numbers. Part 2 lists the bugs caught on the way. Part 3 is where +to go next, and part 4 is what it takes to reproduce the work and what goes +into the merge. + +Numbers about the real screen are summary statistics from +`planning/dev-data/` (gitignored). The data themselves never enter the +repository. + +# 1. Methods + +## Measurement model + +**Reads and totals.** Each tube is one timepoint: it is pulled, read for +OD600, spun down and frozen, and sequenced later. Reads measure +frequencies, and OD600 measures the tube's total. A tube that grew a little +faster than its neighbors scales every genotype and the total by the same +factor, so frequencies do not see it. We therefore modeled reads against +the tube total instead of attaching per-tube noise to every genotype. + +**Count noise.** We measured it before choosing a likelihood +(`planning/studies/noise-anatomy/`). Doubles are 97% of rows, with a median +of 3-5 reads per tube and 23-33% zeros. Per-tube variance was 5-18 times +Poisson at 100-3,000 reads, which points to a bottleneck upstream of the +reads (founder cells or PCR templates) rather than a constant-CV floor. +Today's `ln_cfu_var` understated the noise 5-10 fold. Low counts were +mostly low abundance, not crashes: 88% of double rows under 5 reads in +selective tubes were already under 5 reads without the drug. This study ran +on an older processing snapshot and has not been rerun on the new one. + +**Count likelihood.** We chose to observe reads directly, as +`reads ~ NegBin(mu, mu (1 + phi) + mu^2 inv_r)`, with no pseudocount +(`planning/studies/count-likelihood/`): +- The linear `phi` term matches the bottleneck signature. +- numpyro's own negative binomial lost several nats per observation to + float32 cancellation at 1e4-1e6 reads, so we implemented Loader's + algorithm and tested it against scipy at high depth. +- We paired it with one constant-prior level offset per tube, so that an + error in a tube's supplied total does not move every genotype in it. + +We tested it against the `ln_cfu` Student-t likelihood on 24 simulations: +2 noise models x 3 seeds x 4 fits. On the final run, theta 95% coverage +was 0.83 / 0.84 for counts (Poisson / realistic noise) against 0.72 / 0.83 +for `ln_cfu`. Theta RMSE was 0.035 / 0.072 against 0.100 / 0.109, and +dk_geno coverage was 0.71-0.86 against 0.53-0.55. At a true congression +lambda of 0, counts found 0.02-0.06, where `ln_cfu` found about 0.12, so +most of the spurious congression came from the read floor. Counts with +level offsets became the `tfs-configure-model` default. + +**OD600 calibration.** We ported the lab's OD600-to-CFU notebook into a +generic tool, `tfs-calibrate-od600`. It fits a weighted polynomial and +keeps the full coefficient covariance, because the curve error is shared +by every tube and must not be treated as per-tube noise. The lab's data and +constants stay out of the repository. The port reproduced the notebook's +coefficients to 2e-9 and found that the notebook never divided by the +plated volume, so its totals were 10x low. Growth rates and frequencies are +unaffected; absolute levels are. + +**Simulator realism.** We added the noise sources the data showed, each +switchable so its effect on a fit can be measured: founder sampling, +demographic growth, a shared transformation across replicates, a PCR +template bottleneck and OD600 through an inverse calibration. Matching the +real 5-18x over-Poisson noise needs templates on the order of reads per +tube / 10. + +## The growth-only relative fit + +**Why a relative variable.** A first, model-free look at five spiked +genotypes rejected a single linear map from in vitro binding theta to +growth (reduced chi-square 6.7 in kan; `planning/studies/growth-binding-map/`, +provisional). Growth alone fixes an occupancy-like variable only up to an +affine map, so we fit a wt-relative growth variable X instead of theta +(`theta: hill_relative`). It has Hill shapes with real-valued baselines, +and wt is gauged to X = 1 at 0 IPTG and X = 0 at 1 mM. One X per genotype +and IPTG concentration is shared by kan and 4CP, and each condition's k and +m map it to growth. Then k is wt's rate at 1 mM and m is wt's change +between 0 and 1 mM. The orchestrator refuses combinations the gauge cannot +support: binding data, learned activity, logit rescaling, non-linear growth +and partition-function congression rules. + +**ln_cfu0.** We fit one starting abundance per replicate x pre-condition x +genotype (`ln_cfu0: hierarchical`), because the kanR and pheS libraries are +transformed and grown separately. The factored model, which shares a +genotype's abundance across libraries, pushed the real between-library +difference (SD 0.17-0.29 in simulation) into a confident X error. At more +than 1000 reads, 95% coverage was 0.09 against 0.80, and RMSE was 0.101 +against 0.027. The orchestrator now refuses the factored model across +libraries. + +## Inference: what we tested and chose + +**The m·X ridge.** Growth constrains mostly `m (X_g - X_wt)`. The posterior +is therefore a curved ridge along m·X that only wt's absolute growth change +pins, through the tube totals. Every inference choice below follows from +that. + +**Guides on simulations.** We compared five inference arms on 30 +simulations, after fixing the start, scale and Laplace bugs in Part 2 +(`planning/studies/relative-fit/`, run 4). + +| Arm | X 95% coverage, Poisson / realistic | X RMSE, Poisson / realistic | +|---|---|---| +| Mean-field component guide | 0.06 / 0.38 | 0.30 / 0.29 | +| Low-rank guide (any rank from 20 to numpyro's default) | 0.81 / 0.76 | 0.07 / 0.12 | +| MAP + full Laplace | 0.98 / 0.94 | 0.11 / 0.29 | +| Two-stage fit (refit at Laplace draws of k and m) | 0.91 / 0.78 | 0.08 / 0.11 | + +- The component guide slides along the ridge. +- MAP + full Laplace covered best, with the noisiest point and the widest + intervals. +- Every guide collapsed k and m: k coverage was 0, and m coverage 0-0.25. +- A dense full-covariance guide ran its loss to -1e23 through float32 error + in its own log density. +- NUTS stayed at the maximum tree depth with R-hat 3.5 under every mass + matrix and in float64, so we used calibration over many seeds as the + reference instead. + +**Real data rejected the guides.** On the full library (217,945 +genotypes), both the component and the low-rank guide fit the counts +3.3-3.4e5 nats worse than the MAP, again by sliding along the ridge. +A full Laplace is out of reach there, with about 2.4M parameters. + +**Per-genotype Laplace.** Given the shared parameters (k, m, +hyperparameters, tube offsets), genotypes do not couple. On a simulation's +full Hessian every cross-genotype entry was exactly 0, wt included. So the +Hessian splits into one 9-11 parameter block per genotype, and we get all +the blocks from B Hessian-vector products per genotype chunk. The code +refuses a model whose blocks leak. With the shared parameters held at the +MAP, this block Laplace covered X 0.93 / 0.84 on simulations, with k/m +coverage 0. + +**Arrowhead Laplace.** We then added each genotype's coupling to the shared +parameters and the shared block, at one more Hessian-vector product per +shared parameter. We draw the shared parameters from their marginal, the +Schur complement, and each genotype from its conditional given them. That +is the full Laplace, and on a toy model it matched the dense inverse +Hessian exactly, chunked or not. + +A MAP stopped short of an optimum needed two rules: +- A floored direction of a genotype block carries no coupling. Kept, the + couplings of floored blocks subtracted 6e10 from m's curvature of 1e10. +- A negative direction of the Schur complement is held at the MAP. Floored + at the prior instead, it gave k its prior SD and X widths of 1-3. + +On the simulations it covered X 0.97 / 0.94 (0.95-0.98 above 1000 reads) +and k/m 0.92 / 1.0 (Poisson) and 0.50 / 0.42 (realistic). That is close to +the full Laplace's 0.92 / 0.92 and 0.58 / 0.50, so the realistic-noise +shortfall is not the approximation's. **MAP + arrowhead Laplace is the +full-library route.** On the real library it ran in 20 minutes. + +**Laplace floor.** Each Hessian eigenvalue is floored at the prior's +curvature along its eigenvector, so no direction is wider than the prior. +The old flat 1e-3 floor turned negative eigenvalues into variances of 1000. + +## Fitting the real screen + +**Data.** The screen had 118 tubes after dropping three failed ones, and +22.9M count rows over 217,945 genotypes. +- Tube totals came from OD600 through the calibration. Every screen + reading lay inside the calibrated range. +- Binding curves came from one joint fit of anisotropy to theta and Hill + curves, with per-day levels and robust residuals, interpolated to 10 µM + protein. +- The library has nine spikes. The processing config listed three, and its + M42I codon differed from the spikes'; we corrected both. +- The initial kan and phe samples were swapped, and we swapped them back. + +**Starting point.** A level-offset MAP from a cold start landed in a ±2.8 +offset mode with unphysical k and m. Adam moves every parameter about one +step size per step, so the fast parameters were carried by whole units in +the first window, and the step-size cuts froze them there. We started +instead from the no-offset MAP plus small per-tube offsets +(`tfs-fit-model --init_from`), which was 8.6e5 nats better. The offsets' +SD is held at 0.17, the per-tube OD600 scatter. Learned, it grew until each +tube's offset absorbed the population's growth. + +**Population SDs.** The learned log(hill_n) population SD ran to about 19, +giving n from 0.01 to 13. In 42% of genotypes the curve then changed more +between 0 and 0.1 µM IPTG than across the whole measured range, and the +counts show no such step. We hold that SD at 0.5. Holding X_low, X_delta +and log K as well, at 0.5 / 0.5 / 1.0 against learned 3.6 / 4.3 / 11, is a +sensitivity check (below). + +**Growth priors.** k ~ N(0.015, 0.01) and m ~ N(0, 0.01) per condition, +written by hand, since the pre-fit refuses growth-only models. The wt +monoculture rates as kan priors changed m in the fifth digit. + +**Final fit.** The final fit is `rel_off_n05`. It is growth-only, with the +count likelihood, level offsets at SD 0.17, the n population SD held at +0.5, loose priors and nine spikes, followed by the arrowhead Laplace. It fits +the counts 2.0e4 nats better than the best joint MAP, which used the same +offsets. Without offsets, the growth-only fit was likewise 2.2e4 nats ahead +of the joint one. Its +median X 95% width by genotype read depth is: + +| Reads | 95% width | +|---|---| +| ≤ 100 | 2.6 | +| 1e2-1e3 | 1.2 | +| 1e3-1e4 | 0.69 | +| 1e4-1e5 | 0.42 | +| > 1e5 | 0.33 | + +## Assumptions we checked + +- **Genotype blocks are uncoupled given the shared parameters,** checked on + a full Hessian and guarded by a leak check on every run. +- **The simulator matches the fit model:** growth k + dk + m·A·θ to 2e-16, + θ to the Hill truth to 1.4e-5. +- **Every per-genotype latent is mini-batch safe,** by shape and by order, + registry-wide. +- **The level offsets are not the unassigned-read share** (correlation + -0.21 / -0.41). Closure of the frequencies improved with them (SD 0.16 + to 0.11). +- **The kanR and pheS libraries differ in composition** (Spearman about + 0.1), which supports per-library ln_cfu0. +- **Unselected kan shows no IPTG effect** (wt 1 mM / 0 mM log-ratio about + 0). +- **The split density from the protocol agrees** with extrapolation of the + unselected tubes within 1.5-fold. +- **wt's within-series growth in the screen matches its monoculture** in + kan (IPTG effect 0.013 vs 0.014). In pheS+4CP it does not: library wt + keeps growing at high IPTG where the monoculture stops. + +## What the results are insensitive to + +- **The spike curve shapes, K and n, stay inside their arrowhead Laplace + intervals across:** + - holding all four Hill population SDs (`rel_off_hs`); + - adding the congression mixture (`rel_off_hs_mix`, `max` rule, lambda + prior matched to the measured 0.357 with likelihood-ratio 95% CI + 0.16-0.67); + - the k/m uncertainty. +- **The k and m priors** (loose against monoculture). +- **The SVI guide's rank** (20 against numpyro's default). + +**Sensitive, and under-constrained: m and the absolute X scale.** Holding +the population SDs made every |m| 38% larger and scaled X by 0.77, and the +mixture moved |m| another 21%. These shifts are far outside m's Laplace +interval of ±4-5%. Each step fit the counts about 2.2e4 nats better and +grew the tube offsets. The offsets follow IPTG, in opposite directions in +kan and 4CP. A per-tube offset moves every genotype's predicted frequency +in that tube, so a library-wide IPTG trend can sit in the offsets instead +of in m·X. The Laplace intervals on k and m are therefore conditional on +where the MAP sits on the ridge. We report curve shapes and baselines +relative to wt, not rates. + +## What the fit says about growth and binding + +- **Growth curves are broader than binding curves.** Spike growth n is + 0.6-1.2 (every 95% upper bound ≤ 1.17), against binding n 1.6-2.5. This + holds for clean spikes and bulk-sourced genotypes alike and survived the + congression mixture, so it is not congression. +- **M42I induces below wt in growth,** with K 6.4 µM (95% 4.2-9.6) against + wt 12 µM (8.8-16.7). In binding its K is about 10x wt's, at both 2 and + 20 µM protein. +- **Every mutant falls to a common floor near X -0.5 by 1 mM,** including + weak binders whose binding plateaus sit above it. Taking the floor as + zero occupancy, growth says wt is about 70% induced at 1 mM, where + binding says 93%. +- **No single apparent protein concentration and no shared monotone map + carries binding onto growth.** The best shared concentration left an RMS + of 0.285 in X, and isotonic maps 0.21-0.25 at every concentration. The + departures, 0.3-1.0 in X, are well outside the 0.15-0.35 interval + widths. + +# 2. Bugs caught and fixed + +## Inference and optimization + +- **SVI started far from the pre-MAP point.** The guide's initial scale was + 0.1 in each site's units, plus a 0.1 multiplicative jitter. SVI began + about 1000x above the pre-MAP loss and fell into wrong optima: mirror + modes, halved m. This invalidated the absolute coverage of every earlier + SVI grid. Fix: scale 1e-4, jitter 0 (c57d9ee5). +- **That fix froze the hyperparameter scales.** It put them on their + `greater_than(1e-4)` bound, which is -inf unconstrained, so they never + moved. This affected every component-guide fit from 2026-09-27 to + 2026-09-29, which we reran as relative-fit run 4 and the + svi-overconfidence fix grid. Fix: start above the bound (9db28286). +- **Step-size cuts came mid-descent.** The cut rule judged each window + alone. 20 of 23 first cuts on the count-likelihood v2 grid came during a + descent that pooled to t = 4-14, and one cut led into a wrong optimum. + Fix: pool the stalled windows before any cut or stop (740ebc19). +- **A runaway loss (-8e23) was reported as converged.** Fix: a `diverged` + stop (32a7ec12). +- **NUTS chains started at random,** because no site matched the start + values. Fix (8e244cf0). + +## Laplace and posteriors + +- **The pre-fit took its Hessian at exp(value) of every positive site.** + Constrained and unconstrained values were mixed. phi overflowed and 8 + count fits crashed, and on `ln_cfu` fits the written k scale was 4-9x too + loose. We reran the whole first count-likelihood grid. Fix (c65390aa). +- **The flat 1e-3 Laplace floor** turned negative eigenvalues into + variances of 1000. It caused 5 of 6 Laplace blowups in relative-fit runs + 2-3 and spread m 50x on the two-stage grid. Fix: the prior-curvature + floor (9db28286). +- **The Laplace kept the mini-batch likelihood scale at full batch,** 53x + on the real screen at batch 4096. No result was affected. Fix (cd208d15). + +## Model components and numerics + +- **The factored ln_cfu0 assumed one library.** It moved the kanR/pheS + difference into confident theta errors in every study grid. Fix: refuse + it across libraries and rerun the grids with `hierarchical` (722393cf). +- **The factored ln_cfu0's centered tube offset had an unbounded density,** + so no MAP existed. Fix: non-centered offsets (32a7ec12). +- **The negative binomial concentration underflowed** for near-extinct + genotypes, giving NaN gradients. Fix: a floor (c65390aa). +- **A horseshoe local scale squared to infinity** for low-read doubles. + Fix: overflow-safe helpers (740ebc19). + +## Simulator, processing and I/O + +- **Simulated binding observations were clipped to [0, 1]** against an + unclipped likelihood. That affected 10-17% of anchors. Fix (6e2903d5). +- **`tfs-extract-params` mislabeled ln_cfu0 rows** after any missing + genotype cell. Correlation with presplit was 0.09-0.25 instead of 0.999. + The fits themselves were fine. Fix (079680bd). +- **Prediction failed with per-tube offsets.** Prediction now zeroes them, + giving the typical tube (775f8b79). +- **Absent prior groups reloaded from the priors CSV as NaN.** Fix + (98fb6db0). +- **`read_configuration` silently dropped guesses for sites outside + `init_params`.** This is how the cold-start offset mode arose. Fix: + `--init_from` (faa5b116). + +## Data problems found + +- The OD600 notebook omitted the plated-volume division. +- The initial kan and phe samples were swapped. +- The processing config listed 3 of 9 spikes, with a wrong M42I codon. +- Sample-name forms differed between files. +- One tube was missing from `sample_df`. +- The old snapshot's `presplit.csv` had residue numbering one lower than + `growth.csv`. +- That snapshot's FASTQ processing sent all unknown reads to wt, which is + why the step 0 studies are provisional. + +## Fit pathologies that needed new options + +These were model behaviors, not code bugs: +- runaway Hill population SDs, now held with + `theta_*_hyper_scale_fixed`; +- a learned tube-offset SD that absorbed growth, now held with + `sigma_fixed`; +- a single m prior width for every condition; `m_scale_plus` and + `m_scale_minus` now accept per-condition arrays. + +# 3. Where to go from here + +In rough order of payoff for the science: + +1. **The binding/growth relationship, as its own study.** The facts to + explain are firm: broader growth curves, a common floor, wt only about + 70% induced in vivo, and M42I's K inverted. Candidates: + - cell-to-cell variation in IPTG uptake or repressor level, which + flattens a steep θ(IPTG), testable first on wt's binding titration; + - a full thermodynamic binding model with protein concentration. + + The direct bench test is monoculture growth of the spikes across the + titration (`planning/absolute-abundance-measurements.md`). +2. **The IPTG-patterned tube offsets.** Candidates: + - an OD600-to-CFU relation that shifts under selection (roadmap D11); + - a library-wide growth effect the linear growth model lacks. + + A bench check of OD against plate counts under selection would separate + them, and the population model (roadmap step 6) would carry it. +3. **Pinning the X scale and m, if rates are ever needed.** + - wt monoculture rates as an observer, not just a prior; + - a presplit measurement for this screen; + - a per-tube spike-in counting standard for future screens. +4. **A MAP that reaches its optimum.** Every hierarchical MAP stopped at + its epoch cap, and on real data it is a saddle along the m·X ridge. + - The Laplace needs floor and hold rules for that reason. + - Low-depth genotypes are noisy at the MAP: on simulations, X RMSE was + 1.4 at ≤ 100 reads against 0.22 for the low-rank point. + - Newton steps with the arrowhead Hessian are one route. + - Holding the population SDs shrinks the low-depth genotypes, but only + together with the ridge slide. +5. **Run the pipeline, then simulate the real design at full size** + (`planning/experiment-pipeline.md`; the reviewer simulations in + `planning/studies/real-data-fit/README.md`). +6. **Rerun the provisional step 0 studies** on the new processing: + 0a growth-binding map, 0b noise anatomy, 0c OD600 population. Finish + 0c's curve-form decision (roadmap D9). +7. **Remaining roadmap steps:** + - 2: counts and OD600 through processing; + - 6: the population model; + - 7b: growth transitions, test then purge, with 4CP the likely case; + - 8 and 9: the learned map and the joint fit; + - the kan-vs-4CP consistency check (D5), which a shared X cannot test + alone. +8. **Known calibration gaps on simulations:** + - k/m undercover under realistic noise even with the full Laplace; + - the joint fit's saturated high plateau undercovers (0.31); + - the pre-fit under the count likelihood wrote k priors far from truth. + +# 4. Reproducing this work, and what goes into the merge + +**What reproduces now.** Every package change is committed with tests, and +the CHANGELOG and CLAUDE.md describe it. These studies have tracked scripts +and grids with fixed seeds: +- `count-likelihood` +- `relative-fit` +- `svi-overconfidence` +- `noise-anatomy` +- `od600-population` +- `growth-binding-map` +- `step0-data` + +Their grid outputs are gitignored and live only on the cluster and local +disk. + +**Gaps in the tracked studies, closed 2026-10-03:** +- Each study README's "Commit" section now names the commit each grid ran + on. Where the README had not recorded it, the commit is inferred from the + file times of the downloaded `run.out` files, and the README says so. +- `relative-fit/laplace_blocks.sh block|arrowhead` reproduces the block and + arrowhead Laplace tables from a finished `relative_fit_v4/`. Rerun on + 2026-10-03, the arrowhead mode matched the README's 95% table exactly + (50% coverage within 0.01, from new draws). +- The roadmap's step 5 and 7 entries and the relative-fit decision text are + current, and the relative-fit README opens with the current answer. +- The one-off diagnoses in `svi-overconfidence` (NUTS variants, the plateau + tests, the truth-pinned fit) are marked in its README as records, not + reproducible results. Scripting each one is not worth it. + +**The real-data fit.** Its scripts are now tracked in +`planning/studies/real-data-fit/`, in `processing/`, `fit/` and `analysis/` +(Mike, 2026-10-03). The data, the run outputs and the full lab notebook stay +in the gitignored `planning/dev-data/`, where the scripts are symlinks to +the tracked copies. The study README gives the final fit's start chain as +commands, with the commit behind each run, and lists the dead ends. + +The final fit's start depends on a chain of six earlier runs. We will not +shorten it by rerunning it here. The pipeline plan +(`planning/experiment-pipeline.md`) builds the staged start into +`tfs-fit-model`, and its done criterion is this fit rerun from one +experiment file. That rerun becomes the official, shortened recipe. The +same plan makes the simulator write the raw formats, so full-size +simulations of this design can answer reviewers. + +**The merge itself.** `main` has not moved since 2026-09-26, so the merge +is mechanical. The full suite, slow and smoke tests +included, ran on 2026-10-03: 5,325 passed and 8 smoke tests failed. The 8 +built mock data with `ln_cfu` only and had not been run since the count +likelihood became the configure default (2620a5b3). They now ask for the +`ln_cfu` likelihood their data imply, and all 44 smoke tests pass; the +count path keeps its smoke coverage through the fixture-based tests. The +stale entries are fixed (above). +The roadmap stays `active`: steps 2, 6, 7b, 8 and 9 are not done (Mike, +2026-10-03). diff --git a/planning/analysis-roadmap.md b/planning/analysis-roadmap.md new file mode 100644 index 00000000..5da16549 --- /dev/null +++ b/planning/analysis-roadmap.md @@ -0,0 +1,712 @@ +--- +title: Roadmap for analysis improvements alongside the congression plan +status: active +filed: 2026-09-26 +area: tfmodel +revisit_when: >- + Active now, on a branch parallel to main. Update the step list as steps + finish, and re-check the constraints each time a congression step lands on + main. +related: + - planning/congression-physics-plan.md + - planning/analysis-roadmap-summary.md + - planning/experiment-pipeline.md + - planning/growth-only-relative-fit.md + - planning/count-likelihood.md + - planning/per-sample-level-offset.md + - planning/empirical-mixture-refit.md + - planning/estimate-spike-fraction-from-growth-curvature.md + - planning/estimate-dk-alpha-by-varying-lambda.md + - planning/plasmid-segregation-low-copy.md + - planning/combination-specific-expression-effects.md + - planning/absolute-abundance-measurements.md + - src/tfscreen/process_raw/counts_to_lncfu.py + - src/tfscreen/tfmodel/generative/components/sample_offset/normal.py +--- + +**Outcome (2026-10-02):** `planning/analysis-roadmap-summary.md` records +what this branch built, tested and found, the bugs fixed, where to go next +and how to reproduce it. Steps 2, 6, 7b, 8 and 9 are not done. + +## Why + +The congression plan (`planning/congression-physics-plan.md`) is close to +done on `main` (step 4 is implemented and its cross-rule grid is set up; step +5, the dk rule, is left). Its remaining work is mostly waiting for +simulations. This plan orders the other analysis changes, to be built in +parallel on a second branch, so that they fit together and fit with what +congression still changes. + +The user's priorities (2026-09-26): + +1. **Growth-only, wt-relative fit**, then measure the growth-binding map + empirically (`planning/growth-only-relative-fit.md`). +2. **Low-count genotypes handled rigorously** instead of by pseudocount + (`planning/count-likelihood.md`, with `planning/per-sample-level-offset.md` + as its first half). + +A third item joined on 2026-09-26: **model the total population in each +tube**, from OD600 through a lab-specific calibration, instead of ingesting a +per-tube CFU estimate. Once the fit works from reads, the total is the only +source of the absolute scale, so it has to be modeled at the same level. + +## Reads and totals: where each number comes from + +The experiment (`docs/source/process-raw.rst`, "One tube per time-point"): +after the `presplit` sample, a replicate's culture is split into one tube per +condition x IPTG concentration x timepoint; a timepoint pulls a whole row of +tubes. Each pulled tube gets an OD600 reading, is spun down and frozen, and +is later extracted and sequenced. OD600 is read for three bioreplicates; +two are sequenced. + +For tube s, with true total population `N_s` and genotype populations +`n_{g,s}`: + +- **Reads measure frequencies.** `reads_{g,s} ~ depth_s * n_{g,s} / N_s`, + plus a small per-tube composition offset `u_s` (PCR and genotype-calling + efficiency: the `__unknown__` fraction differs between tubes; the + denominator review measured it at about 0.026 ln units). +- **OD600 measures the total.** `N_s = cal(OD_s) * volume`, where `cal` is + the lab's OD-to-CFU calibration. +- **Tube growth noise cancels from frequencies.** A tube that grew a little + faster or slower than its neighbors (a shared rate shift `eps_s`) scales + every `n_{g,s}` and `N_s` by the same `exp(eps_s)`, so `n_{g,s} / N_s` does + not see it. It shows up only in the OD600 value. + +So the genotype model can predict smooth `n_{g,s}` (no tube term), the +population is a smooth curve `P_s = ln N_s` fitted across tubes, and +`ln_cfu_{g,s} = ln(freq_{g,s}) + P_s` is free of tube noise. This is why the +earlier practice (every tube assigned the pooled CFU of its condition and +timepoint across the three bioreplicates) worked so much better than per-tube +CFU: it removed the tube noise along with the OD noise. The replacement is a +smooth population curve per replicate x condition x concentration, with a +shared shape and per-replicate levels, observed through every OD600 reading, +including the bioreplicate that was not sequenced. + +The curve and the genotype model describe the same cells: +`P_s = logsumexp_g ln n_{g,s}`. That coupling is not enforced directly (it +would need every genotype in every step, which breaks mini-batching). It is +enforced by the data: fitting every genotype's frequency makes +`sum_g exp(ln n_{g,s} - P_s) = 1` up to the `__unknown__` share. + +What stays per tube and genotype is only what does not cancel: counting +noise, PCR jackpotting, and genotype-by-tube differences (a tube whose +selection was slightly stronger treats genotypes differently). + +**Where the total is known and where it is not** (user, 2026-09-26): + +- **Start: known.** The culture grows to a presplit OD600 and is diluted by + the same factor into every tube, so every tube of a replicate starts at + `cal(OD_presplit) * volume / dilution`, one value shared by all of them. + Units: `ln_cfu = ln(cfu/mL * 5 mL)`, the calibration giving cfu/mL. +- **Visible tubes: measured.** OD600 on every pulled tube, same reader, plate + and 200 uL as the calibration. +- **Blind tubes: not measured.** Under the strongest selection (particularly + kan) the population barely grows and OD600 stays below the detection + threshold at the timepoints we have; the next screen wants earlier + timepoints, also below it. Censoring gives only an upper bound. A blind + tube is constrained by its curve's known start and any later visible + points, and through the reads by other columns of the same condition + (`k_c`, `m_c` shared; each genotype's X smooth across columns). When every + column of a condition is blind at a timepoint, `k_c` trades freely with + the curves. Measurements that close this are filed separately: + `planning/absolute-abundance-measurements.md` (a few for the current + data, notably wt monoculture growth rates at the gauge concentrations; a + cell spike-in counting standard for the next screen). + +## Disposition of every idea in `planning/` + +| Idea | Disposition | Where | +|---|---|---| +| `growth-only-relative-fit` | **Track G**, priority 1 | steps 0a, 1, 5, 8, 9 | +| `count-likelihood` | **Track N**, priority 2 | steps 0b, 2, 4, 7 | +| `per-sample-level-offset` | Track N; becomes the population model plus the composition offset `u_s` | step 6 | +| OD-to-CFU at the experimental level (new, 2026-09-26) | Track N | steps 0c, 2, 3, 4, 6 | +| `absolute-abundance-measurements` (new, 2026-09-26) | Bench work, user's choice; the model side is in steps 4, 6, 7 | now (1-3), next screen (4-5) | +| censoring floor rows (congression 3.5 open item) | **Superseded by step 7**; not built (D1) | - | +| lowering the default `binding_weight` (congression open item) | Stays on main; this plan asks it to default to 1 (C7) | main | +| setting lambda from the measurement (congression open item) | Stays on main; independent of this plan | main | +| `empirical-mixture-refit` | Deferred until main's step 5 and this plan's step 5 | after step 9 | +| `estimate-spike-fraction-from-growth-curvature` | Park: future runs tag spikes by codon | - | +| `estimate-dk-alpha-by-varying-lambda` | Experiment; needs main's step 5 | - | +| `plasmid-segregation-low-copy` | Independent; simulator-only; pick up when the low-copy design is planned | any time | +| `combination-specific-expression-effects` | Park: needs bench data | - | + +## Compatibility constraints + +These hold across all steps. Each one names the collision it prevents. + +**C1. Congression rule vs relative X.** Step 4 on main made `homodimer` the +default theta rule. It takes `logit(theta)`, so it needs absolute theta in +(0, 1). The relative growth variable X is real-valued and defined only up to +an affine map, so the partition-function rules do not apply to it (the idea +file's invariance argument holds only for `max`: `max f(X) = f(max X)` for +increasing f). Rule: a relative-X theta component runs with `single` or with +`mixture` under `congression_theta_rule: max`, and the orchestrator refuses +`homodimer`/`heterodimer` with it (the same pattern as the refusal of a +learned activity). The dk rules (dilution now, soft-min after main's step 5) +act on dk in rate units and are unaffected. X must be oriented to increase +with repression for `max` to mean "tightest binder wins". The `max` branch of +`mixture.cell_classes` must not clip X to [THETA_EPS, 1 - THETA_EPS]; check +this in step 5. Once step 9 learns the map f, the partition-function rules +act on `f^-1(X)` and are available again. + +**C2. Observers become selectable.** Track N replaces the growth and presplit +likelihoods. Make the observation model an orchestrator setting written to +the config and carried as static data (`growth_likelihood: lncfu | counts`), +the same pattern as `congression_theta_rule`. Do not use a YAML `components:` +key: observers are not registry components, and every `components:` key +becomes an orchestrator kwarg. `tfs-prefit-calibration`, `tfs-sample-prior` +and prediction then follow the setting with no change of their own. + +**C3. Tube noise lives in the OD600 likelihood, not on genotypes.** A tube is +a timepoint (one sequenced sample, independent of the other timepoints). Its +shared growth deviation cancels from frequencies (see "Reads and totals"), so +no per-tube latent is added to every genotype for it. It appears as scatter +of OD600 about the population curve, with variance `(sigma_tube * t)^2` plus +the OD reading noise, over `t_pre + t_sel` (pre-growth and selection both +happen in the tube, D7). The only per-tube latent on genotypes is the +composition offset `u_s`, with a time-independent prior. Today's +`sample_offset/normal` (a per-tube rate offset on every genotype) becomes +that `u_s` (D2). The simulator's tube noise is already per timepoint (its +`condition_selector` includes `t_sel`). + +**C4. The absolute scale, and with it `k_c` and `m_c`, comes from OD600.** +From reads alone, any per-tube constant can be added to every genotype, which +absorbs `k_condition * t` entirely: reads fix relative growth (dk_geno and +`m * theta` differences), not absolute rates. The population curves fix the +absolute rates. In Track G this matters twice: `k_c` is wt's absolute growth +at 1 mM IPTG, and `m_c` is wt's growth difference between 0 and 1 mM, which +are different tubes (different grid columns), so it rests on the columns' +population curves. Genotypes whose X does not depend on IPTG (broken +repressors) are internal standards across columns and should be reported as a +check. Steps 5 and 6 must report `k_c` and `m_c` recovery, not only theta +coverage. + +**C5. Everything stays mini-batch safe.** The population curve parameters +and `u_s` are global sites (per replicate x condition x concentration, per +tube), like the condition parameters. The count likelihood is per genotype +and scaled by `scale_vector`. Index hopping into tube s comes from the same +genotype in the lane's other tubes, which are in the same batch row. +Sequencing errors that turn one genotype into another would couple +genotypes; leave them in `__unknown__`. The OD600 likelihood does not involve +genotypes and is never scaled. The new theta component follows the registry +rules (library-sized plates, `batch_idx` slicing, `return_population`, +`{site}_loc(s)`/`{site}_scale(s)`), and `test_batch_safety.py`, +`test_component_population.py` and `test_initialization.py` must pass +registry-wide. + +**C6. Process the raw data once.** Before any Track N model work, the +processed files carry: counts, per-tube depth (including `__unknown__`), and +a per-tube OD600 table covering every bioreplicate, sequenced or not, with +the reader's detection flag. The real data is reprocessed once (step 2) and +every later step reads the same files. `ln_cfu`/`ln_cfu_var` stay, for plots +and for the `lncfu` likelihood. + +**C7. Binding at weight 1 when it returns.** Track G removes binding from the +fit; step 9 brings it back as an explicit measurement of the map at weight 1 +with realistic SDs. The binding-weight change on main should therefore make +1 the default, not a smaller ratio. + +**C8. Downstream tools on the X scale.** Logit epistasis, `in_regime` and +anything that assumes theta in (0, 1) do not apply to X. `tfs-predict-theta` +must label its output as X for a relative fit (a column or metadata flag, so +nothing downstream mistakes one for the other); `tfs-extract-epistasis` +allows only `--scale add` on it until f is known. `tfs-summarize-calibration` +needs the simulated truth on the X scale: with growth linear in theta and +the gauge of D4 (`X_wt(c_lo) = 1`, `X_wt(c_hi) = 0`), true +`X_g(c) = (theta_g(c) - theta_wt(c_hi)) / (theta_wt(c_lo) - theta_wt(c_hi))`. + +**C9. Shared files, small diffs.** Main will still touch +`transformation/mixture.py`, `simulate/cell_rules.py`, +`simulate/selection_experiment.py` (cell rules), `model_orchestrator.py` and +`configure_model_cli.py` (a `congression_dk_rule` setting). This branch stays +out of `mixture.py` and `cell_rules.py` except for C1's check, keeps its +orchestrator and CLI changes additive, and syncs with main after every +congression step lands. + +**C10. The method is generic; the calibration is the lab's.** An OD-to-CFU +calibration depends on the plate reader, plate, volume and strain, so no +lab's calibration data or constants go into this repository. The repository +holds the method: a CLI that fits a calibration from a documented input +format and writes a calibration file, the functions that read and apply it, +and an example built from synthetic data. The existing lab calibration +(`tfscreen-notebooks/od600-to-cfu/`, outside the repo) is the first user and +the test that the port reproduces it. A lab that measures totals another +way (plating, flow) supplies per-tube totals with their own error model; +that is a scientific alternative, not a compatibility path (D12). + +**C11. The calibration's own uncertainty is shared by every tube.** The +current `od600_to_cfu` adds the calibration-curve error (`J C J^T`, +approximated by a quadratic in OD) to each tube's CFU variance as if it were +independent per tube. It is one error, shared by every tube in every +experiment calibrated with that curve: it cannot average out across tubes, +and it mostly does not matter for growth rates (a multiplicative error +cancels from slopes; only curvature error does not). The calibration file +therefore keeps the full parameter covariance, and the fit treats the +calibration parameters as global quantities (D10), never as per-tube noise. + +**C12. Count noise scales with the mean, and rare genotypes start from few +cells.** Study 0b found per-tube variance 5-18 times Poisson at 100-3,000 +reads, the signature of a bottleneck upstream of the reads, not a +constant-CV floor. The count likelihood's dispersion must allow +`var = phi * mu` (NB1 / quasi-Poisson), with at most a small NB2 floor. A +typical double has a few founder cells per tube at the split, drawn anew for +every tube, so its starting abundance differs between tubes; the shared +`ln_cfu0` is a mean, and founder noise is part of the per-tube dispersion. +The simulator must produce both (step 4), or its calibration grids +understate real noise. + +## Steps + +Keep this list current. Two tracks, G (growth-only) and N (noise, counts and +totals), interleaved in this order. G comes first by priority; step 8 (the +map) waits for Track N because a real map can look like "no map" when the +posteriors are overconfident (bulk theta 95% coverage was 0.66 in the +congression calibration grid, 0.78-0.87 after convergence fixes). +Breaking changes are allowed throughout (D12); each modeling step ends with +a simulation checked by `tfs-summarize-calibration`. + +**Implementation order (D14, 2026-09-26).** While the refined data are +gathered, build what does not hang on it: 1 -> 4 -> 7 (with supplied +totals) -> 5 -> 3, then 2's OD600 part, 6, 7b, 8, 9 as data arrive. Step 7 +comes before step 6: until the population model exists, each tube's total +is the supplied `sample_ln_cfu` (as today) and the tube offset is +`ln depth_s - sample_ln_cfu_s + u_s`; step 6 later replaces the supplied +totals. The count and depth columns of step 2 come with step 7. The +congression plan's step 6 (re-validate on the count likelihood) waits on +step 7. + +0. **Studies on real data** (no package code; run in parallel; all can start + now). + Data: the 2026-07-23 snapshot, extracted by + `planning/studies/step0-data/` (113 sequenced tubes, 215,402 genotypes). + **The first results are provisional** (user, 2026-09-26): that + snapshot's FASTQ processing assigned every unknown read to wt and used a + very stringent Q cutoff, and its binding data were five genotypes at the + original protein concentration. A new processing batch is running, and + new binding data exist (10 genotypes, two lower repressor + concentrations whose curves match the in vivo concentration range). + Re-run 0a and 0b on those before acting on them. + - [x] **0a. Growth-binding map, model-free** (G's step 0). Per genotype with + binding data: slope of `ln_cfu` on `t_sel` per replicate x condition x + concentration; subtract wt; plot against binding theta (Hill fits if + concentrations differ), spiked and bulk separately. Decides the base + case (one collapsing curve: linear or monotone; genotype-specific scatter: + no map). Also answers the idea file's open question on whether growth + and binding concentrations coincide. Study: + `planning/studies/growth-binding-map/`. + Result (2026-09-26): concentrations coincide (eight IPTG points). Only + five genotypes (all spiked, all near wt in vitro) have binding data. + wt-relative, total-free slopes have no leverage. Absolute slopes (with + the smoothed totals) reject one linear map at s = 1 in kan (reduced + chi2 6.7; genotype-specific slopes p = 2e-5): growth rises between + 0.001 and 0.01 mM IPTG where in vitro theta is still 1. An in vivo + concentration scale s = 30-100 brings reduced chi2 to about 3 (kan m + about -0.025 per min). 4CP growth does not track binding. **The base + case is not supported**; the map needs s (step 8), and the direct test + is monoculture growth of these genotypes across the titration + (`planning/absolute-abundance-measurements.md`). + - [x] **0b. Read noise anatomy.** From the real counts: fraction of rows + below 20 and below 5 reads, by genotype class and condition; the + per-tube composition offset `u_s` (shared residual of frequencies of + well-measured genotypes, wt and spikes, within a tube); a look for + block structure in residual level or dispersion along the likely + processing order (the extraction batches were not recorded, D8); + replicate + spread of high-count spike ratios against the binomial expectation, + which sizes the count dispersion. Sets priors for steps 6 and 7 and the + simulator parameters of step 4. Study: `planning/studies/noise-anatomy/`. + Result (2026-09-26): doubles (97% of rows) have a median of 3-5 reads + per tube (23-33% zero, 72-85% under 20); singles about 5,300; spikes + 29,000-56,000. Composition offset `u_s` SD 0.018. Per-tube count + variance is 5-18 times Poisson at 100-3,000 reads (pooled: variance + = 0.0055 + 8.4 x Poisson): a bottleneck upstream of the reads + (template molecules and/or founder cells), so the dispersion scales + with the mean (C12). Today's `ln_cfu_var` understates the noise 5-10 + fold. No extraction-batch analysis was possible from the recorded + order. Low counts are mostly low abundance, not crashes: about 88% of + double rows under 5 reads in selective tubes belong to genotypes + already under 5 reads without the drug; only 0.1% of doubles with >= 20 + reads without the drug fall below 5 in half their selective tubes. + - [ ] **0c. OD600 anatomy.** (First pass done; waiting on data.) All OD600 readings for the three + bioreplicates, through the lab calibration: the shape of `ln N(t)` per + condition x concentration (pre-growth and selection, lag, any + saturation), how much of the between-bioreplicate difference is a level + (shared split density) and how much a slope; scatter about a smooth + curve against the 2% reading noise (the excess is tube growth noise); + readings below the detection threshold (0.096) or above the calibrated + range (0.59), listed by condition, concentration and timepoint (which + tubes are blind decides which bench experiments are worth running); + whether kan and 4CP tubes differ in ways that suggest the + calibration does not transfer to stressed cells (D11). Decides the + curve form (D9). Study: `planning/studies/od600-population/`. + First pass (2026-09-26, the two sequenced replicates only): replicate + 2 reads +0.24 ln units above replicate 1 (a level; per-condition + levels add nothing), but replicate-specific slopes are also + significant (p = 7e-5). Tube scatter 0.17 ln units against 0.03 + reading noise, not growing with time. No tube below threshold (lowest + 1.5x, kan+ 0 mM, which barely grows). Per-condition population slopes + have SE 0.003-0.007 per min, comparable to `m`: these tubes alone do + not pin `k_c`. Still needed: the third bioreplicate's OD600, the + repeated OD600 runs behind the smoothed totals, and the presplit + OD600 and dilution. +1. [x] **Binding-optional plumbing** (G). + The list in the idea file: `binding_df` becomes `--binding_df` in + `tfs-configure-model` (at least one of growth/binding required; no phantom + `data.binding`); the orchestrator gates binding tensors, `BindingData`, the + auto `binding_weight`, `theta_binding_noise` and `observe_binding` on + `binding_df is None`; `model.py` gates the binding path statically; + `analysis/prediction.py` and `summarize_fit_cli.py` cope (default + trajectory subset: wt, spikes, and a fixed-seed sample); fail fast on + `binding_weight` or non-`zero` binding noise without binding; + `tfs-prefit-calibration` refuses a growth-only config with a clear + message. Update positional callers (`examples/simulate-and-analyze/run.sh`, + the grid templates) and the tests named in the idea file; a growth-only + smoke test on `growth-smoke.csv` + `library-smoke.yaml`. Check that a + joint fit is unchanged (fixed-seed MAP loss against main): not for + compatibility, but to show the plumbing changed nothing it should not. + Done 2026-09-26. Joint model log density at a fixed seed identical to + before (`single` and `mixture`). Also fixed: absent prior groups + (`priors.binding` here, `priors.growth` for binding-only) reloaded from + the priors CSV as NaN; they are now left out and stay `None`. Tests: + `tests/tfscreen/tfmodel/test_growth_only.py`, a growth-only smoke test + in `tests/smoke-tests/test_configure_run_smoke.py`. +2. **Counts and OD600 through processing** (N; breaking). + - `counts_to_lncfu` writes `counts` and per-tube depth (with + `__unknown__`) next to today's columns. The same for + `tfs-process-presplit`. + - A per-tube OD600 input (all bioreplicates, sequenced or not; a + `sequenced` flag or the absence of reads marks the third) and the + calibration file of step 3. Per-tube CFU columns are no longer an input; + totals come from OD600 (or another lab's total measurement, C10). + - An interim smoothed total: `tfs-process-counts` can compute + `sample_ln_cfu` from a population curve fitted to all bioreplicates' OD + (the form from 0c), in place of per-tube or pooled CFU. It generalizes + the pooled practice and gives step 5 a sound total on real data before + step 6 exists. Its per-tube SDs are correlated (one curve), which the + `lncfu` likelihood ignores; step 6 fixes that. + - The orchestrator requires counts, depth and the OD table. Old + processed files are not supported; reprocess the real data (C6) and + regenerate simulations. +3. [x] **Generic OD600 calibration** (N; new CLI; C10, C11). + `tfs-calibrate-od600`: port the notebook's method (technical-replicate + reading noise, detection threshold, errors-in-variables polynomial of CFU + on OD, counting plus pipetting error on the plate counts) with a + documented CSV input format and a configurable polynomial degree. It + writes a calibration YAML with coefficients, the **full** parameter + covariance, reading noise, detection threshold, calibrated OD range and + the volume convention (per mL; the model's `ln_cfu` is per 5 mL sample, + so the tube volume is explicit). Shared read/apply functions in + `process_raw/`. Test: reproduces the lab notebook's constants and CFU + estimates from its spreadsheets (run locally; the data stay out of the + repo), and a synthetic example in `examples/`. + Done 2026-09-27: `tfs-calibrate-od600` (`process_raw/od600.py`, + `scripts/calibrate_od600_cli.py`), documented CSV inputs, a + `kind: od600_to_cfu_per_mL` YAML with the full covariance, and + `examples/od600/`. The simulator's `od600` block reads the new format + (the old one is refused). Run locally on the lab's spreadsheets, it + reproduces the notebook's coefficients to 2e-9, the detection threshold + (0.0957) and the reading noise (1.96%) exactly, given the notebook's + CFU/mL. **But the notebook's CFU/mL omits the division by the plated + volume** (0.1 mL): the sheet's own `CFU_per_mL` column is 10x the value + fitted, so the lab's constants, and every CFU total and `ln_cfu` made + with them, are 10x (ln 10 = 2.3) low. Frequencies, growth rates, theta + and dk_geno are unaffected (a constant factor cancels); `ln_cfu0`, the + presplit level, simulated `cfu0` tuned against real `ln_cfu`, and the + absolute totals the population model (step 6) will use are affected. + The notebook's quadratic approximation of the curve error (`P/Q/R`) is + replaced by the exact `J C J^T`; they differ by up to 1.8x at low OD. +4. [x] **Simulator realism** (N; simulator only). + The simulator has index hopping (`prob_index_hop`) and per-tube growth + noise (per timepoint, matching the design) but writes + `sample_cfu_std = 0.0` and no OD. Add: OD600 per tube through an inverse + calibration (from a calibration YAML) with reading noise, the detection + threshold and the calibrated range; bioreplicates that get OD only (no + reads); founder sampling per tube (each tube seeded with a Poisson + draw of cells per genotype at the split, not `total_cfu0 * freq`; 0b); + a template bottleneck before sequencing (reads drawn from a limited + number of template molecules; 0b); PCR jackpotting (Gamma-Poisson counts with a configured + dispersion); a per-tube composition offset; optional monoculture + growth-rate measurements and a spike-in counting standard, to size how + much each would help before it is run. Each source can be switched off, + so its effect on the fit can be measured (a scientific switch, D12). + Values from 0b and 0c. Needed to validate steps 6 and 7, and step 5 + under realistic noise. + Done 2026-09-26: `founder_sampling`, `demographic_growth`, + `pcr_template_molecules`, `pcr_amplification_cv`, an `od600` block + (`simulate/od600.py`; OD-only replicates, `tfs_sim_od600.csv`, + `sample_cfu_from_od600`), plus `shared_transformation`: every replicate + used to redraw the library assembly and transformation, unlike the real + protocol's single glycerol stock. With all options off the simulator's + output is byte-identical to before. Checked on the example config: the + template bottleneck adds `(reads / templates)(1 + cv^2)` to the + variance/Poisson ratio, as designed; the example config's 260k reads + per tube against 2M templates gives only 1.2-1.8x, so matching the real + 5-18x needs templates on the order of reads per tube / 10 (about 1-2M + at the real ~18M reads per tube). The example config's tubes also reach + OD600 2.9, above a realistic calibrated range; tune `cfu0` and timing + before using its OD600. Deferred: monoculture growth-rate outputs and a + spike-in counting standard (waiting on the bench decisions). +5. [x] **Relative-X fit** (G; new component, opt-in). (Done 2026-09-28; + `planning/studies/relative-fit/`. Point estimates match the anchored + joint fit without binding data when fit with + `--guide_type auto_low_rank_multivariate_normal`. The mean-field + component guide is biased along the m·X ridge. Intervals from every + SVI arm, joint included, are too narrow; that is a separate problem. + Updated 2026-10-02: run 4, after the start, scale and Laplace fixes, + confirmed the guide verdict (component X 95% coverage 0.06 / 0.38, + low-rank 0.81 / 0.76, Poisson / realistic). On the real library the + low-rank guide slid along the ridge (3.3e5 nats worse than the MAP), so + the full-library route is the MAP plus the arrowhead Laplace + (`tfs-sample-posterior --laplace_blocks --laplace_shared`): X 0.97 / + 0.94 and k/m close to the full Laplace on run 4. Real-data results: + `planning/dev-data/real_fit/README.md` (gitignored) and + `planning/analysis-roadmap-summary.md`.) + - A relative Hill theta component: hill_geno's curve with real-valued + baselines, population priors centered on wt, registry rules (C5). Gauge + (D4): wt's curve is pinned to `X = 1` at `c_lo` and `X = 0` at `c_hi` + (per titrant), so wt keeps its K and n free and its baselines follow + from them. `c_lo`/`c_hi` are an orchestrator setting written to the + config, defaulting to the lowest and highest measured concentration + (0 and 1 mM IPTG in the current data). + - One X per genotype x titrant concentration, shared by every condition + (D5); each condition's `k_c` and `m_c` (sign included) map it to growth. + Report the check: per genotype, kan and 4CP growth should be affine in + each other. Genotypes that break it are the first evidence against the + design hypothesis. + - A growth-only configuration forces `activity: fixed`, + `theta_rescale: passthrough`, `condition_growth: linear`, and refuses + `hill_mut`/thermo theta components and partition-function congression + rules (C1). + - Outputs labelled as X; downstream restrictions (C8). + - `tfs-summarize-calibration` computes the X-scale truth (C8). + - Validation on one simulation: joint vs relative fit, the relative fit + compared with X-scale truth; genotypes with simulated binding reported + separately (the calibration tool already stratifies on the simulated + binding file, so this is a held-out comparison); `k_c`/`m_c` recovery + and the IPTG-independent genotypes as internal standards (C4). Then + under step 4's noise. Real data runs use step 2's smoothed total. + Implemented 2026-09-26: `theta: hill_relative` + (`generative/components/theta/hill_relative.py`; baselines `X_low` and + `X_delta` real-valued and hierarchical, priors centered on wt's 1 and + -1; wt's baselines computed from its K and n by `gauge_baselines`, with + the occupancy span floored at 1e-3). The gauge is the orchestrator + setting `theta_gauge_conc` (`--theta_gauge_conc`; default the measured + min and max, written to the config). The orchestrator refuses binding + data, learned activity, `logit` rescaling, non-linear growth, theta + noise, and non-`max` rules under `mixture` (C1; `mixture`'s `max` + branch does not clip, checked). C8: `THETA_SCALE = "X"`; + `tfs-predict-theta` writes `theta_scale = X`; epistasis tools allow only + `add` (no `in_regime`); `tfs-summarize-fit` maps theta and + `growth_k`/`growth_m` truth onto the gauge (with simulated activity); + `tfs-summarize-calibration` labels those rows `theta_regime = X`. + Prediction on a genotype subset keeps wt for the gauge. Registry-wide + batch-safety, batch-order and guide-naming tests pass for it. Not built: + the kan-vs-4CP consistency check (X is shared by construction, so a fit + cannot test it alone; needs a per-condition-family X variant or a + model-free comparison, to be decided with the real-data run). Validation + grid: `planning/studies/relative-fit/` (24 runs: noise x likelihood x + joint/relative x 3 seeds; not yet run). +6. **Population model in the fit** (N; replaces the per-tube offset). + - Population curves: `P = ln N` per replicate x condition x concentration + over `t_pre + t_sel`, starting at the replicate's split density (from + the presplit OD600 and the dilution, with calibration and pipetting + error; one value per replicate), with a shape shared across replicates + and per-replicate deviations (form from 0c, D9). Global sites, a few + per curve. + - OD600 likelihood: every reading of every bioreplicate, through the + calibration (global, D10), with reading noise plus tube growth noise + `(sigma_tube * t)^2` (C3); readings below the threshold are censored, + readings above the calibrated range are refused or flagged. + - Optional monoculture growth observer, for bench experiment 1 of + `planning/absolute-abundance-measurements.md`: a measured growth rate + of a named genotype in a named condition and concentration observes + the clean-class rate `k_c + dk_g + m_c X_g(c)`. Unlike `base_growth` + (`k_ref + dk_geno` in one reference condition), it pins `k_c`/`m_c` + where OD600 is blind. + - The `lncfu` likelihood observes frequencies against the model: + `ln freq_{g,s} ~ ln n_{g,s} - P_s + u_s`, counting variance only. The + per-row `ln_cfu` stays an output, now built from `P`. + - `sample_offset` becomes the composition offset `u_s`: per tube, constant + prior (0b), no time scaling (D2). Exposed in `tfs-configure-model`. + - Validation: the calibration grid with step 4's noise on, against the + step 2 interim total: theta coverage, `k_c`/`m_c` recovery (C4), in the + joint and the relative fit. +7. [x] **Count likelihood** (N). (Validated 2026-09-27. The fourth run, + 2026-09-28, with `ln_cfu0: hierarchical`: theta 95% coverage 0.83 / 0.84 + against 0.72 / 0.83 for `lncfu`, RMSE 0.035 / 0.072 against 0.100 / + 0.109. `counts` with `sample_offset: level` is the + `tfs-configure-model` default since 2620a5b3, and the real-data fits use + it.) + `growth_likelihood: counts` (C2): growth and presplit observers with + `reads_{g,s} ~ NegBin(depth_s * exp(ln n_{g,s} - P_s + u_s), dispersion)`, + `P` from step 6 (until then, the supplied `sample_ln_cfu`, D14), `u` + from step 6's `sample_offset`, and a learned dispersion with both a term + proportional to the mean and a quadratic term, + `var = mu (1 + phi) + mu^2 / r`, so the refined data only set priors + (C12; one per experiment; no + per-batch term, D8), optional index hopping (`E[reads] = N (p_g + h q_g)`), + and, for future screens, a spike-in counting standard (a non-growing + pseudo-genotype of known abundance per tube, which fixes `P_s`). + No pseudocount in the fit. This supersedes censoring (D1). Validation: the + calibration grid against step 6's `lncfu` model, including the + true-lambda-0 arm where the mixture currently finds lambda ~0.12, a + leading suspect for which is the floor. Low-count genotypes are the + stratum to watch. + Implemented 2026-09-26 (with supplied totals, D14): + `generative/observe/growth_counts.py` (`reads ~ NegBin(mu, mu (1 + phi) + + mu^2 inv_r)`, `log mu = ln reads + ln_cfu_pred - ln total`), a + float32-accurate log-pmf (Loader's algorithm; numpyro's negative + binomial lost several nats per observation at 1e4-1e6 reads), + `sample_offset: level` (one constant-prior offset per tube, D2), and + `sample_reads` from `counts_to_lncfu`; the orchestrator derives tube + reads from `adjusted_counts / frequency` for files processed before, so + the new processing batch needs no rerun. Refuses `growth_noise` other + than `zero`. Presplit stays `ln_cfu` (its data have no counts yet); + index hopping is not modeled yet. Validation grid: + `planning/studies/count-likelihood/` (24 runs). First run (2026-09-27): + 11 of 12 `counts` runs failed, on two bugs now fixed (the pre-fit's + Hessian was taken at `exp` of every positive site's MAP, overflowing on + `phi`; the concentration underflowed for near-extinct genotypes, giving + NaN gradients). Rerun needed, `lncfu` arms included (the Hessian bug + also loosened their `k_scale`). The one finished `counts` mixture run + found lambda 0.027 at a true 0 (the `lncfu` mixture found ~0.12). + Second run (2026-09-27, 23 of 24 finished): counts cuts theta RMSE by a + third to two thirds against `lncfu`, at coverage close to `lncfu`/`zero` + (all arms still under-cover, 0.66-0.81 at 95%); lambda at a true 0 is + 0.02-0.06 (not ~0.12), with intervals that exclude 0. Two failures + outside the likelihood: hill_mut's horseshoe local scales widen for + low-read doubles until a draw overflows (NaN), and a premature step-size + cut during a noisy descent (one wrong fit). Both fixed (overflow-safe + horseshoe helpers; pooled check of stalled windows before a cut, which + would have kept 20 of 23 v2 runs from cutting mid-descent). A third run + is needed. Third run (2026-09-27, all 24 fine): counts cuts theta + RMSE 35-60% and dk_geno RMSE 40-85% against `lncfu`, at about the same + coverage as `lncfu`/`zero` (0.79 against 0.82 realistic; 0.65 against + 0.76 poisson), under-covering most for mixed-purity and near-saturated + genotypes; lambda at a true 0 is 0.02-0.06. The remaining + overconfidence is the fit's, not the likelihood's. Details: + `planning/studies/count-likelihood/README.md`. + - [ ] **7b. Growth regime: test, choose, purge** (N). The + `growth_transition` components (`instant`, `memory`, `baranyi`, + `baranyi_k`, `baranyi_tau`, `two_pop`) were unconstrained by the data + under the old noise model (user, 2026-09-26). Re-test them once steps + 6-7 are in: on simulations with known transitions (does the fit recover + a lag it was given, and stay at `instant` when there is none?), then on + real data. Keep `instant` (the null) and at most one physically + motivated model, most likely `two_pop` (cells switch from pre-selection + to selection growth at a first-order rate `k_trans`, so each genotype's + trajectory is a two-exponential, the per-genotype counterpart of D9's + population form); fix its fallback for `g_pre - g_sel - k_trans <= 0` + if it is kept. **Purge the rest** from the fit + (`generative/components/growth_transition/`, `_baranyi.py`), the + simulator (`simulate/growth/transition_linkage.py`, + `TRANSITION_REGISTRY`), examples, grid `auto` enumeration, tests, docs + and CLAUDE.md (D12, D13). +8. **The growth-binding map** (G's stage 2; study first). + On real data, relative-X fits (step 5) with the step 7 likelihood; errors in + both variables, genotype as the unit; the nested family identity -> affine + f -> monotone f, with a global concentration scale s; spiked and bulk + separately (bulk on a compressed spiked curve = congression). A study in + `planning/studies/`; a CLI only if it earns one. On simulation first with a + known nonlinear f and scale s, to check recovery. +9. **Joint fit with the learned map** (G; scope set by step 8). + A learnable `theta_rescale` (f) and concentration scale s; binding back at + weight 1 (C7); absolute theta and the partition-function congression rules + return through `f^-1` (C1). If step 8 finds no map, this step becomes a + report of the disagreement, not a model. + +After step 9: revisit `planning/empirical-mixture-refit.md` (Stage 1 could +then use the count likelihood and needs no binding-based prefit). + +## Decisions + +User, 2026-09-26. + +- **D1. No censoring.** Step 7 replaces it (and the masked 3.5 grid showed + the floor does not drive the mixture's behavior). If main needs a floor fix + before step 7 lands, dropping rows below a read threshold is available now. +- **D2. `sample_offset` becomes the composition offset `u_s`.** Revised + 2026-09-26 after the OD discussion: tube growth noise moves to the OD600 + likelihood (C3), so the per-tube latent on genotypes keeps only the + time-independent composition part (PCR, calling). No new component. +- **D3. One `u_s` per tube, independent across timepoints.** No random walk. +- **D4. Gauge: pin wt at two reference concentrations**, `X_wt = 1` at 0 and + `X_wt = 0` at 1 mM IPTG, whether or not those are asymptotes. It uses the + full accessible dynamic range of growth. +- **D5. X is shared between kan and 4CP** (the core design hypothesis), with + a consistency check (step 5). +- **D6. One long-lived branch.** +- **D7. Tube growth noise scales with `t_pre + t_sel`.** Protocol: transform, + grow to a set OD, take `presplit`, split into one tube per condition x + concentration x timepoint (a grid: rows timepoints, columns IPTG), pull a + whole row per timepoint. Pre-growth and selection both happen in the + tube. Everything before the split is shared by a replicate's tubes and + lands in `ln_cfu0`, which `presplit` measures. +- **D8. No per-row term.** A row's tubes are read, spun down and frozen + immediately (stop-time spread: seconds); all samples are then thawed and + extracted in batches of about 20 (three for 60 samples) and amplified in + one PCR block. Extraction batch is the only shared step; yield does not + change genotype frequencies, but it could change PCR template input and so + count dispersion. The batch assignment was not recorded (user, + 2026-09-26): batches of 20 generally followed processing order, but not + reliably. So the model has **no batch term**: an assignment that is only + probably right would put guesses into the likelihood. If 0b sees block + structure along the likely order, it is reported as unexplained + dispersion and absorbed by the shared dispersion, not modeled. Future + screens should record each sample's extraction batch (a `sample_df` + column), so the question can be answered then. + +- **D10. Calibration parameters are latent** (user, 2026-09-26; may be + revised): the calibration file's mean and covariance are a + multivariate-normal prior on the polynomial coefficients (three for a + quadratic), so their shared uncertainty is carried once (C11). Fixed at + the best fit stays available as a check. +- **D12. Breaking changes are fine** (user, 2026-09-26). No compatibility + with old files, configs or checkpoints is kept for its own sake. A + switch or an old path survives only for a scientific reason: to measure + whether something matters (the `lncfu` vs `counts` likelihood, the noise + sources in the simulator, `instant` as the null growth transition). +- **D13. One framing for growth regimes** (user, 2026-09-26). Once a + physically motivated model is chosen (7b; D9 for the population curve), + models outside that framing are removed, not kept as options. +- **D14. Build order while data are gathered** (user, 2026-09-26): steps + 1, 4, 7 (supplied totals), 5, 3; see "Steps". + +## Open + +- **D9. Population curve form.** Per replicate x condition x concentration, + a known start (presplit), a shared shape plus per-replicate deviations + (the user's observation: slopes agree across bioreplicates, levels do + not). Under selection the total can fall before it rises (susceptible + cells stall or die while resistant genotypes take over), so a monotone + form is wrong there. Candidates: a sum of two exponentials, + `log(a e^{r1 t} + b e^{r2 t})` (physically motivated: two + subpopulations); piecewise linear over pre-growth and selection; a + low-order polynomial or spline. 0c decides. Deriving the curve from the + genotype model (`logsumexp_g ln n_g`) is exact in principle but not + closable: `__unknown__` reads (about two thirds of kan reads) are cells + of unmodeled genotypes that still count toward OD600. + Decide together with 7b, but keep the two questions apart: the + population curve can bend with no per-genotype lag at all, because + selection changes the library's composition (fast growers take over the + total). Curvature in the totals is not evidence for `two_pop` or a + Baranyi lag; only the per-genotype trajectories in the reads are. If + genotypes do follow `two_pop`, the population is a sum of their + two-exponentials, and a two-exponential population curve is a + reduction of that, not the same model. +- **D11. Does the calibration transfer to selected cells?** It was measured + on unstressed Keio cells in LB. Kanamycin- or 4CP-stressed cells can change + size and shape, and dead cells still scatter light (and still carry + plasmid DNA into the reads). 0c looks for signs of it; a bench check + (OD vs plate counts under selection) would settle it. +- **Where the extra count noise comes from** (C12). A technical replicate + (re-amplify and re-sequence the same extracted DNA for a few tubes) + separates the template/PCR part from founder sampling and biology. Cheap + if DNA remains. + +Answered (user, 2026-09-26): OD600 is read with the calibration's setup +(same reader, plate, 200 uL); the presplit is read and then diluted +identically into every tube (above); `ln_cfu` is `ln(cfu/mL * 5 mL)`; each +row of a bioreplicate is read as a block within 5 minutes of its freeze. diff --git a/planning/combination-specific-expression-effects.md b/planning/combination-specific-expression-effects.md new file mode 100644 index 00000000..48f93cd6 --- /dev/null +++ b/planning/combination-specific-expression-effects.md @@ -0,0 +1,29 @@ +--- +title: Combination-specific effects of co-expressed variants on protein level +status: idea +filed: 2026-09-13 +area: tfmodel +revisit_when: >- + If double-transformation or native mass spec experiments show effects that + depend on the specific pair of variants rather than on each variant alone. +related: + - planning/congression-physics-plan.md +--- + +**Context:** The congression plan assumes total repressor expression per cell +does not depend on how many plasmids a cell carries, and that each variant +contributes independently (a share-weighted theta and a soft-min dk). + +**Idea:** Specific combinations could break this. For example, a very +unstable heterodimer might trigger a strong unfolded-protein response that +lowers the functional protein concentration or adds a growth cost beyond +either variant alone. That is a genotype-by-genotype interaction that feeds +back onto protein concentration, and so onto both theta and dk. + +**Why not now:** Not resolvable with a single-lambda screen, where any given +pair of genotypes almost never shares a cell, and there is no evidence yet +that it matters. + +**What it would take:** Bench data first (pairs co-expressed, protein levels +and growth measured). A model would need pair-specific terms, which is only +plausible for a small, targeted set of pairs. diff --git a/planning/congression-physics-plan.md b/planning/congression-physics-plan.md new file mode 100644 index 00000000..82982911 --- /dev/null +++ b/planning/congression-physics-plan.md @@ -0,0 +1,788 @@ +--- +title: Match simulator and fit on the physics of congression +status: active +filed: 2026-09-13 +area: tfmodel +revisit_when: >- + Active now. Update the step list as steps finish. +related: + - src/tfscreen/simulate/selection_experiment.py + - src/tfscreen/tfmodel/generative/components/transformation/mixture.py + - src/tfscreen/tfmodel/generative/model.py + - src/tfscreen/tfmodel/model_orchestrator.py + - src/tfscreen/tfmodel/analysis/prediction.py + - planning/empirical-mixture-refit.md + - src/tfscreen/simulate/transformation_lam_output.py + - src/tfscreen/genetics/library_design.py + - planning/estimate-dk-alpha-by-varying-lambda.md + - planning/plasmid-segregation-low-copy.md + - planning/combination-specific-expression-effects.md + - planning/estimate-spike-fraction-from-growth-curvature.md + - planning/per-sample-level-offset.md + - planning/count-likelihood.md +--- + +## Why + +A code analysis revealed the simulator and the fit describe congression +(cells carrying more than one plasmid) differently: + +1. **The simulator combines whole growth rates.** `_sim_growth` in + `simulate/selection_experiment.py` takes the min (kanR) or max (pheS) of each + co-resident plasmid's full `k*t`, including its `dk_geno` and the condition + baseline. The min/max is meant to encode "the tightest binder sets theta," + but `dk_geno` varies more across genotypes (SD about 0.018 per minute in the + sim prior) than the whole theta signal (|m| = 0.0099 kanR, 0.0062 pheS). So + the selection is driven by `dk_geno`: in cells with a co-resident, the chosen + plasmid is the highest-theta one only about 52% of the time (chance level). + Monte Carlo over the sim prior (theta uniform as a stand-in) gives a growth + error with SD of about 0.01 per minute, 1 to 1.5 theta units. The fit + corrects theta only, so the error lands on lambda, `m`, the growth + baselines and theta. +2. **"Spiked" genotypes are not congression-free.** Their codons also occur in + the bulk sub-libraries. In the old sim config only 0.3% of wt cells and 14 to + 20% of M42I/H74A/K84L cells came from the monoclonal spike, yet + `--spiked` masks all of them as clean. This is also true of the real data, + because spiked and library copies can't be told apart by sequence. + +Checked and ruled out: the definition of lambda (sim cell sizes are +zero-truncated Poisson(lambda), so a focal plasmid's co-residents are exactly +Poisson(lambda), as the fit assumes; true only while every plasmid got its +whole cell's abundance. Since step 1 a cell's abundance is shared among its +plasmids, and the co-resident weights are zero-truncated; corrected in 3.4), +and the fit's unweighted background +distribution (the lambda that reproduces the sim's true theta inflation is 0.30 +to 0.35 against a true 0.357). + +## Where we start (code on `main`, 2026-09-23) + +**Fit.** Congression is a correction to theta alone, applied in +`generative/model.py` through the `transformation` component: + +- A congressed genotype's growth theta is replaced by the expected dominant + maximum `E[max(theta_g, theta_background)]` over Poisson(lambda) + co-residents (`_congression.update_thetas`). The background distribution is + logit-normal (`logit_norm`) or the empirical theta of the whole population + (`empirical`); `single` turns the correction off. +- Which genotypes are corrected is a yes/no `congression_mask`, built in + `model_orchestrator.py` from the spiked genotypes (from the library + composition table's `in_spiked_origin`, or the legacy `spiked_genotypes` + list). `ln_cfu0_spiked_mask` is exactly `~congression_mask`, so one flag sets + both purity and the `ln_cfu0` prior class. +- dk_geno is never touched by congression. +- The corrected theta then passes through `theta_growth_noise` and + `theta_rescale` before growth is assembled. +- Under mini-batching the empirical background is computed over the full + population (`population_theta`), and prediction code that subsets genotypes + must supply `external_theta_population` (`analysis/prediction.py`). +- The same `update_thetas` operator is reused by Stage 1.5 of + `tfs-fit-genotypes` / `tfs-build-empirical` + (`tfmodel/genotype_fit/congression.py`). + +**Library composition.** `tfs-configure-model --library_config` writes +`{out_prefix}_library.csv` (`genetics.library_composition_table`), with a +per-genotype `bulk_fraction` (the `f_g` below) and `pool_fraction`. The model +reads the table but does not yet use either fraction; it only derives the +spiked set from `in_spiked_origin`. + +**Simulator.** `_sim_growth` (`simulate/selection_experiment.py`) combined each +co-resident plasmid's whole `k*t` with a per-marker-library +`multi_plasmid_combine_fcn` (`gmean`/`mean`/`min`/`max`/`sum`; default `mean`; +the example configs used `min` for kanR and `max` for pheS). Replaced in +step 2 (2026-09-23) by cell-level theta and dk rules. + +**Known simulator/design mismatch.** `_sim_sequencing` and +`_calc_genotype_cfu0` credit every plasmid in a cell with that cell's whole +abundance, so a co-transformed cell counts once per plasmid. +`expected_library_composition` assumes the opposite: a cell's plasmid copies +are shared among the variants it carries, so `bulk_fraction` does not depend +on lambda. Fixed in step 1 (2026-09-23): the simulator now splits a cell's +abundance among its plasmids. + +## Biology and assumptions + +Provenance: user, 2026-09-13, unless noted. + +- **The repressor is a dimeric lac repressor, not tetrameric.** +- **Protein is in large excess over DNA:** about 10 uM dimer against about + 10 nM operator. No depletion, so each variant's statistical weight on the + operator is linear in its share of the protein. +- **Heterodimers: unknown.** Working assumption is mostly homodimers, which is + intuition, not data. Native mass spectrometry (collaborator) should resolve + it. +- **Total repressor expression per cell does not depend on plasmid number.** + Each variant is diluted by its share. Combination-specific effects on + expression are out of scope (`planning/combination-specific-expression-effects.md`). +- **Plasmids are high copy and assumed to persist without segregating** over + the experiment (roughly 3 to 7 doublings). The next design moves to a + low-copy plasmid, where segregation matters + (`planning/plasmid-segregation-low-copy.md`). +- **The dataset has a single lambda** (0.3572) and is headed for publication. + Parameters that only a lambda series could identify are set from biology + and checked by sensitivity, not fit. +- **Spiked and library copies are indistinguishable** in this dataset. Future + runs will give spikes distinct codons. +- **dk_geno spread is tight in this library** (bench, 2026-09-23): monoculture + growth rates of eight genotypes, chosen to be biased toward expected poor + growers, were 0.024 +/- 0.001. Not guaranteed for other libraries. +- **Overnight outgrowth does not shift the clean/congressed class weights** + (user, 2026-09-23). The mixture weights `w_clean`/`w_congressed` are the + split at transformation (`f_g`, lambda), but the mixture needs the split at + `t = -t_pre`, after an overnight of non-selective growth. Congressed cells + of genotype g would be reweighted by roughly + `exp((dk_cell - dk_g) * t_overnight)`; theta plays little role without + selection. Presplit sees only the total, so it cannot correct this. We + assume the shift is negligible and leave it unmodeled. Revisit if step 3's + calibration shows bias concentrated in bulk genotypes with large |dk_geno|. + +## Decisions + +### Combine each physical quantity at its own level + +Build the cell's theta and dk from its plasmids, then assemble growth once: + +``` +k_cell = k_condition + dk_cell + m * A * theta_cell +``` + +Never combine whole growth rates across plasmids. + +### Theta: partition function over the protein mixture + +Write `z_g = theta_g / (1 - theta_g)` (odds at single-variant concentration) +and `x_g` for variant g's share of subunits in the cell. Two limits: + +| Assembly | Cell odds | Logit form | +|---|---|---| +| Homodimers only | `sum_g x_g * z_g` | `log sum_g x_g exp(l_g)` | +| Random heterodimers, additive half-site energies | `(sum_g x_g * sqrt(z_g))^2` | `2 log sum_g x_g exp(l_g / 2)` | + +Both are soft maxima between the share-weighted mean logit and the max; the +heterodimer form sits closer to the mean. A strong binder in a 1:1 cell loses +log 2 (homodimer) or log 4 (heterodimer) logit units compared with a max rule. +The heterodimer rule is exact per titrant concentration only if subunits bind +ligand independently; allosteric coupling between subunits makes it +approximate. Default: homodimers, until native mass spec says otherwise. +The choice is made in step 4 from the bench experiments below. + +### dk_geno: soft-min family + +``` +dk_cell = -(1/alpha) * log sum_g x_g exp(-alpha * dk_g) +``` + +`alpha -> 0` is dilution (share-weighted mean burden); `alpha -> inf` is +dominance (the worst variant sets the cost). Alpha is set from biology with a +sensitivity check, not fit (`planning/estimate-dk-alpha-by-varying-lambda.md`). +Under dilution the average effect is a uniform shrinkage of each genotype's +dk toward the population mean, by `E[1/M]` over the abundance-weighted cell +size (`M` zero-truncated Poisson, see 3.4) = 0.913 at lambda = 0.357 (0.841 +under the old size-biased weighting), which the hierarchical dk_geno prior +mostly absorbs. + +The fit's current behavior (each plasmid keeps its own dk) has no cell-level +counterpart: a cell carrying two plasmids has one growth rate. It is wrong, +not just approximate, so we accept a breaking change here (user, 2026-09-23). +Both simulator and fit start from **dilution** (alpha -> 0); step 5 adds the +rest of the family and the sensitivity check. + +Like theta's background, dk's co-resident background must come from the full +genotype population, not the current mini-batch, and prediction paths that +subset genotypes need an external dk population reference. + +### Mix at the observable level + +A genotype's cells are a mixture of classes growing at different rates, so +mix `exp(ln_cfu)`, not rates or theta. With `f_g` the fraction of genotype g's +cells from the bulk library (computed from the library design), a focal plasmid +has no co-residents with probability `exp(-lambda)` (superseded in 3.4: with +a cell's abundance shared among its plasmids, the clean fraction of a bulk +genotype's abundance is the zero-truncated `P(M = 1) = lambda exp(-lambda) / +(1 - exp(-lambda))`, so `w_congressed = f_g (1 - P(M = 1))`): + +``` +w_congressed = f_g * (1 - exp(-lambda)) +w_clean = 1 - w_congressed +ln_cfu_g(t) = ln_cfu0_g + log[ w_clean * exp(k_clean t) + + w_congressed * E_S exp(k_{g,S} t) ] +``` + +`S` is the random set of co-resident genotypes. This form covers every +genotype: a pure spike has `f = 0`, a pure bulk genotype has `f = 1` (and is +still 70% clean cells). Whether `E_S exp(k t)` can be approximated by +`exp(E_S[k] t)` must be checked numerically on the real design and timepoints. + +`k t` above is shorthand. The model's growth is pre-growth plus selection, +routed through the `growth_transition` component, and both `t_pre` and `t_sel` +growth can carry congression artifacts (user, 2026-09-23). So the mixture is +over each class's whole `ln_cfu` trajectory: each class gets its own `k_pre` +and `k_sel`, the `growth_transition` model is applied per class, and the +classes are mixed last. + +Other observations: + +- **base_growth** measures a constant rate (`k_ref + dk_geno`) and is not + mixed (user, 2026-09-23). +- **presplit** observes `ln_cfu0` directly at `t = -t_pre`. The pool there is + a mixture of clean and congressed cells, but no growth has accrued yet in + the model's time frame, so the mixture term is `log(w_clean + w_cong) = 0` + and the presplit likelihood stays `ln_cfu0_g ~ presplit`. Presplit is a + direct estimate of `ln_cfu0_g` for every genotype. Congression still enters + it in two ways: + - *Counting.* For presplit and growth to estimate the same `ln_cfu0_g`, + both must count a co-transformed cell with its abundance split among its + variants (simulator: step 1; fit: same definition of `ln_cfu0`). + - *Overnight growth.* Differential growth of clean and congressed cells + during the overnight outgrowth before `t = -t_pre` is absorbed into the + free per-genotype `ln_cfu0_g`. It also shifts the class weights away from + the design values; that shift is not modeled (see assumptions). +- **binding** stays clean (below). + +Open: `theta_growth_noise` and `theta_rescale` currently act on the corrected +theta. In the mixture they must act on each class's theta before growth is +assembled. Decide in step 3 whether the noise term is drawn once per genotype +and shared across classes (the natural reading) or per class. + +### Split `--spiked` into two concepts + +`--spiked` currently sets both congression purity and the high `ln_cfu0` prior +class. Keep one `ln_cfu0` latent per genotype for total abundance, with the +existing spiked/bulk prior on it; carry purity in `f_g`. Genotypes can't simply +be removed from `--spiked`, because their abundance really is far higher than +the bulk's. + +### Fit design (step 3) + +Agreed 2026-09-23 (user). Today's fit replaces a bulk genotype's theta by +`E[max]` over co-residents and grows one trajectory with the genotype's own +dk, i.e. it averages theta over the genotype's cells where the reads average +`exp(ln_cfu)`. The replacement: + +``` +ln_cfu_g(t) = ln_cfu0_g + logsumexp_c [ log w_{g,c} + G_{g,c}(t) ] + delta_sample +``` + +- **Classes c.** *Clean*: the genotype's own theta, activity and dk. + *Congressed*: the genotype plus co-residents; cell theta and activity from + the theta rule (max), dk from the dk rule (dilution), matching the + simulator (`simulate/cell_rules.py`). +- **Weights.** `w_cong = f_g * (1 - exp(-lambda))`, `w_clean = 1 - w_cong` + (zero-truncated form since 3.4: `w_cong = f_g (1 - P(M = 1))`); + time-independent. `f_g` is `bulk_fraction` from the library table. +- **G** is each class's whole trajectory through the existing pipeline + (`theta_rescale` -> `calculate_growth` -> `growth_transition`) over pre and + selection growth. Classes go on a new leading axis; the growth and + transition components are elementwise and should broadcast (verify). +- **Unchanged:** `ln_cfu0`, `delta_sample`, growth noise, presplit, + base_growth, binding. +- **`congression_mask` is split.** Purity -> `f_g`; the `ln_cfu0` prior + class -> `in_spiked_origin` (spikes really are far more abundant). Legacy + `spiked_genotypes` -> `f_g = 0` for spiked, 1 otherwise. +- **Co-resident background** needs the full-population theta (exists) and + dk (new), mini-batch safe, with external references for prediction. + Pooled vs per-origin background is decided by 3.0 (each sub-library is + transformed separately, so a single's co-residents come from the single + library and a double's from the double library). +- **`theta_growth_noise`** acts on the genotype's own theta before classes + are built; co-residents use noiseless population values. +- **lambda** stays a latent with the measured prior; it now enters the + weights as well as the co-resident draws. +- **Estimator (from 3.0; user, 2026-09-23): fixed Monte Carlo quadrature.** + Before fitting, draw once, per genotype, K co-resident sets from the pooled + bulk composition (`pool_fraction * bulk_fraction`), stratified by count: + K_1 sets with N = 1, K_2 with N = 2, K_3 with N = 3 (e.g. 12 / 3 / 1; N >= 4 + is about 0.3% of congressed cells at lambda = 0.357 and is dropped). The sets + are integer indices stored with the data and never redrawn. Each step builds + 1 + K cells per genotype from the *current* parameters (focal and + co-resident theta/activity/dk looked up by index), weights set k by + `w_cong * P(N = n_k | N >= 1; lambda) / K_{n_k}` so lambda stays + differentiable, and evaluates the likelihood once. Per-genotype sets + (rather than one shared set) keep the quadrature errors independent across + genotypes, so they do not push shared parameters (k, m, lambda) in a common + direction. Redrawing every step is rejected: the log of a fresh K-sample + mean is biased low and adds gradient noise. K is configurable (default 16). +- **Components.** Keep `single` and `empirical`; retire `logit_norm` (a + parametric theta background with no dk counterpart). + +### Binding data stays uncongressed + +`congression_mask` already acts only on the growth theta path +(`generative/model.py`), so binding measurements on monoclonal stocks stay +clean even when a genotype's growth data is mixed. Keep it that way. + +## Real library design + +Tune simulations to the design used for the dataset: + +```yaml +transform_sizes: + single-1: 100_000 + single-2: 100_000 + double-1-2: 300_000 + spiked: 1_000 + +library_mixture: + single-1: 100 + single-2: 100 + double-1-2: 1_000 + spiked: 1 +``` + +## Experiments (bench) + +Underway (user, 2026-09-23); first result 2026-09-24 (below). They decide the +theta rule in step 4. + +- **Double transformation, tight + dead binder:** relative to the tight binder + alone, a max rule predicts no shift, homodimer mixing about 2x weaker + apparent K, heterodimer mixing about 4x, dominant negative more. The sign + (shift or not) is expected to be easy to resolve; magnitude is hard. Control: + the same variant on two distinguishable plasmids should reproduce the single + transformant under every rule. +- **Native mass spectrometry** of co-expressed variants (collaborator) to + measure heterodimer formation. + +Record results here when they arrive. + +**Result, 2026-09-24 (user; student's experiment).** Growth in the pheS +selection (4CP) at three IPTG concentrations; high IPTG induces the toxin and +slows growth. Transformations at high DNA concentration of wt, D88A (super +repressor, tight binder), K84L (barely binds), and the pairs wt:D88A and +wt:K84L. At the intermediate IPTG: D88A fastest, wt intermediate, K84L no +growth. wt:D88A grew faster than wt, slightly slower than D88A; wt:K84L grew +faster than K84L, slightly slower than wt. So each pair grows between its +parents, close to the tighter binder. + +- User's reading: not consistent with weaker-binder dominance; does not + separate stronger-binder dominance (max) from a partition-function rule. + Encouraging that the sign of the correction is right. +- Limits (user): no direct lambda, so the mix of double transformants and + single transformants of each parent in the paired cultures is unknown + (high DNA concentration should give many doubles); pheS arm only. +- Caveat (Claude): single transformants of both parents are present in a + paired culture, and the faster parent's singles take over under + exponential growth. So "between the parents, close to the faster" is + partly expected under any rule, including weak-binder dominance, if singles + make up much of the culture. What separates the rules is how close to the + faster parent the pair is given the fraction of doubles. It could be + firmed up by estimating that fraction (genotyping single colonies from the + paired transformation), by the shape of the growth curve (a mixture of + subpopulations speeds up over time as the fast one takes over; a clonal + double grows at a constant rate), or by growing verified clonal doubles, + though those segregate over generations + (`planning/plasmid-segregation-low-copy.md`). +- Native MS: not available. + +## Steps + +Keep this list current. Work the steps in order and keep this conversation's +scope to them; anything else goes into its own `planning/` file or task. +Each step from 1 on ends with a simulation and a fit that use the same rules, +checked with `tfs-summarize-calibration`. + +-2. **Bring in the mini-batch scrambling fix.** + - [x] Port the MAP/autoguide genotype-aliasing fix and its tests + (`inference/batch_safety.py`, `test_batch_safety.py`) from + `guide-selection` as a separate commit, code only. Every fit used to + check the steps below depends on it. +-1. **Update this plan** with the review of 2026-09-23. + - [x] Starting-point description, simulator/design mismatch, dk and + mixture decisions, bench pointer, step list. 2026-09-23. +0. **Groundwork.** + - [x] Per-genotype `f_g` from the library definition: + `genetics.library_composition_table` → `bulk_fraction` in + `{out_prefix}_library.csv`, via `tfs-configure-model --library_config` + (commit 77befcb). + - [x] Bring in `tfs-summarize-calibration` + (`analysis/calibration_grid.py`, `analysis/scripts/summarize_calibration_cli.py`) + from `guide-selection` (commit e925b90), code only, no grids or results. + 2026-09-23. +1. **Simulator: share a cell's plasmid copies.** + - [x] `_sim_sequencing` and `_calc_genotype_cfu0` split a cell's abundance + among the variants it carries, so simulated composition matches + `bulk_fraction` and no longer depends on lambda. 2026-09-23 + (`_plasmid_shares`; a test checks simulated abundance against the + design `pool_fraction` at lambda = 0, 0.357 and 1.5). +2. **Simulator restructure.** + - [x] Build each cell's theta and dk from its plasmids with pluggable rules, + then assemble growth once + (`k_cell = k_condition + dk_cell + m * A * theta_cell`). Replaces + `multi_plasmid_combine_fcn` on whole `k*t`. 2026-09-23: + `simulate/cell_rules.py`, `selection_experiment._cell_kt`; config keys + `congression_theta_rule`, `congression_dk_rule`. + - [x] First rules: max theta (the fit's current theta rule) and dilution dk. + TF activity follows the theta rule: it comes from the plasmid that sets + theta (user, 2026-09-23). Revisit in step 4, where a partition-function + theta rule would put activity into the plasmid weights. +3. **Fit: observable-level mixture.** Design agreed 2026-09-23 (see "Fit + design (step 3)"). 3.0 runs alongside 3.1/3.2; only 3.3's estimator + waits on it. 3.1, 3.2 and 3.4 are behavior-preserving commits; 3.3 is the + breaking one. + - [x] **3.0 Numerical study** (no fitting), 2026-09-23. Ground truth from + `library_prediction` on `examples/simulate/simulate_config.yaml` (843 + genotypes, thermo theta prior, memory growth transition, 48 growth + conditions, t_sel 95 to 200 min) with the real design's + `library_mixture` and lambda = 0.357, and the step 2 cell rules. "Exact" + `E_S exp(G)`: N = 1 enumerated over the co-resident library, N >= 2 by + Monte Carlo. dk_geno deviations scaled by s (sim prior at s = 1: SD + 0.068/min, 5th to 95th percentile -0.13 to 0.014/min). Error = approx - + exact `ln_cfu`, RMS over genotype x condition cells within 7 ln units of + wt (detectable): + + | estimator | s = 0 | s = 0.015 | s = 0.03 | s = 0.1 | s = 0.3 | s = 1 | + |---|---|---|---|---|---|---| + | no congression | 0.041 | 0.053 | 0.097 | 0.287 | 0.758 | 1.507 | + | today's fit rule | 0.019 | 0.041 | 0.093 | 0.293 | 0.770 | 1.524 | + | `exp(E_S[G])` | 0.006 | 0.006 | 0.008 | 0.036 | 0.206 | 1.086 | + | `E[G] + Var[G]/2` | 0.000 | 0.000 | 0.001 | 0.025 | 0.509 | 12.6 | + | N = 1 only (exact) | 0.003 | 0.004 | 0.010 | 0.030 | 0.080 | 0.127 | + | MC, K = 4 | 0.016 | 0.016 | 0.032 | 0.072 | 0.181 | 0.379 | + | MC, K = 16 | 0.008 | 0.011 | 0.010 | 0.028 | 0.072 | 0.187 | + | MC, K = 32 | 0.005 | 0.006 | 0.010 | 0.019 | 0.064 | 0.142 | + | pooled background | 0.002 | 0.002 | 0.002 | 0.004 | 0.008 | 0.024 | + + s = 0.015 is the measured regime for the current library (dk SD about + 0.001/min; see "Biology and assumptions"). + + Findings: + - The congression effect is large once dk varies (0.29 to 1.5 ln units + RMS), and **today's fit rule is no better than no correction** at any + s > 0: averaging theta cannot represent a mixture of growth rates. + - Averaging the rate first is adequate only for a narrow dk spread + (s <= 0.1) and worsens with time; the second-order cumulant fails + (heavy dk tails). + - Monte Carlo with K fixed co-resident sets (common random numbers, + shared by all genotypes) has near-zero mean error at every s, and its + RMS falls roughly as 1/sqrt(K). This is the only estimator that holds + across the dk range. + - **Pooled background is fine** (<= 0.024): no per-origin backgrounds + needed. Caveat: in the sim prior, singles and doubles have similar + theta/dk distributions; real doubles may be more often broken. + - N >= 2 matters little but not nothing (N = 1 only: 0.13 at s = 1); + Monte Carlo covers it at no extra cost. + - **In the measured regime (s ~ 0.015) the whole effect is modest**: + 0.053 RMS, 0.105 p95 (comparable to per-point `ln_cfu` noise, but + systematic), and today's rule removes only about a quarter of it. Every + estimator except small-K MC is accurate there. We still build MC + (decision 2026-09-23): tight dk is not guaranteed for other libraries, + and K is a knob. + Study: `planning/studies/congression-estimator/`. + - [x] **3.1 Plumbing.** Done 2026-09-23: `GrowthData.bulk_fraction`, + `ModelOrchestrator._build_bulk_fraction`; `congression_mask` still + drives today's correction until 3.3. `f_g` (from `bulk_fraction`) and the `ln_cfu0` + prior class (from `in_spiked_origin`) as separate `GrowthData` fields, + built by the orchestrator from the library table or the legacy + `spiked_genotypes` list (spiked -> `f_g = 0`, others 1). No behavior + change: fits identical. + - [x] **3.2 Full-population dk_geno and activity access** (API only). + Done 2026-09-23. Activity was added to the step: the cell's activity + follows the theta rule, so co-residents' activity is needed too. Every + `dk_geno` and `activity` component takes `return_population=False`; + when True it also returns the library-ordered per-genotype values from + the same draw (`generative/components/_population.py::per_genotype`), or + `None` when the latents arrived as a batch-sized substitution. + `GrowthData.external_dk_population` / `external_activity_population` + are the fallback. Nothing requests the population yet, so fits are + identical. Building the external arrays in `analysis/prediction.py` + moves to 3.4. + Found for 3.3: `theta/categorical_geno` slices its latent to the batch + at sample time, so `model.py`'s full-population `calc_theta` (arange + `batch_idx`/`geno_theta_idx`) returns batch-ordered theta at full batch + and indexes past the batch under mini-batching. Fixed in 3.3a. + - 3.3 lands as three commits: 3.3a library-ordered population theta + (bug fix), 3.3b co-resident sets drawn into `GrowthData` (no behavior + change), 3.3c the mixture itself (breaking; includes the + `analysis/prediction.py` change, since prediction runs the model and + must supply full-library populations and the fit's co-resident sets). + - [x] **3.3a Library-ordered population theta.** Done 2026-09-23: + `categorical_geno` keeps theta library-sized and indexes through + `batch_idx[geno_theta_idx]`; `test_population_theta.py` checks every + theta component. + - [x] **3.3b Co-resident sets.** Done 2026-09-23: + `ModelOrchestrator._draw_coresident_sets` / `_coresident_pool`; + `GrowthData.coresident_idx` (num_genotype, K, N_max; -1 = empty) and + `coresident_n`; settings `congression_sets` (default [12, 3, 1]) and + `congression_seed` (default 0), recorded in the config. Pool: bulk share + (`pool_fraction * bulk_fraction`) of genotypes with growth data, + `__unknown__` excluded; legacy path uniform over non-spiked (user, + 2026-09-23). An empty pool leaves every slot -1; 3.3c must refuse a + congression model with `bulk_fraction > 0` and an empty pool. + - [x] **3.3c Mixture.** Done 2026-09-23. Code plan and decisions agreed 2026-09-23 (user): + - *Transformation interface.* `define_model`/`guide` sample lambda only + (no `anchors`). New `cell_classes(focal, population, params, data)` + returns `(theta, activity, dk, log_w)` with a leading class axis. + `NEEDS_POPULATION` replaces `NEEDS_FULL_POPULATION_THETA`. `single`: + one class, `log_w = 0`, identical to today. The mixture component is + named **`mixture`**; `transformation: empirical` is refused with a + message explaining the change, so old configs do not silently switch + physics. `logit_norm` is removed. `_congression.update_thetas` stays as + a plain function for Stage 1.5 until 3.4. + - *`model.py`.* theta -> noise on the genotype's own theta -> population + theta/dk/activity (external if supplied, else the full-population + `calc_theta` and `return_population=True`) -> gather + `coresident_idx[batch_idx]` -> classes (max theta and its activity per + titrant point; dilution dk; -1 slots masked) on a class axis ahead of + `(rep, time, cp, cs, tn, tc, batch)` -> per class `theta_rescale`, + `calculate_growth`, `growth_transition` (each called once; they + broadcast) -> `ln_cfu_pred = ln_cfu0 + logsumexp(log_w + G, 0) + + delta_sample`. `theta_growth_pred` stays the genotype's own + uncorrected theta. The guide skips class construction. + - *Weights* (superseded in 3.4 by the zero-truncated form). `w_cong = f_g (1 - exp(-lambda))`; `log w_clean = + log1p(-w_cong)`; `log w_k = log f_g + log(1 - exp(-lambda)) + log P(N + = n_k | N in strata; lambda) - log K_{n_k}` (strata renormalized over + the counts that have sets), so the lambda gradient stays finite for + `f_g = 0`. + - *Refusals.* A `mixture` model with some `bulk_fraction > 0` and an + empty co-resident pool is refused at build time. The + `categorical_geno`/`empirical` incompatibility check is deleted (fixed + in 3.3a). + - *`congression_mask` is removed* from `GrowthData`, `batch.py`, the + orchestrator and tests. + - *Prediction.* `predict()` passes full-library + `external_{theta,dk,activity}_population` (dk/activity: posterior + medians of their deterministic sites; theta: the theta component at + posterior-median parameters, evaluated on the prediction's + concentration grid) and the subset's rows of + the original `coresident_idx`. A raw MAP checkpoint (no stored + deterministic sites) with a `mixture` model raises, pointing to + `tfs-sample-posterior`. + - *Cost.* Growth tensors grow by (1 + K); default K stays 16 + (`[12, 3, 1]`). Main runs mini-batch. A memory-flat `lax.scan` over + classes would need growth_transition components split into sample and + compute; deferred unless memory bites. + - *Tests.* `single` regression; mixture vs an independent numpy + implementation; lambda -> 0 and `f_g = 0` reduce to `single`; finite + gradients; registry-wide batch shape/order checks; subset prediction + equals full prediction rows; refusals; smoke tests. + - [x] **3.4 Other consumers.** Done 2026-09-24, three commits: + - *Weights (a 3.3c fix found while checking the lambda echo).* Since + step 1 the simulator shares a cell's abundance among its plasmids, so + a genotype's abundance by co-resident count n is the zero-truncated + `P(M = n + 1)`, not Poisson(n; lambda) (which assumed each plasmid got + the whole cell). At lambda = 0.357 a bulk genotype is 83% clean, not + 70%; the old weights would have recovered lambda ~ 0.18 against a + simulated 0.357. `mixture._log_class_weights` now uses + `w_cong = f_g (1 - P(M = 1))`, `P(M = 1) = lambda e^-lambda / + (1 - e^-lambda)`, and strata by `P(M = n + 1)`; a test checks them + against simulated cell shares. Zero-truncation is physical: the zero + class (no plasmid, marker dead, ...) never grows under selection + (user, 2026-09-24). The 3.0 study used the old weights for its + "exact" reference as well as the estimators, so its comparisons + stand. + - *Lambda echo.* With the weights fixed, the fit's `lam` is the + simulator's `transformation_poisson_lambda`; docstrings and + `docs/source/model-inputs.rst` define it. + - *Stage 1.5 retired* (user, 2026-09-24): `--congression_lambda` removed + from `tfs-fit-genotypes`/`tfs-build-empirical`; `genotype_fit/ + congression.py` and `transformation/_congression.py` deleted. The + whole empirical pipeline is to be reviewed before it is used again; + the replacement is filed as `planning/empirical-mixture-refit.md`. + - *`tfs-summarize-calibration`* strata: `origin` (spiked/bulk) became + `purity` (`spike`/`mixed`/`bulk`, from the sub-libraries encoding each + genotype, i.e. `bulk_fraction` 0, between, or 1). + - [x] **3.5 Calibration.** Step 2 simulator vs the new fit, with + `tfs-summarize-calibration`; watch bulk genotypes without binding data. + Grid set up 2026-09-24 (user: cluster, both extra arms, `mixed` left + unstratified): `planning/studies/congression-calibration/`, 54 runs = + simulated lambda {0, 0.357, 1.0} x dk spread {wide, tight} x 3 seeds x + fit {single, mixture with matched lambda prior, mixture with the + measured 0.357 prior}, paired against `single`. Both sides use the max + theta rule, so this checks the machinery and also serves as the + baseline for step 4. Whether to run it before the bench result on the + dominance rule is still open (2026-09-24). + Ran 2026-09-24 (all 54 converged); not yet usable. The k/dk_geno slide + is unanchored in this pipeline (no wt pin, no `base_growth`, prefit + `k_scale` at its ceiling); congression sits below the design's noise + floor (`single`'s errors do not change with lambda); and lambda is + pulled up whatever the truth. Diagnosis: the mixture uses its + congressed classes to fit the read-count detection floor (pseudocount + 1 puts 0-read rows +1.3 ln above the truth), which real data share. + Next: `grid_masked.yaml` (rows with < 5 reads dropped) to test the + diagnosis; then a censored likelihood for floor observations (user's + preferred fix), then the full grid re-run with a `base_growth` anchor. + Details: `planning/studies/congression-calibration/README.md`. + Filed 2026-09-24 as follow-ups: `planning/per-sample-level-offset.md` + and `planning/count-likelihood.md`. Censoring waits on the masked + results (user). + Masked re-run (2026-09-24): dropping rows with < 5 reads changed almost + nothing, so the floor was not the driver. Revised diagnosis: the + default `binding_weight` (growth rows / binding rows, ~251) with + simulated binding SD 0.001 dominates the objective; the mixture's + congressed classes let bulk genotypes hand their growth away (dk + dilution) so theta, and shared hill_mut effects, can follow binding. + Next: `grid_bw1.yaml` (binding_weight 1). The weight and the real + binding SDs (SE of a fitted Hill curve) are open questions for real + data (user, 2026-09-24). + Weight 1 (2026-09-24) did not fix it either (it did improve theta + coverage). Third diagnosis: the k/dk_geno slide is held only by wt's + dk_geno pin, the Student-t growth likelihood makes abandoning wt cheap + (the mixture fit misses wt by ~13.6 ln units), and the prefit k prior + never engaged (k_scale at its 0.1 ceiling). No gradient bug: the + mixture's SVI gradient matches `single`'s. Next: `grid_anchor.yaml` + (prefit `--k_scale_ceiling 0.005`, weight 1). + Review (2026-09-25): none of the 3.5 fits converged. The stop rule is + relative to the loss at the start of SVI (~500 times the final loss). + The anchored grid's epoch checkpoints show that the mixture's k + overshoot is an early SVI excursion, still unwinding at the stop, + while `single` holds k at the truth. The simulated co-resident pool is + also 39% wt. Earlier diagnoses need re-judging on converged fits. + Proposed routes forward, not yet decided: + `planning/studies/congression-calibration-convergence/`. + Integrated 2026-09-25: the review's claims check out, and the + controls agree (under MAP the lambda-0 mixture matches `single`, k + offset +0.0003). The endpoint diagnoses in the calibration README are + superseded. Next (user): resume anchored runs 0001/0002/0005 to + convergence (`resume.srun`); fixing the SVI stop rule and warm-up goes + to its own session, since it affects every `tfs-fit-model` fit. + Resumed to convergence (2026-09-25): the lambda-0 mixture matches + `single` (k at the truth, theta RMSE 0.131 vs 0.129), confirming the + transient; converging halves theta RMSE (0.23 to 0.13) but leaves + coverage low (0.55-0.66). Still open on converged fits: lambda pulled + up (0.043 at truth 0; 1.30 at truth 1.0 under a 1.0 +/- 0.05 prior). + The library enumeration bug behind the 39% wt pool is fixed + (bff2c51); new grids simulate the corrected library. Converged at + lambda 1, the mixture beats `single` on fit (loss 5.40e4 vs 5.59e4) + and dk_geno (RMSE 0.0049 vs 0.0105) but not theta (0.173 vs 0.155), + with lambda at 1.30. The convergence overhaul (5bf5b78) is in; next: + a lambda profile and the coverage question, on new grids. + Baseline and profile grids (2026-09-25, new convergence code and + library): the stop rule never fired (13/13 hit the cap), and the + lambda-1 seed-1 simulation blew up at full step size in `single` and + the mixture alike, making the profile grid uninterpretable; both went + to the convergence work. On the stable pairs, the lambda-1 mixture + recovers lambda (0.98) and cuts dk_geno error ~3x, theta slightly + worse than `single`; at lambda 0 the mixture still finds lambda + ~0.15. Next: re-run after the convergence fixes. + Before the re-run (2026-09-26): study binding noise 0.025 (was 0.001) + and a simulated Hill-coefficient limit of 4 (new + `theta_sim_priors.max_hill_n`); the optimizer was updated. Lowering the + model's default binding weight is planned as its own step (user). + Re-run (2026-09-26): all 13 runs converged, no blow-ups. At lambda 1 + the mixture beats `single` (bulk theta RMSE 0.104 vs 0.125 and 0.074 + vs 0.084; dk_geno error ~3x lower; coverage 0.81-0.87 vs 0.58-0.77), + with the gain in bulk genotypes without binding data. Lambda is + recovered roughly (0.94, 0.80 at a true 1.0) with overconfident + intervals; at a true 0 the mixture finds ~0.12 yet still fits theta + better than `single`. The lambda profile is flat within loss noise + over 0.6-1.4, and theta error changes little across it. + Closing (user, 2026-09-26): a confirming round (`grid_measured.yaml`: + lambda 0.357 x seeds 1-3; `grid_seed3.yaml`: a third seed at lambda 0 + and 1), then step 4. Open items, recorded rather than pursued now: + - Lambda's intervals are overconfident and its profile is flat over + 0.6-1.4 in these data; this supports setting lambda from the + measurement (a tight prior or a constant) rather than learning it. + - At a true lambda 0 the mixture finds ~0.12 and still fits theta + better than `single`: its congressed classes absorb something in the + data (the read-count floor is the leading candidate; censoring is + the planned fix). + - Theta 95% coverage is 0.78-0.87 for both models. + - The model's default binding weight (growth rows / binding rows) is + to be lowered, and the binding SDs rethought (raw anisotropy points, + an assay-to-cell term); its own step. + Confirming round (2026-09-26, all 10 converged): the mixture beats + `single` in all 9 pairs (3 seeds x lambda 0, 0.357, 1) on loss, bulk + theta RMSE (-0.018, -0.034, -0.017), coverage (+0.04, +0.13, +0.18) + and dk_geno. Lambda is right on average at 0.357 (0.27, 0.44, 0.35) + but low at 1.0 (0.94, 0.80, 0.66) and high at 0 (~0.12); the + seed-to-seed spread is 30-50 times the reported posterior SD, which + strengthens the case for setting lambda from the measurement. Step 3.5 + closed. +4. **Theta rule.** Homodimer vs heterodimer soft max, chosen from the bench + results, in both simulator and fit. + Design agreed 2026-09-26 (user): + - Rules `max` (kept), `homodimer` (cell logit `log sum_g x_g exp(l_g)`) + and `heterodimer` (`2 log sum_g x_g exp(l_g / 2)`), with `l_g` = logit + theta_g per titrant concentration. The bench result rules out + weak-binder dominance but cannot separate `max` from the + partition-function rules; **default `homodimer`**, as the plan said, + until native mass spec says otherwise. + - **Shares are equal:** `x_g = 1/M` for a cell with M plasmids. Library + frequency sets which plasmids share a cell (the co-resident draw), not + their shares once together. + - **Activity is held at 1.** The partition-function rules require it, on + both sides (the simulator and the fit refuse otherwise), and the + default activity component becomes `fixed` (was `horseshoe_geno`). + Physically, activity would scale a variant's share of active repressor + (cell odds `sum_g x_g A_g z_g`), which disagrees with the clean class's + `A * theta` whenever A != 1. The first paper holds A = 1 for the lac + repressor; the real treatment waits for activators. + - Fit: `mixture.cell_classes` takes a log-sum-exp over the focal plasmid + and its valid co-resident slots in place of the argmax. The rule is an + orchestrator setting (`congression_theta_rule`, default `homodimer`) + written to the config and carried as a static `GrowthData` field; + `tfs-configure-model --congression_theta_rule`. Simulator: new rules in + `simulate/cell_rules.py`; `congression_theta_rule` defaults to + `homodimer`. + - Validation: a cross-rule grid (`max` vs `homodimer`, simulated with one + and fit with the other) at lambda 0.357 and 1. + Implemented 2026-09-26. Cross-rule grid set up: + `planning/studies/congression-calibration/grid_crossrule.yaml` + (simulate {max, homodimer} x fit {single, mixture-max, + mixture-homodimer} x lambda {0.357, 1} x seeds 1-2; 24 runs). + Cross-rule results (2026-09-26, all 24 converged): the rule is not + identifiable from screen data at this design. Homodimer vs max fits + of the same simulation differ by -0.006 to +0.005 in bulk theta RMSE + and ~0 in dk_geno, whichever rule simulated the data, and both beat + `single` in all 16 pairs. The homodimer fit has the lower loss in 7 + of 8 pairs and puts lambda 0.01-0.07 higher. `homodimer` stays the + default; the physical choice waits on native mass spec. Step 4 done. +5. **dk rule.** Soft-min family with an alpha sensitivity check, in both + simulator and fit. + Design agreed 2026-09-26 (user): + - Rules `dilution` (alpha -> 0, the share-weighted mean; **stays the + default**), `softmin` (finite alpha from `congression_dk_alpha`, in + units of 1/dk) and `min` (alpha -> inf, the worst variant sets the + cost). The limits are named rules, so neither end is approximated by + an extreme alpha; alpha is required by `softmin` and refused by the + others. Equal shares `x_g = 1/M`, as for theta. + - Scale: alpha matters once alpha times the spread of a cell's dk values + reaches ~1. For a 1:1 cell with dk 0 and -0.03: dilution -0.015, + alpha 100 -0.024, min -0.030. + - Fit: `mixture.cell_classes` takes a log-sum-exp (softmin) or a min + over the focal plasmid and its valid co-resident slots; orchestrator + settings `congression_dk_rule`/`congression_dk_alpha`, written to the + config and carried as static `GrowthData` fields; + `tfs-configure-model --congression_dk_rule/--congression_dk_alpha`. + Simulator: `simulate/cell_rules.py` (`dk_softmin`, `dk_min`), config + keys of the same names. In float32 the fit's softmin has absolute + error ~1e-7 / alpha, so tiny alpha is a job for `dilution`. + - Sensitivity check: simulate {dilution, min} x fit {single, + mixture-dilution, mixture-min}, homodimer theta rule on both sides, + x seeds 1-2, on the study's wide dk spread (a deliberate worst case, + user) at lambda 0.357 and 1, plus the realistic tight spread at 0.357 + (36 runs, `planning/studies/congression-calibration/grid_dkrule.yaml`). + No intermediate alpha: if the two ends agree, everything between does. + - Side effect to watch: under `min` a slow genotype's congressed cells + are at least as slow as it is, closing the route (an earlier, + unconverged diagnosis) by which a genotype pushes its own dk down and + lets its congressed cells carry its growth. + Results (2026-09-26, all 36 converged): at the realistic tight dk + spread the rule does not matter (theta, dk_geno and loss agree within + 0.001 RMSE and 30 loss units), and the mixture does not beat `single` + on theta there. At the wide spread the dilution fit gives the best + theta whichever rule simulated the data, while the min fit gains + nothing over `single` even when min is true; dk_geno and lambda need + the right rule (min data at lambda 1: min fit lambda 1.02/0.89 and best + dk_geno; dilution fit lambda 0.47/0.38). The dilution mixture's theta + gain does not track congression (same size on min data, at lambda 0 + and 1, absent at tight spread): it needs slow genotypes that a faster + congressed class can rescue, which points at the read-count floor + (untested). At the tight spread lambda is pulled to 0.54-0.97 at a + true 0.357. `dilution` stays the default. Step 5 done. +6. **Re-validate on the count likelihood.** Open (user, 2026-09-26): the + congression arc (steps 1-5) is closed; the read-count floor is to be + handled by fitting read counts directly (`planning/count-likelihood.md`, + a separate piece of work), not by the masked test proposed after step 5. + Once that model exists, re-run `grid_baseline.yaml` (lambda 0 and 1) and + `grid_dkrule.yaml` on it (`planning/studies/congression-calibration/`): + - The theta gain of the mixture over `single` that remains is what + congression costs. Today's gain (about -0.018 bulk theta RMSE) is the + same at lambda 0 and 1 and on min-simulated data, so it is probably + mostly the floor. + - The dk_geno gain should persist: it grows with lambda and follows the + true dk rule (about 3x lower error at lambda 1), so it is congression. + - Check the fitted lambda at the tight dk spread, pulled to 0.54-0.97 at + a true 0.357 against a 0.357 +/- 0.05 prior, before trusting a fitted + lambda on real data (whose dk spread is tight). + The other open items under 3.5 (lambda's overconfident intervals, + coverage below nominal, the binding weight and SDs) stand. diff --git a/planning/count-likelihood.md b/planning/count-likelihood.md new file mode 100644 index 00000000..740484a8 --- /dev/null +++ b/planning/count-likelihood.md @@ -0,0 +1,80 @@ +--- +title: Fit read counts directly instead of ln_cfu +status: promoted +filed: 2026-09-24 +area: tfmodel +revisit_when: >- + After censoring and the per-sample offset are in and the congression + calibration grid has been re-run. The user expects to get here (2026-09-24): + the data side is mostly propagating information we drop today. +related: + - planning/analysis-roadmap.md + - planning/congression-physics-plan.md + - planning/per-sample-level-offset.md + - planning/studies/congression-calibration/README.md + - src/tfscreen/process_raw/counts_to_lncfu.py + - src/tfscreen/tfmodel/generative/observe/growth.py + - src/tfscreen/tfmodel/generative/observe/presplit.py +--- + +**Promoted 2026-09-26** into `planning/analysis-roadmap.md` (Track N, step 7, +implemented with supplied tube totals; validation grid in +`planning/studies/count-likelihood/`). + +**Context:** The congression calibration grid (2026-09-24) found that the +mixture used its congressed classes to fit the read-count detection floor. +`ln_cfu` is computed from counts with a pseudocount of 1, so rows with 0 +reads sit +1.26 ln units above the truth and 1-2 reads +0.62. Censoring those +rows is the next step. It keeps "below the floor" but loses the difference +between 0 and 4 reads, and it leaves the rest of the approximation in place. + +**Idea:** Make read counts the observation. Per sample s and genotype g: + +``` +reads[g, s] ~ NegBin(mean = exp(ln_cfu_pred[g, s] + o_s), dispersion) +``` + +with `o_s` a per-sample intercept whose prior is centered on plating, +`o_s ~ ln N_s - sample_ln_cfu_s` with the plating uncertainty as its width. +With `o_s` free this matches a multinomial over the tube (the Poisson trick), +but each genotype's term stands alone, so genotype mini-batching still works. +The reads carry how genotypes compare within a tube; plating carries the +absolute scale. (Revised 2026-09-26: the total comes from OD600, not +plating, and `o_s` is built from a smooth population curve fitted to every +bioreplicate's OD600; see `planning/analysis-roadmap.md`, "Reads and +totals".) Predictions and all downstream outputs stay in `ln_cfu`. + +What it gives that `ln_cfu` with censoring cannot: + +- **Exact low-count information:** 0 vs 4 reads pins down when a genotype + crashed, which carries theta information at strong selection. +- **The right noise model everywhere:** no first-order binomial approximation + (poor below ~20 reads), no pseudocount, and a learned overdispersion for + PCR jackpotting, which is probably why high-count rows look overconfident. +- **Misassignment as additive physics:** index hopping, cross-contamination + and barcode-calling efficiency add reads in count space, + `E[reads] = N (p_g + h q_g + ...)`, which is what produces a floor. Index + hopping into sample s comes from genotype g in the other samples of the + lane, so it only couples a genotype to itself across samples (batch safe). + Sequencing errors that turn one genotype into another would couple + genotypes; leave them to the `__unknown__` bucket. + +**Why not now:** The user wants the masked grid results and censoring first +(2026-09-24). The core noise model needs design: today tube growth noise is +per-row variance (`sigma_k`). In a count model, the per-tube shared part goes +into `o_s` (see `planning/per-sample-level-offset.md`) and the per-row part +into the NegBin dispersion; a per-(tube, genotype) latent would be far too +many parameters. + +**What it would take:** + +- **Data:** keep raw counts, per-sample depth `N_s` (including the + `__unknown__` reads) and plating through `tfs-process-counts` and + `tfs-process-presplit`. Most of it is already in the processed CSVs. +- **Model:** new growth and presplit observers; per-sample `o_s` latents + (built by the per-sample offset idea); a dispersion model; optionally index + hopping and calling efficiency. +- **Interactions:** `o_s` vs `ln_cfu0` and the k/dk_geno slide; the congression + mixture is unchanged (it acts on `ln_cfu_pred`). +- **Checks:** the calibration grid with index hopping and PCR overdispersion + turned on in the simulator, against the censored `ln_cfu` model. diff --git a/planning/deep-coverage.md b/planning/deep-coverage.md new file mode 100644 index 00000000..f0e4b0a1 --- /dev/null +++ b/planning/deep-coverage.md @@ -0,0 +1,230 @@ +--- +title: Honest intervals for the well-measured genotypes +status: active +filed: 2026-10-09 +area: tfmodel +revisit_when: >- + Active. Goal: what this experiment's data support, without new + sequencing (user, 2026-10-09). +related: + - planning/two-stage-fit.md + - planning/studies/m-bias-factorial/README.md + - planning/studies/km-anchor/README.md + - planning/studies/full-size-sim/README.md + - planning/experiment-pipeline.md + - planning/offset-mode-growth-transition.md + - src/tfscreen/tfmodel/priors_edit.py + - src/tfscreen/tfmodel/inference/posteriors.py +--- + +## Why + +On the full-size simulation calibrated to the real screen's depth +(`planning/studies/full-size-sim/`, 2026-10-09), genotypes above 1e3 reads +rank well (X r 0.93-0.99) but their 95% intervals cover 0.56 down to 0.08 +with depth (realistic noise), and worse under Poisson counts. The error is +shared: X is compressed (|m| 30-45% too large) and shifted (k slid up +against dk_geno), and the arrowhead Laplace holds exactly those directions +at the MAP, so k and m carry no interval. About 77,000 real doubles have +more than 1e3 reads; these are the curves the manuscript can use, and +their intervals have to be honest. + +The real fit is off the same way in kanR+kan (k 0.030 and m -0.020 against +the wt monoculture's 0.015 and -0.014, as in the simulation, whose truth +is the monoculture), so the real kanR curves are likely compressed too. + +## Assumptions + +- Ranking is not the problem; the absolute X scale (k and m) is. Counts + pin only relative frequencies; the tube totals pin the scale weakly + through free per-tube offsets. +- The wt monoculture rates are the experiment's independent measurement of + that scale, with their own error (SE of the replicate mean, at least + 0.002 per minute day to day). +- No new sequencing. New fits, and new simulations only to test. + +## Steps + +1. [x] **Anchor k and m with the monoculture rates** + (`planning/studies/km-anchor/`). Three fit-only arms: the realistic + simulation with the monoculture priors, the same with the rates + perturbed by their own error, and the real data with the monoculture + priors. Read: deep X coverage and slope against the baseline, k/m + against truth, held directions, and on the real data whether k and m + move to the monoculture with the offsets staying clean (or whether the + data fight the anchor, which would point at a real in-pool difference + such as the selection-onset transient). + Done 2026-10-09: the data reject the anchor in the simulation and the + real data alike. k moves only along the k/dk_geno slide; m stays about + 40% too large in kanR (sim truth -0.0138, fit -0.018 to -0.020 even with + the truth as a 0.002-SD prior; real fit -0.020 against the monoculture's + -0.014). The bias is in the fitted likelihood, a misspecification + (`planning/studies/km-anchor/`). +1b. [x] **Find the misspecification behind |m|.** A reduced-library + factorial on the simulation, local and fast: the same design and + conditions, switching off one of congression, per-tube rate noise and + realistic count noise at a time (and the dk_geno prior's shape), and + measuring the m bias. Whatever removes it is what the fit must model or + correct before step 2 is worth doing. Set up 2026-10-09 as + `planning/studies/m-bias-factorial/` (5 arms x 2 seeds, 5,256 doubles, + cluster: the staged MAP was too slow on a laptop). + Round 1 (2026-10-09) was inconclusive. The reduced `base` arm gave an m + ratio of 1.03-1.14 against 1.47 at full size, because shrinking the + library left wt at 81% of each tube's reads (31% at full size) and + dropped 25 tubes below OD600 detection. Every arm fell within seed + noise. Round 2 refits subsets of the full-size simulation's genotypes + instead: no doubles, plus deep doubles, plus as many shallow doubles. + Round 2 (2026-10-10) found it: m is right without the doubles (kanR + 1.03, pheS 0.89), about 1.2 with 81,000 doubles of either depth, and + 1.47 with all 208,000. Each double pulls m a little, and the pulls add. + That fits the joint MAP's Neyman-Scott bias, or a per-genotype + misspecification of the doubles. Next: the same no-doubles fit on the + real data (does kanR m reach the monoculture's -0.014?), and on the + simulation the full library with k and m held at the no-doubles values + (round 3, `run_held.srun`). + Real data without doubles (2026-10-10): kanR+kan k 0.0160, m -0.0141, + on the wt monoculture (0.0150, -0.0141) with loose priors; the real + kanR curves fitted with all genotypes are compressed by about 1.45. + pheS+4CP moves away from the monoculture (m 0.0061 against 0.0144): + step 3. + Round 3 (2026-10-10), the simulation's full library with k and m held + at its no-doubles MAP: above 1e4 reads X's slope 0.95 (was 1.36), + RMSE 0.05-0.07 (0.16), 95% coverage 0.90 / 0.65 at 1e4-1e5 / >1e5 + (0.29 / 0.10); the k/dk_geno slide is gone. The remaining gap is k and + m's own error, which the held fit leaves out: step 2. + Round 4 (set up 2026-10-10): the real fit holds kanR only (tight prior + at the real no-doubles MAP; pheS loose until step 3), checked first on + the simulation, since X is shared and pheS's m stays free. + Round 4 result: holding kanR alone is enough (the free pheS m follows + to 1.04 of the truth), but the tight prior leaked about 11% of the + doubles' pull (deep coverage 0.82 / 0.42 against the pins' 0.90 / + 0.65). Real: kanR m -0.0133, pheS m 0.0064, half the monoculture's. + Round 5 repeats both arms with `linear`'s new per-condition pins. + Round 5 result (2026-10-11): the pins hold exactly and pheS's free m is + right, but pheS's free k reopens the k/dk_geno slide: deep X shifts up + by 0.04-0.09, coverage 0.67 / 0.20. Pinning every condition at the + no-doubles fit (round 3) closes it, and the real pheS no-doubles m is + what the data give with X set by kanR, so the real fit should pin all + four conditions. + Round 6 (2026-10-11), the real fit with all four pinned: every + genotype's X stretches by a uniform 1.47 against the recipe fit (r + 0.99-1.00 above 100 reads), the factor the simulation predicted; no k, + m or slide direction is held. The two-stage fit (k and m from wt, + spikes and singles; the full library at those values) is the route. + Still open: k/m uncertainty in X (step 2), a second simulation seed for + stage 1 (pheS m 0.89 of the truth on seed 1; `sim_realistic_s2` is the second calibration, seed 1), pheS (step 3), and + moving the two stages into the pipeline once step 2 settles how. +2. [ ] **Carry k and m's error into X.** The held fit (step 1b, round 3) + gives X intervals conditional on k and m. Take k and m's uncertainty + from the no-doubles fit (its Laplace), refit the full library held at a + handful of draws, and pool the draws' posteriors, as the two-stage fit + of `planning/studies/svi-overconfidence/two_stage.py` did. A soft prior + at the no-doubles SD will not do: the doubles pulled m through a 0.002 + prior (`planning/studies/km-anchor/`). Check coverage by depth against + round 3 (0.90 / 0.65 above 1e4 reads). + Set up 2026-10-11 on the seed-1 simulation (`km_draws.py`, + `run_km_draws.srun`, `run_pool.srun`; 10 draws), with a second + simulation seed run through both stages (`run_two_stage.srun`) to test + stage 1's own error, which a stage-1 Laplace may understate + (`planning/studies/m-bias-factorial/`, "Step 2", "Second seed"). + Found 2026-10-11: the stage-1 Laplace is far too narrow (m SD about + 0.6%), and stage 1's m error is a bias that repeats across seeds + (kanR 2% steep, pheS 10% shallow; 12% under Poisson counts), so the + draws were cancelled. k's error is noise several times its Laplace SD. + Congression is the pheS bias's cause: without it stage 1's pheS m is + 1.035 of the truth (0.895 with it); kanR does not move. Next: a stage + 1 of wt and the spikes (little congression, few genotypes), on both + seeds and the real data. Step 2 now needs (a) the stage-1 m bias + fixed or corrected and (b) k's uncertainty from resampling stage 1, + not its Laplace. + (a) done 2026-10-12 in the simulation. Spikes alone worked on the + simulation but not the real data (nine genotypes cannot pin X's + scale), and adding the 300 deepest singles brought the pheS bias back. + Congression is not the doubles' pull, which persists without it. A + stage 1 with the congression mixture (max rule, lambda prior 0.357 +- + 0.15) removes the pheS bias (pheS m 1.02-1.04 of the truth). The recipe + is a mixture stage 1 and a `single` stage 2: deepest-genotype coverage + 0.76 / 0.67 on seeds 1 / 2 (0.65 / 0.51 before); a mixture stage 2 + adds nothing for the deep genotypes. Left: a small steep m bias + (2-6%), k's uncertainty (b), and, for the real data, wt's purity (the + X gauge), which the mixture needs. + Real mixture stage 1 (2026-10-12): congression barely moves the real + pheS m (0.0061 to 0.0068), so the real pheS gap (about 2x the + monoculture) is mostly not congression (step 3). The data lean toward + wt mostly bulk (the real config as written: 25 nats better, fitted + lambda 0.365 on the measured 0.357). kanR within 4% of the monoculture. + The real stage 2 at that stage 1 (`held_mix_real`, 2026-10-12) + converged with no k, m or slide direction held; its X is round 6's + compressed about 6% (r >= 0.998), 1.39 times the original recipe's. + Next (decided 2026-10-12, user): (i) k and m's uncertainty from a + delete-d jackknife of the mixture stage 1 (drop 20% of tubes, + stratified by condition and IPTG; tubes cannot be duplicated, so no + classic bootstrap), checked on the simulation against stage 1's known + errors on both seeds (`run_jackknife.srun`, `jackknife_summary.py`); + (ii) then carry it into X. Refitting stage 2 at every draw costs + 2-5 GPU hours each; k and m mostly shift and stretch X, so refit + stage 2 at two or three draws, measure that map per genotype, and + apply it to all the jackknife draws. Check coverage by depth on the + simulation. + Jackknife result (seed 1, 2026-10-12): k's jackknife SE is about five + times its Laplace SD and covers k's errors (0.5-0.8 SE); m's agrees + with its Laplace and its error (6-9 SE) is the accepted bias. 94% of + k's variance is one direction, so two stage 2 refits along it (plus + and minus one SD) measure X's k sensitivity; `k_band.py` widens X's + intervals with it. Scored on seed 1 (2026-10-13): 95% coverage 0.94 at + 1e4-1e5 reads and 0.91 above 1e5 (0.90 and 0.76 without), the rest + being the accepted m bias. k's uncertainty is a shift shared by every + non-wt genotype (one SD moves X by -0.032 to -0.038, genotype-to- + genotype SD 0.005), so the pipeline should add it as one shared + offset per posterior draw, not as independent width. + Accepted as a limitation (user, 2026-10-12): stage 1's small steep m + bias (2-6% in the simulation, an X slope of 1.03-1.05). Resampling + does not cover a bias; quantify it from the simulation for the paper, + and correct it from the simulation only if the methods need it. +3. [ ] **pheS+4CP.** Accepted as a limitation for now, possibly for the + paper (user, 2026-10-12): the real pool's pheS selection is about half + the wt monoculture's (m 0.0068 against 0.0144) and congression explains + little of it. The two-stage fit takes pheS's m from the pool data with X + set through kanR, so the gap need not distort X's scale. It could + distort the curves' shapes if its cause depends on genotype (a + selection-onset transient, `planning/offset-mode-growth-transition.md`). + Check: stage 2 on the kanR data alone, against the both-marker fit, on + the real data and on the seed-1 simulation (which shows what agreement + looks like when pheS is fine). If they agree, the gap is an observation + about the pool, not a problem with the curves. + Result (2026-10-12): the median real genotype agrees across markers as + in the simulation, but 8% of the deep doubles disagree beyond 2 SD + (2.5% in the simulation). Keep the gap as a limitation and flag those + genotypes (marker-consistency z between the kanR-only and both-marker + fits). Candidate cause: dk_geno shared across markers while real + fitness effects may differ in kan and 4CP; a marker-specific dk_geno + would test it. Tested 2026-10-12 without a refit (`control_dk_check.py`): + the tail's X offsets track the difference in control-condition growth + between libraries no more than on the simulation, where dk is shared + (r 0.40 in both; slopes about 6, against about 34 for a real effect), + and the real controls barely agree on any genotype's dk (r 0.04). Not + supported; the tail stays unexplained and flagged. +4. [ ] **Then the shallow genotypes' points.** Hold the X population SDs + (`theta_{X_low,X_delta,log_hill_K}_hyper_scale_fixed`) so genotypes + below 1e3 reads shrink toward the population instead of scattering; + their intervals already cover. +5. [ ] **The pipeline.** Make the two-stage fit one documented recipe (or + one command): the stage-1 subset (wt, spikes, singles) fitted with the + congression mixture, stage 2 on the full library with every + condition's k and m pinned at it, and the resampled intervals of step + 2. The students' design simulations and the NSF preliminary data use + it. Step 2's method is settled (2026-10-13): a delete-d jackknife of + stage 1 (20% of tubes, about 30 refits of about 950 genotypes), two + stage 2 refits at plus and minus one SD along k's main direction + (started from the stage 2 MAP), and the X shift between them added to + the posterior draws as one shared offset per draw. Pieces to build: + the stage-1 subset and its mixture fit, the jackknife, the two + refits, and the shared-offset draws in `tfs-sample-posterior` or a + post-processing step; plus the marker-consistency flag (step 3). + Planned in `planning/two-stage-fit.md` (active, 2026-10-13). + +## Out of scope + +New sequencing or a new screen design (`planning/design-noise-sweep.md`), +and the simulator's remaining depth mismatches (dropout tail, wt's share of +reads), which do not touch the deep genotypes. diff --git a/planning/design-noise-sweep.md b/planning/design-noise-sweep.md new file mode 100644 index 00000000..d7650b45 --- /dev/null +++ b/planning/design-noise-sweep.md @@ -0,0 +1,47 @@ +--- +title: Which noise source to buy down in the next screen (PCR templates, cells per tube) +status: idea +filed: 2026-10-09 +area: experiment +revisit_when: >- + When the next experiment is being designed (students' simulations for the + NSF preliminary data, due 2026-11-20), and after the deep genotypes' + coverage is fixed on this experiment's data, so the sweep's coverage + numbers mean something. +related: + - planning/studies/full-size-sim/README.md + - planning/studies/full-size-sim/make_sim_config.py + - planning/studies/noise-anatomy/ +--- + +**Context.** The calibrated full-size simulation of the dev-data screen +(2026-10-09, `planning/studies/full-size-sim/`, arms `sim_realistic_s2` and +`sim_poisson_s2`) matches the real screen's read depth. The two arms differ +only in count noise, and the difference is large where most real doubles +sit: at 100-1,000 total reads (about 90,000 doubles) X correlates with the +truth at 0.96 under Poisson counts and 0.62 under the realistic noise; at +100 reads or fewer, 0.72 against 0.26. The realistic noise is founder +sampling plus a PCR template bottleneck, calibrated to the real counts' +5-18x overdispersion (`planning/studies/noise-anatomy/`): templates at a +tenth of the reads, which by itself multiplies count variance about +13-fold. + +**Idea.** Sweep the noise sources on this simulation to say what each buys +before the next screen: `pcr_template_molecules` (more template DNA into +PCR), `cfu0` (more cells per tube, so less founder noise), and +`total_num_reads` for comparison. Score X recovery and coverage by depth +bin, especially at 100-1,000 reads, against the cost of each change at the +bench. + +**Why not now.** The immediate goal is what this experiment's data support +without new sequencing: first the deep genotypes' coverage, which is lost +to shared k/m errors with no interval. The sweep's coverage numbers inherit +that problem until it is fixed. + +**What it would take.** `make_sim_config.py` already writes the realistic +config; add arms with templates at 1x, 3x and 10x the current value, `cfu0` +at 3x and 10x, and reads at 2x. Each arm is one simulation (about 15 +minutes locally) plus the cluster fit. Students can run the grid once the +fit's k/m handling is settled. Open: whether the realistic noise model's +split between founder and PCR noise matches the real one (the noise-anatomy +study fit their sum to the counts, not each part). diff --git a/planning/empirical-mixture-refit.md b/planning/empirical-mixture-refit.md new file mode 100644 index 00000000..ccf420d0 --- /dev/null +++ b/planning/empirical-mixture-refit.md @@ -0,0 +1,63 @@ +--- +title: Correct the empirical pipeline's per-genotype fits for congression +status: idea +filed: 2026-09-24 +area: simulate +revisit_when: >- + When the empirical-phenotype pipeline (tfs-build-empirical, + tfs-fit-genotypes) is reviewed as a whole, and after steps 4-5 of the + congression plan settle the theta and dk cell rules. +related: + - planning/congression-physics-plan.md + - src/tfscreen/tfmodel/genotype_fit/fit.py + - src/tfscreen/simulate/scripts/build_empirical_cli.py + - src/tfscreen/tfmodel/scripts/fit_genotypes_cli.py + - src/tfscreen/simulate/cell_rules.py + - src/tfscreen/tfmodel/generative/components/transformation/mixture.py +--- + +**Context:** Step 3.4 of the congression plan. `tfs-build-empirical` builds +simulation ground truth from real data: per-genotype maximum-likelihood fits +(Stage 1), a population distribution (Stage 2), resampling (Stage 3). Real +bulk genotypes are measured with congression built in, so resampling their +fits and simulating congression again counts it twice. The optional Stage 1.5 +(`--congression_lambda`) removed that bias from the theta curves with the +theta-only `E[max(theta_g, co-residents)]` operator. Step 3.3c dropped that +operator from the main fit (the 3.0 study showed averaging theta cannot +represent a mixture of growth rates), and Stage 1.5 also used Poisson rather +than zero-truncated co-resident weights, treated purity as spiked/bulk, and +ignored dk dilution. It was retired on 2026-09-24 (user); the pipeline now +builds its distribution from uncorrected fits. + +**Idea:** Replace Stage 1.5 with an iterated Stage 1 refit under the same +observable-level mixture as `tfs-fit-model`: + +- Pass 1 is plain Stage 1. +- In each later pass, refit every genotype with + `ln_cfu = ln_cfu0 + logsumexp_c(log w_c + G_c)`. The genotype's own + parameters are free; its congressed classes combine them with fixed + co-resident sets whose theta, activity and dk come from the previous pass, + through the simulator's cell rules (`simulate/cell_rules.py`) and the + zero-truncated weights with `bulk_fraction`. +- Iterate until the fits stop moving (expect 2-3 passes). + +This corrects dk as well as theta, follows any change to the cell rules, and +gives each genotype a covariance under the corrected model. + +**Why not now:** The user wants the whole empirical pipeline reviewed before +it is used again, and the theta and dk rules are about to change (steps 4-5). +In the measured regime the bias is modest (about 0.05 ln units RMS; see +`planning/studies/congression-estimator/`). + +**What it would take:** + +- A mixture likelihood in `genotype_fit/fit.py`, sharing the class + construction with `transformation/mixture.py` or `cell_rules.py` rather + than a third copy. +- Co-resident sets as in `ModelOrchestrator._draw_coresident_sets`. +- Cost: about (1 + K) times the work per fit, times the number of passes. The + fits within a pass stay independent and parallel. +- Tests: reduction to plain Stage 1 as lambda -> 0; recovery of simulated + phenotypes from a congressed simulation. +- Open question: whether the refit should share lambda with, or estimate it + alongside, the main fit. diff --git a/planning/estimate-dk-alpha-by-varying-lambda.md b/planning/estimate-dk-alpha-by-varying-lambda.md new file mode 100644 index 00000000..d2634d4e --- /dev/null +++ b/planning/estimate-dk-alpha-by-varying-lambda.md @@ -0,0 +1,44 @@ +--- +title: Estimate the dk_geno combining exponent (alpha) by varying congression +status: idea +filed: 2026-09-13 +area: experiment +revisit_when: >- + After the congression physics work settles on the soft-min dk_geno rule and + the current single-lambda dataset is published. +related: + - planning/congression-physics-plan.md + - src/tfscreen/simulate/selection_experiment.py + - src/tfscreen/tfmodel/generative/components/transformation/mixture.py +--- + +**Context:** While fixing the simulator/fit mismatch in how co-resident +plasmids combine growth effects, we chose to combine each physical quantity at +its own level. For the pleiotropic growth cost, a cell carrying variants with +dimer shares `x_g` gets + +``` +dk_cell = -(1/alpha) * log( sum_g x_g * exp(-alpha * dk_g) ) +``` + +`alpha -> 0` is dilution (share-weighted mean burden); `alpha -> inf` is +dominance (the worst variant sets the cost). Burden that scales with the amount +of misfolded protein points toward small alpha; aggregation or mixed-oligomer +poisoning points toward large alpha. + +**Idea:** Estimate alpha empirically with a screen run at several deliberately +different congression levels (lambda). Alpha only acts through co-resident +cells, so a single low lambda (0.36 in our dataset, where 70% of a genotype's +cells have no co-resident) leaves it weakly identified. Varying lambda changes +the co-resident fraction and the burden mixture it sees while the per-genotype +dk_geno values stay fixed. + +**Why not now:** Our dataset has a single lambda, so alpha is set from biology +and checked by sensitivity (alpha = 0 vs. large), not fit. The experiment is a +separate project. + +**What it would take:** +- Wet lab: the same library transformed at 2 to 3 lambda values (for example + 0.1 / 0.4 / 1.0), with lambda measured independently for each. +- Model: alpha as a shared latent with lambda-indexed transformation parameters. +- Simulation first, to find the lambda spacing that identifies alpha. diff --git a/planning/estimate-spike-fraction-from-growth-curvature.md b/planning/estimate-spike-fraction-from-growth-curvature.md new file mode 100644 index 00000000..77e8b901 --- /dev/null +++ b/planning/estimate-spike-fraction-from-growth-curvature.md @@ -0,0 +1,31 @@ +--- +title: Estimate a genotype's spike/bulk fraction from ln_cfu curvature +status: idea +filed: 2026-09-13 +area: tfmodel +revisit_when: >- + If design-computed spike/bulk fractions (f_g) look wrong for real data, or + after the observable-level mixture (congression plan step 3) is in place. +related: + - planning/congression-physics-plan.md +--- + +**Context:** In the congression plan, each genotype's cells mix clean and +congressed classes, weighted by `f_g`, the fraction of its cells from the bulk +library. We compute `f_g` from the library design (`transform_sizes`, +`library_mixture`, codon degeneracy). + +**Idea:** Two subpopulations growing at different rates give a +`ln_cfu(t)` that is not linear in time. A genotype that is genuinely clean +grows along a straight line; a mixed one curves. With enough timepoints that +curvature carries information about `f_g`, and could check or refine the +design-based value for real data where transformation efficiencies are +uncertain. + +**Why not now:** `f_g` from the design is good enough to start, the curvature +signal is weak over 3 timepoints per condition, and future runs will +distinguish spikes by codon, which measures purity directly. + +**What it would take:** The observable-level mixture from the plan, then a +`f_g` prior centered on the design value (for example a Beta) instead of a +fixed value, plus a simulation to see whether it is identifiable. diff --git a/planning/experiment-pipeline.md b/planning/experiment-pipeline.md new file mode 100644 index 00000000..6fb35845 --- /dev/null +++ b/planning/experiment-pipeline.md @@ -0,0 +1,293 @@ +--- +title: One experiment file from raw data to posterior +status: active +filed: 2026-10-03 +area: tfmodel +revisit_when: >- + Agreed 2026-10-03 for the next branch, after + claude/physics-improvements-roadmap-03a273 merges. Keep the step list + current while it is worked. Re-ordered 2026-10-05: steps 6 and 7 next, + step 5 (the orchestrator) deferred, then a science prioritization. +related: + - planning/analysis-roadmap.md + - planning/analysis-roadmap-summary.md + - planning/studies/real-data-fit/README.md + - src/tfscreen/process_raw/scripts/process_counts_cli.py + - src/tfscreen/tfmodel/scripts/configure_model_cli.py + - src/tfscreen/tfmodel/scripts/prefit_calibration_cli.py + - src/tfscreen/tfmodel/scripts/fit_model_cli.py + - src/tfscreen/simulate/empirical/ +--- + +## Why + +The CLI chain was built for the old flow: `tfs-process-fastq`, then +`tfs-process-counts`, then `tfs-configure-model`, then +`tfs-prefit-calibration`, then `tfs-fit-model`. The analysis-roadmap +branch changed what goes in and how a fit has to be run. It added the +count likelihood, OD600 tube totals, the growth-only relative fit, held +population SDs, staged MAP starts and the arrowhead Laplace. Each gap +between the old chain and the new needs was patched by hand. The real fit +(`planning/studies/real-data-fit/`) took three hand steps: + +1. **Processing.** `prep_dev_data.py` turned OD600 into tube totals through + the calibration, fixed the library config and cleaned up lab problems. + `tfs-process-counts` then ran once per library, and the outputs were + joined by hand. +2. **Priors.** `set_growth_priors.py` wrote k/m priors, because + `tfs-prefit-calibration` refuses growth-only models, and `sed` set the + held population SDs and the tube-offset SD in the generated priors CSV. +3. **The staged MAP.** A no-offset MAP, then `make_init_npz.py` and + `check_warm.py`, then `tfs-fit-model --init_from`, over a chain of six + runs. + +Nobody else can run that, and it cannot be rerun on a full-size simulation +without redoing it by hand. + +## Approach + +Fix the three seams first, each as a small, tested change to an existing +CLI. Then add a thin orchestrator that only calls the CLIs in order. An +orchestrator built first would wrap the hacks and freeze them. +`tfs-build-empirical` is the precedent for a CLI that runs others. + +## Done when + +Revised 2026-10-05 (user), with step 5 deferred and the offset mode found +to be a model problem (`planning/offset-mode-growth-transition.md`): + +- The real fit reruns through the documented CLI chain (process, configure, + fit, posterior, extract, predict) with no hand-edited intermediate file + and no hand-built start. The chain's commands are the official recipe + for the real fit, replacing the `real-data-fit` study's six-run chain. +- A full-size simulation of the same design, written by `tfs-simulate` in + the raw formats (step 6), runs through the same chain with only the data + paths changed. That is the start of the reviewer simulations in + `planning/studies/real-data-fit/README.md`, "Next", and the footing for + every model study after this plan: a user can reproduce one end to end. +- Both report the tube-offset diagnostic (step 7), so a fit in the offset + mode is flagged, not accepted silently. + +The original criterion, every spike's K and n inside the `rel_off_n05_alap` +intervals, was dropped: run to convergence, the start behind +`rel_off_n05` moves into the offset mode, so its intervals are not a +reference. + +## Constraints + +- **P1. Lab data and calibrations stay out of the repository** (roadmap + C10). The experiment YAML names files; the repository holds formats, + methods and synthetic examples. +- **P2. Breaking changes are fine** (roadmap D12). Old processed files and + configs need not load. +- **P3. Leave room for the population model** (roadmap step 6). Processing + supplies per-tube totals from OD600 today. Step 6 will instead fit a + population curve to every OD600 reading inside the model. Keep the OD600 + table a first-class input, so that change replaces the totals step + without changing the experiment file's shape. +- **P4. Lab-specific cleanup stays the lab's.** Swapped samples, pooled + resequencing and failed tubes are the lab's business. The pipeline takes + a clean tube table and documents its format. It may offer a hook for a + per-experiment cleanup script, but it encodes none of that cleanup. +- **P5. Every run records its provenance.** Each CLI writes the tfscreen + version and git commit into its outputs, and the orchestrator also + records input checksums. The study READMEs had to infer commits from + file times (2026-10-03). + +## Steps + +0. [x] **Provenance in every CLI** (P5). `generalized_main` prints, and + each writer records, the version, the commit (with a dirty flag) and the + command line. Do this first and small; it pays off at once. + Done in 0.5.0 (`util/provenance.py`): every run prints it and writes + `{out_prefix}_provenance.json`; the configure YAML and fit checkpoints + embed it. Posterior `.h5` files do not yet; their run's JSON sits + beside them. +1. [x] **Processing takes OD600 directly** (seam 1). + - `tfs-process-counts` takes the per-tube OD600 table and a + `tfs-calibrate-od600` file. It computes `sample_cfu` and its SD + itself, keeping the shared curve error apart from the per-reading + noise (C11). Supplied `sample_cfu` stays available. + - It processes several libraries in one call, from one tube table with + a `library` column. That replaces the per-library runs and the join. + - Document the tube-table and OD600 formats with a synthetic example in + `examples/`. + - The spike list and the M42I codon problem came from a processing + config that disagreed with the real spikes. Check at configure time + that every spike named in the library YAML is present in the counts. + Done in 0.5.0: `--od600_file`, `--od600_calibration_file`, + `--tube_volume_mL` (`od600.tube_totals_from_od600`); several libraries + already worked in one call (per-library filter and fill), now tested; + `examples/process_raw/`; `tfs-configure-model` refuses a missing spike + unless `--allow_missing_spikes`. `tfs-process-presplit` became + `tfs-process-counts --presplit`. +2. [x] **Configure exposes everything that was hand-edited** (seam 2). + - Flags for `sigma_fixed` on level offsets and for every + `theta_*_hyper_scale_fixed`. + - A growth-prior step for growth-only models, replacing + `set_growth_priors.py`. Per-condition k/m loc and scale come from + defaults, a table, or wt monoculture rates (the `monokan` rule: + k = rate at the high gauge concentration, m = the difference). The + pre-fit stays for joint models. + - Decide whether the growth-only defaults change: n held at 0.5, X_low, + X_delta and log K held, offsets at 0.17. The real fit and the + sensitivity runs say how much each matters + (`planning/analysis-roadmap-summary.md`). + Flags done in 0.5.0: `--set_priors name=value ...` (any scalar prior, + by full name or unique suffix), `--growth_priors` (per-condition table) + and `--growth_priors_wt_rates` (the monokan rule, hill_relative only), + in `tfmodel/priors_edit.py`. Closed 2026-10-05. The defaults did not + change (user, 2026-10-04); the decision moved to + `planning/offset-mode-growth-transition.md`, since the held SDs and the + offset SD are likely to change with the growth-transition model. +3. [x] **The staged MAP inside `tfs-fit-model`** (seam 3). + - With level offsets on, the fit stages itself. First a MAP with the + offsets held at 0. Then each tube's offset alone, with everything else + held (what `small_offsets.py` did, cheap). Then the joint MAP from + there at a small step size. + - Each stage writes its own checkpoint and convergence record, so a + stage can be inspected or resumed. + - This retires `make_init_npz.py`, `make_warm_start.py` and + `check_warm.py`. Keep `--init_from` for experiments. + - Test on a simulation: does a cold level-offset MAP reproduce the ±2.8 + offset trap, and does the staged one avoid it? Then on the real data. + Built after 0.5.0: `tfs-fit-model --stage_offsets` (default `auto`), + `inference/staged_map.py`, with unit and smoke tests. The trap was only + ever seen on the full dev data, so the test runs there first + (`planning/studies/staged-map/`); a full-size simulation waits on step + 6. Retire the hand scripts once the study passes. + Study result (2026-10-05): the staged MAP avoids the offset mode, but run + to convergence on the exact full-batch loss (also added) the offset mode + scores best: it is the model's preferred optimum, not a trap, and its + IPTG-structured offsets point at a missing selection-onset transient + (roadmap 7b). + Closed 2026-10-05 as machinery (user): the staged MAP and exact-loss + convergence work, and the staged fit is the physical one (3.7e5 nats + better than `rel_off_n05` as it was used). The open problem is the + model, carried by `planning/offset-mode-growth-transition.md`: under + the current model the offset mode is the better optimum, so a longer + or better fit can still leave the physical basin. Step 7's offset + diagnostic guards against that until the model is fixed. The hand + scripts retire in step 7. +4. [x] **Posterior defaults for large libraries.** Choose the Laplace + automatically from the library size: the full Laplace below a + parameter threshold, the arrowhead (`--laplace_blocks --laplace_shared`) + above it. Report the held Schur directions in a file as well as the + log, since they say which k/m directions have no interval. + Done in 0.5.0: `tfs-sample-posterior --laplace auto|full|arrowhead|blocks|point`, + auto switching at `--laplace_max_params` (20,000 MAP parameters, about + 1.6 GB of float32 Hessian); `{out_prefix}_held_directions.csv`. A model + whose genotypes couple (hill_mut) cannot use the arrowhead, so auto + fails on a large one of those; there is no full-library route for it. +5. [ ] **The orchestrator.** `tfs-run-experiment experiment.yaml`. + Deferred 2026-10-05 (user): a convenience over the CLI chain, which + steps 6 and 7 document and test first. + - The YAML names the inputs (counts directory, tube table, OD600 table, + calibration, library YAML, optional binding and monoculture rates) + and the choices (model, priors, staging, posterior, predictions). + - It runs processing, configure, priors, fit, posterior, extract and + predict in order, calling the existing CLI functions. + - Each stage writes into its own subdirectory and is skipped if its + outputs are already there, so a run is resumable and a stage can be + rerun by deleting it. + - It writes a manifest: commit, input checksums, the resolved YAML. + - It can emit a SLURM script for the fit and posterior stages. + Cluster-specific settings (account, partition, memory) come from the + YAML, never from code. + - Scoring (`score_counts.py`) becomes a CLI if the orchestrator reports + a fit's count likelihood; otherwise it stays a study script. +6. [x] **The simulator writes the same raw formats**. + - `tfs-simulate` writes per-tube count files in `tfs-process-fastq`'s + format (`counts_.csv`: `genotype`, `counts`, with the + `__unknown__` row), the tube table and the OD600 table in step 1's + formats, so a simulated experiment enters at `tfs-process-counts`, + exactly where real data do. Ground truth stays in its own files. + - Keep the current direct outputs (`tfs_sim_growth.csv` and the rest) + until every study grid and example has moved to the raw path, then + decide whether to drop them. + - Test: `tfs-process-counts` on the simulator's raw output reproduces + the simulator's own growth table (same counts, tube totals from the + simulated OD600 through the same calibration). + - Then the full-size simulation of the real design (library, tube grid, + depth, OD600), as a study. + Raw formats done 2026-10-05 (`simulate/raw_output.py`, written by + `tfs-simulate` by default): the round trip matched the direct table + exactly on the simulate-and-analyze example (51,000 rows) and, through + OD600, on the simulate example; `examples/simulate-and-analyze/run.sh` + uses it. Raw presplit is not written yet (the presplit table is still + direct). Left: the full-size simulation of the real design. + Full-size simulation set up 2026-10-05: `planning/studies/full-size-sim/` + (config derived from the dev data by `make_sim_config.py`; the + simulator's new `design` key copies the real 118-tube layout), to run on + the cluster. A local pilot ran (85 min, 68 GB peak); `cfu0` is now + calibrated per library from a pilot. The calibrated simulation still + misses the real tube totals in a structured way (IPTG, time), which is + recorded for step 8. +7. [x] **Validate and document** (after step 6). Without the orchestrator: + - Run the real data and the full-size simulation through the CLI chain + with no hand step, and write the commands down as the recipe + (`planning/studies/real-data-fit/`, and a page in `docs/`). + - Add the tube-offset diagnostic: `tfs-summarize-fit` reports each + level-offset fit's offsets in prior SDs and their correlation with + titrant within each selection condition, and flags a fit whose + offsets carry structure (the offset mode). Pass test for this step: + both runs reproduce, and the diagnostic is clean or flagged. + - Update `docs/` and CLAUDE.md, and retire `make_init_npz.py`, + `make_warm_start.py` and `check_warm.py` (frozen in the study; no + longer part of any recipe). + Code and docs done 2026-10-05: `tfs-extract-params` writes the tube + offsets labeled by tube (`sample_offset/_tubes.py`), and + `tfs-summarize-fit` reports them in prior SDs with a Spearman trend + against titrant and time per condition, flagging BH q < 0.05 + (`tfmodel/analysis/tube_offsets.py`; JSON `tube_offsets`). The recipe is + `planning/studies/real-data-fit/fit/run_recipe.srun` and + `docs/source/pipeline.rst` (step 7, "Check the fit", added). One seam + found and closed: `prep_dev_data.py`'s tube table carried OD600 and + totals both, which `tfs-process-counts` refuses, so it now also writes + `tube_table.csv` and `tube_od600.csv`. The hand scripts are marked + retired in the study README. Left: the two cluster runs (recipe, and + `full-size-sim`), each checked for the diagnostic's verdict. + Recipe run 2026-10-06: end to end, offsets clean, reproduces + `staged_auto2` (log K r 0.992; README "Recipe"). The full-size + simulation ran end to end and its offsets are clean, but it found a + simulator bug (hill_geno drew wt like any genotype; seed 1's wt was + flat), fixed the same day. Corrected arms 2026-10-07: chain clean, + offsets clean; X undercovers, worse with depth (95% coverage 0.66 + realistic, 0.29 Poisson), from shared k/m/dk_geno errors the arrowhead + Laplace holds at the MAP (`planning/studies/full-size-sim/` results). + Step 7 done 2026-10-07: both runs reproduce through the documented + chain and the diagnostic gives a verdict; the recovery numbers are step + 8's baseline. Also found: the low overall X r is depth, not k/m (r + 0.96-0.99 above 1e4 reads), and the simulated depth distribution does + not match the real screen (real median double 480 total reads against + ~1e4 simulated; 64% of real doubles below 1e3). Both in the study + README. +8. [ ] **Then: prioritize the science** (user, 2026-10-05). With steps 6 + and 7 in, any model change can be tested end to end on simulations a + user can reproduce. Step back and list the model work (the growth + transition, the offsets' freedom, the population model, the + growth-binding map, and others), how the changes interact, and what + order to do them in. From step 7's simulation, on the list: match the + simulated library's depth distribution to the real screen first (every + simulated recovery number depends on it); anchor k against dk_geno and + give the held shared directions an interval; hold the X population SDs + (shrinkage for shallow genotypes); pooling across doubles (hill_mut) + for the 64% of real doubles below 1e3 reads; a level-like simulated + tube noise matched to the real offsets. Not to be decided before steps 6 and 7. + +## Out of scope + +- The population model and the OD600 likelihood (roadmap step 6), and the + growth-transition decision (7b, now + `planning/offset-mode-growth-transition.md`). This plan only keeps room + for them (P3). +- The growth-binding relationship study (summary, Part 3, item 1). +- Workflow engines (Snakemake, Nextflow). A Python orchestrator over the + CLI functions is enough at this size, and adds no dependency. + +## Open + +- Whether the orchestrator replaces `tfs-build-empirical`'s internal + configure and pre-fit, or calls it. +- How binding data enter a growth-only experiment file. Today they are only + compared after the fit; roadmap step 9 brings them back into the model. diff --git a/planning/findings-ledger.md b/planning/findings-ledger.md new file mode 100644 index 00000000..2dd12b61 --- /dev/null +++ b/planning/findings-ledger.md @@ -0,0 +1,210 @@ +--- +title: Findings ledger for the screen's model (established, refuted, open) +status: active +filed: 2026-10-18 +area: tfmodel +revisit_when: >- + Read before filing a new plan or study on the growth/binding model, + lambda, the k/dk_geno slide or the slope m; add a line whenever a study + establishes, refutes or reopens something. +related: + - planning/joint-fit-return.md + - planning/analysis-roadmap-summary.md + - planning/growth-only-relative-fit.md + - planning/congression-physics-plan.md + - planning/two-stage-fit.md + - planning/absolute-abundance-measurements.md + - planning/studies/joint-fit-sim/README.md + - planning/studies/joint-fit-real/README.md +--- + +## Why this file + +On 2026-10-18 a review of the planning files, the studies and the git log +found that a week of joint-fit work had re-found several results from July +and early October (listed under "Re-found" below). Plans had cited history +only back to their own start. This page collects what is known, so a new +plan starts from it. Each entry names its source; `planning/dev-data/` is +gitignored and is cited by file. Dates are when a result was found. + +## Established + +**Growth identifies occupancy only up to an affine map.** k and m absorb any +affine change of theta (`g = k + dk + m theta`), so growth alone cannot fix +theta's scale (CLAUDE.md, Relative-X fit; roadmap step 5). Every "squeeze" +below is this direction: the doubles' pull on m, the singles' squeeze in +an absolute-theta stage 1 (joint-fit-sim run 6: theta_fit = 0.048 + 0.90 +theta_true), and the slide of X against k. + +**Binding genotypes are what resist the squeeze, as about 1/sqrt(n).** +July: about 20 in-library binding anchors with m free gave a theta slope of +0.91 and m ratios 0.95/1.14, robust to a wrong lambda (memory +`project_theta_recovery_congression_confound`, `out11/libexp`, 2026-07-08). +Quantified 2026-10-18 (joint-fit-sim run 10): without congression the +growth-only singles' theta RMSE is 0.010 + 0.053 / sqrt(n + 2) for n = 0-50 +binding singles, insensitive to which singles; coverage 0.11 to 0.51. +This holds only if binding and in vivo occupancy share one scale (see Open). + +**The full-library MAP steepens m about 1.45x through the doubles** (not +congression: still 1.37 without it); the two-stage fit (k and m from wt, +spikes and singles, pinned for the full library) removes most of it +(m-bias-factorial rounds 2-5, 7fcef80c, b8be0d2b, 2026-10-10/12). Stage 1 +itself keeps a smaller bias (2-6% relative; 14% absolute without binding +singles, joint-fit-sim run 5). + +**The markers read occupancy through differently shaped maps:** kanR +switch-like (Hill n at the bound of 4 on X), pheS graded (n about 1); one +shared linear X blends them; a logistic warp of kanR gains about 8,000 nats +(relative-m runs 2-4, 6aa9ae4b, d540edfc). In an absolute-theta model the +warp, s and per-genotype offsets are recoverable on simulations when the +full model starts from the plain one (joint-fit-sim runs 1-3). + +**lambda absorbs misfit.** Converged fits recover it on simulations +(0.004 at 0, 0.98 at 1; count likelihood 0.02-0.06 at 0), but its profile +is flat over 0.6-1.4, seeds spread it 30-50 posterior SDs, and it runs away +when the model conflicts with the data: 0.6-2.4 in July (2.41 with m +clamped against binding anchors), 0.82-2.10 in joint-fit-real run 1. Real +fits that did not conflict gave 0.17-0.37 (congression-calibration README; +congression-physics-plan step 3.5; m-bias-factorial lines ~1226-1236; +dev-data real_fit 2026-10-02). + +**The congression theta rule is not identifiable from the screen** +(cross-rule grid, 3d60dfd2): homodimer is the physical default; +`hill_relative` needs `max` (the only rule invariant to an increasing map). +The fit's fixed co-resident draws leave about 3 points of m bias however +many there are (joint-fit-sim run 9); more draws (75) are cheap except in +Laplace memory. + +**The k/dk_geno/k_ref slide is a symmetry, not a bug** (`k += C, dk -= C`). +Per-condition k priors (128cd477), base_growth/k_ref (7e083271) and the wt +monoculture anchor (34b8143a) each failed to close it on its own; pinning +k and m from stage 1 nearly closes it (median dk error -0.0057 to -0.0008); +what remains is the dk_geno population location against wt, which the +stage 2 Laplace holds (two-stage-validation, 7d01721a). + +**dk_geno is not rankable from the screen:** the two control conditions +disagree (r = 0.03-0.04) and the noise exceeds the spread (memory +`project_dk_geno_not_rankable`; deep-coverage step 3). + +**The binding weight was not the driver of the joint fit's distortion.** +At the old default (about 251 x SD 0.001) binding dominated the objective; +at weight 1 theta coverage rose but the mixture's drift stayed +(congression-calibration second diagnosis, grid_bw1, 2026-09-24). The +default is 1 since 45ffb7fe. + +**Theta intervals undercover because of the shared parameters' bias, not +the per-genotype uncertainty:** at the true k and m without congression +they cover 0.91; with congression 0.70 (the mixture adds per-genotype error +the model does not carry); fitted, 0.1-0.5 (joint-fit-sim runs 5, 10). + +## The binding-to-growth link (the central open problem) + +Facts, each found more than once: + +- **wt's IPTG midpoint in growth is about 10x its 2 uM binding midpoint:** + growth near 30 uM IPTG, 2 uM binding near 3 uM; 20 uM binding matches + wt's midpoint (x-vs-binding, 2026-10-15; joint-fit-real run 1). An + earlier form with 10 uM binding: dev-data real_fit, 2026-09-29. +- **wt is not fully induced in vivo:** about 70% at 1 mM where binding says + 93% (2026-10-01; roadmap summary), a floor of 0.15-0.24 in run 1. +- **Growth curves are broader than binding curves:** spike growth n + 0.6-1.2 against binding n 1.6-2.5 (2026-10-01). Rechecked on the + kanR-only two-stage X: n 0.7-0.95 for most genotypes against 1.9-3.2 at + 20 uM (x-vs-binding run 2). +- **M42I's order inverts:** binding K about 10x wt's at both 2 and 20 uM; + growth K about half of wt's (6.4 against 12 uM IPTG). Protein + concentration does not move that ratio, so no single concentration + removes it; per genotype, wt prefers about 12 uM protein and M42I about + 1.9 (dev-data real_fit 2026-10-01, `scan_protein.py`). Rechecked on + the two-stage, kanR-only X (x-vs-binding run 2, 2026-10-18): M42I's K is + 0.18x wt's in growth against 3.6x (2 uM) and 5.3x (20 uM) in binding, a + factor of 20-30; M42I/H74A keeps the order but sits 10x lower in growth; + D88A and V95N agree within 2x. +- **No single protein concentration or shared monotone map carries binding + onto growth** (best shared concentration 4.5 uM, rms 0.285; isotonic + 0.21-0.25; departures 0.3-1.0 against interval widths 0.15-0.35; + 2026-10-01, "no map"). +- **The 2 uM agreement that motivated rejoining (kanR X against binding, + rms 0.12) was in a gauge built from wt's own binding endpoints**, which + absorbs any affine level and scale difference; in an absolute model the + difference reappears (joint-fit-real run 1: every binding offset negative, + -0.07 to -0.28). +- **Westerns put total repressor in the 100s of uM,** so an "apparent + protein concentration" is a lumped mapping, not a concentration + (dev-data README, binding plan). + +So binding cannot yet set theta's absolute scale: every route that uses +it that way (the 2026-09 joint fit, roadmap step 8's maps, the 10 uM +interpolation, joint-fit-real run 1) has met these facts. + +## Refuted or retired + +- Up-weighting binding (N_growth/N_binding) as needed for binding to + matter; and as the cause of the joint fit's distortion (see above). +- One shared X for both markers (D5's test failed; relative-m). +- OD600 tube totals as the cause of the within-tube/fit m gap (freeing the + totals left the ratio at -0.49; relative-m run 1). +- A free per-concentration IPTG k (unidentified; within-tube-m run 2). +- Tighter or moved theta population priors as a fix for the squeeze + (joint-fit-sim run 6: moving the location changes nothing; holding the + SDs at the truth makes it worse). +- The k-shift refits for k's uncertainty in X (they leave the MAP and + slide; two-stage-validation). +- "The prefit zeros dk_geno, so k is low" (July; disconfirmed: wt anchors + k). + +## Open + +1. **The binding-to-growth link** (above). Candidates on record: cell-to-cell + variation in IPTG uptake or repressor level (broadens and shifts in vivo + curves; testable on wt first); a binding model in protein concentration + with a fitted in vivo concentration; an affine calibration between the + assays. The bench tests on record: monoculture growth of the spikes + across the titration (`planning/absolute-abundance-measurements.md`); + binding on more singles helps only once the link holds. +2. ~~Whether M42I's inversion and the broad growth curves survive on the + two-stage, kanR-only X.~~ Both survive (x-vs-binding run 2, 2026-10-18). + Both genotypes carrying M42I respond at lower IPTG in vivo than in vitro + relative to wt; a genotype-specific in vivo factor (repressor level?) + is a hypothesis, testable with expression data. +3. **lambda's prior:** fixing it from the measurement was proposed + (2026-09-26, congression-physics-plan step 3.5) and never done; later + decisions loosened it ("a readout, not a measurement", 2026-10-02; + "keep it fitted", 2026-10-11). Unreconciled. wt purity (excess wt, about + 11-16% of the pool) moves fitted lambda 2x (0.365 cfg against 0.174 + wtspiked) and is still open at the bench. +4. **The dk_geno population location in stage 2:** holding it at stage 1's + value is proposed, not run. +5. **Honest theta intervals:** the shared-scale error (carry its uncertainty + into each genotype, as `--shared_shift` does for k) and congression's + per-genotype error (the mixture's form). The k shift from stage 2 + refits fails both ways tried: at step size 1e-3 the refits slide along + the dk_geno population (real X +0.32 with k unchanged; two-stage + validation run 1), and with that population held they under-respond + (a third or less of the true shift on simulations; run 2). The 0.5.0 + release ships without it: deep coverage 0.88-0.91 up to 1e5 reads, + 0.76-0.81 above (2026-10-20). +6. **The theta_low/dk_geno ridge at 0 IPTG** (July): proposed fixes (drop 0 + IPTG from epistasis inputs; base_growth on genotypes other than the + binding ones) never tested. + +## Contradictions to keep in mind + +- Study 0a (2026-09-26, buggy data, 22 uM protein) put growth's response at + 30-100x lower IPTG than binding; every later comparison puts wt's at + about 10x higher IPTG than its 2 uM binding. +- Within-tube pheS m 0.018 (within-tube-m run 3) predates the swapped-row + fix; without the row it is 0.0125. +- July "wt alone anchors k" against September/October "wt's pin is too + weak against the slide" (different models and scales). + +## Re-found on 2026-10-17/18 (joint-fit-sim, joint-fit-real) + +- Binding singles pin m (July's in-library anchors). +- The singles' affine squeeze (the documented gauge symmetry). +- lambda running away under a model conflict (July). +- wt's 10x midpoint offset and its in vivo floor (2026-10-01, 2026-10-15). +- New in that work: the joint model's pieces recover on simulations; the + full model must start from the plain one; the 1/sqrt(n + 2) scaling; the + mixture's fixed-draw floor; intervals honest at the true k and m without + congression. diff --git a/planning/growth-only-relative-fit.md b/planning/growth-only-relative-fit.md new file mode 100644 index 00000000..e13b4ac7 --- /dev/null +++ b/planning/growth-only-relative-fit.md @@ -0,0 +1,183 @@ +--- +title: Fit growth alone on a wt-relative scale, then measure the growth-binding map +status: promoted +filed: 2026-09-26 +area: tfmodel +revisit_when: >- + When we want to test, rather than assume, that in vitro binding and in vivo + growth report the same occupancy. Step 0 (model-free look) can be run any + time on the existing real data. +related: + - planning/analysis-roadmap.md + - src/tfscreen/tfmodel/model_orchestrator.py + - src/tfscreen/tfmodel/generative/model.py + - src/tfscreen/tfmodel/scripts/configure_model_cli.py + - src/tfscreen/tfmodel/scripts/prefit_calibration_cli.py + - src/tfscreen/tfmodel/analysis/prediction.py + - src/tfscreen/tfmodel/generative/components/theta/hill_geno.py + - src/tfscreen/tfmodel/generative/components/theta_rescale/ + - planning/congression-physics-plan.md + - planning/studies/congression-calibration/README.md +--- + +**Promoted 2026-09-26** into `planning/analysis-roadmap.md` (Track G: steps +0a, 1, 5, 8, 9). Step 1, the growth-only plumbing listed below, is done; +step 5, the relative-X fit, is implemented as `theta: hill_relative` (gauge: +wt pinned at the measured concentration range's ends), with its validation +grid in `planning/studies/relative-fit/`. + +**Context:** While calibrating fits for congression, it became clear we want +to fit growth data without binding data, at least to check whether the two +are consistent. An investigation (2026-09-25) found the plumbing is small but +that growth data alone do not fix an absolute theta scale. Discussion then +reframed the goal: don't ask for absolute theta yet. + +The joint fit currently asserts, at minimum: + +1. In vitro binding equals in vivo occupancy (same effective [IPTG], same + [protein], no genomic-site competition, ...). +2. Growth is linear in that occupancy: `g = k_c + dk_g + m_c·A_g·θ_g`. + +It also overweights binding: each binding row is scaled by +`N_growth_rows / N_binding_rows` (`model_orchestrator.py`, `binding_weight` +auto-computation), so the binding set as a whole counts as much as the entire +growth dataset. Any binding/growth mismatch is resolved by distorting growth. + +**Idea:** Infer a hidden, IPTG-dependent growth variable X from growth alone, +on a scale defined relative to wt. Then ask empirically what transfer function +maps X to the theta measured in vitro. Base case: linear. Likely case: +monotone but nonlinear. Worst case: no map. Only after that, build whatever +map we find into a joint fit. + +*Why a relative scale is identifiable.* Growth data fix X only up to a global +affine map `X -> a·X + b` (m and k compensate); this is exactly the +scale/sign non-identifiability of theta without binding (m -> c·m, +θ -> θ/c; θ -> 1-θ, m -> -m, k -> k+m). Hill K and n are invariant under it, +as are ratios to wt (e.g. a genotype's dynamic range relative to wt's). Fix +the gauge by convention: `X_wt = 1` at no IPTG, `X_wt = 0` at saturating +IPTG. Then `k_c` = wt's growth at saturating IPTG and `m_c` = wt's growth +change across IPTG, read straight from abundant wt data; the sign of m falls +out per condition. No binding data and no binding-based prefit needed. +(Design choice: pin wt's Hill asymptotes, simplest, or its values at the +measured end concentrations, if wt does not saturate in range.) + +*What stays assumed.* +- One X per genotype × [IPTG], shared across conditions and pre/sel phases. + Within a condition "growth linear in X" is a definition; across conditions + it is testable (kan and 4CP responses should be affine in each other per + genotype). +- X is Hill-shaped in IPTG. A strongly nonlinear true map would show as lack + of fit. +- dk_geno is independent of IPTG and condition. The baselines are the weakest + quantity: a genotype's X at no IPTG trades against dk_geno and is separated + only by selective-vs-control contrasts (control conditions have not ranked + dk_geno well so far). +- Congression: the mixture's `max` theta rule is invariant under any + increasing map (`max f(θ) = f(max θ)`), so it carries over if X is oriented + to increase with repression. The `homodimer` default (congression step 4) + and `heterodimer` take `logit(θ)` and are not invariant; a relative fit must + use `max` or `single` (`planning/analysis-roadmap.md`, C1). + +**Step 0 (model-free, do first).** For the genotypes with binding data: +regress ln_cfu on t_sel per genotype × replicate × condition × concentration +(slope = selection-phase growth rate, since t_pre is fixed within a sample); +subtract wt (cancels per-tube artifacts, leaving `dk_g + m_c·(X_g - X_wt)`); +plot against binding theta, interpolating binding through its Hill fits if +concentrations differ. Spiked genotypes are deep and congression-free. One +collapsing curve → linear/monotone case; genotype-specific scatter → the +worse case. This answers which case we are in before any component is built. + +**Stage 2 design (the map).** +- The map is between curves, not points: `X_g(c) = f(θ_g(s·c))` with a global + concentration scale s (in vivo effective [IPTG], assumption 1) and a value + map f (assumption 2). Nested family: identity → affine f → monotone f, plus + per-genotype deviations (activity, in-cell effects). +- Stratify spiked vs bulk: congression should show as bulk genotypes on a + compressed version of the spiked curve. +- Errors in both variables; genotype is the unit, concentrations are + within-genotype points. +- Risk: converged mean-field SVI is overconfident (bulk theta 95% coverage + ~0.66 in the congression-calibration study), which would make a real + monotone map look like "no map". Needs an extra dispersion term or better + calibrated posteriors. +- Eventual formalization: `theta_rescale` is already a fixed transfer + function slot (`passthrough`/`logit`); the joint model would make it + learnable and add s, with binding at weight 1. + +**Why not now:** Current work is the congression physics plan; this is a +change of strategy for what the fit asserts, and step 0 should decide whether +it is worth building. + +**What it would take:** + +*Growth-only plumbing* (found 2026-09-25; no model component reads +`data.binding`, and batching, `get_batch`, `batch_safety`, `RunInference`, +extraction, `predict-theta`/`-epistasis`, `model_stats` counts and config data +paths already cope with no binding): +- `configure_model_cli.py`: `binding_df` is a required positional that raises + when None → make it `--binding_df`, require at least one of growth/binding + in the body (like `--library_config`). Line ~389 writes a phantom + `data.binding: binding.csv` when binding is absent. Breaks positional + callers: `examples/simulate-and-analyze/run.sh`, `test_base_growth.py`, + `test_configure_and_run.py` (which asserts `configure_model(None)` raises). +- `ModelOrchestrator`: joint `_initialize_data` builds binding tensors, + `BindingData` and auto `binding_weight` unconditionally (first failure at + `_read_binding_df`); gate them. `_setup_batching` with an empty binding set + already gives `num_binding=0`. In `_initialize_classes`, drop + `theta_binding_noise`, skip `observe_binding`, set `priors.binding=None` + explicitly (`populate_dataclass(BindingPriors, {})` raises). Infer the mode + from `binding_df is None` rather than adding a `growth_only` config key + (every `components:` key becomes an orchestrator kwarg). +- `generative/model.py`: gate the binding prediction, noise, deterministics + and observer on `data.binding is not None` (static structure, jit-safe). +- `analysis/prediction.py`: `orchestrator.binding_df.copy()` crashes, breaking + `tfs-predict-growth`, `tfs-sample-prior` and summarize-fit trajectories. +- `summarize_fit_cli.py`: with no binding genotypes the trajectory plots fan + out to every genotype (a CSV + PDF each); needs a default subset. +- Fail fast on `binding_weight` or non-`zero` `theta_binding_noise_model` + without binding. Minor: `predict_growth_cli.py` survives only via a + try/except; `model_stats` warning wording; setup-grid `auto` enumerates + `theta_binding_noise`; `None` prior leaves round-trip through the priors CSV + as NaN (pre-existing, binding-only has it too). +- Tests: none cover growth-only. `growth-smoke.csv` + `library-smoke.yaml` + without binding is a ready fixture. Needs analogues in `test_model.py`, + `test_model_orchestrator.py`, `test_batch_safety.py`, `test_model_stats.py`, + `test_configure_and_run.py`, and a smoke test. + +*Relative-X fit:* +- New theta component (e.g. relative Hill): hill_geno's curve form but with + real-valued baselines (not logit-bounded to [0,1], since mutants can fall + outside wt's range), wt baselines pinned to (1, 0), population priors + re-centered on wt. Must follow the registry rules (library-sized plates, + `return_population`, `{site}_loc(s)`/`{site}_scale(s)` naming). +- Force `activity=fixed` (redundant with X's amplitude), `passthrough`, + `linear` growth; per-genotype curves only (hill_mut's logit additivity and + the thermo components presuppose absolute theta). +- `tfs-prefit-calibration` (needs binding by construction) becomes + unnecessary or a wt-only fit. +- Downstream: logit epistasis, `in_regime` and anything assuming θ in (0,1) + do not apply to X; only additive-on-X epistasis until f is known. + +*Stage 2 analysis:* a study script first (`planning/studies/`), a CLI only if +it earns one. + +*Validation:* on one sim, compare joint vs relative-X fits; +`tfs-summarize-calibration` stratifies by `has_binding` from the simulated +binding file (not what the fit saw), so held-out comparison is free. A sim +with a known nonlinear f and an [IPTG] scale s would test whether stage 2 +recovers them. + +*Alternative considered (absolute theta without binding in the fit):* keep +absolute theta but take m from binding only through the prefit +(`tfs-prefit-calibration --binding_df` for a growth-only config, with +`--pin_m`), or copy a calibration from another run, or rely on priors. Set +aside because it still asserts assumptions 1 and 2; the out11 ablation also +found free m with ~20 in-library binding anchors beat the prefit clamp +(theta slope 0.91 vs 0.85; older machinery). + +**Open questions:** +- Gauge: decided (roadmap D4): pin wt at 0 and 1 mM IPTG. +- Shared X across conditions: decided (roadmap D5): shared, with a kan-vs-4CP + consistency check. +- Do growth and binding concentrations coincide in the real data? +- Default genotype subset for summarize-fit trajectories without binding. diff --git a/planning/joint-fit-return.md b/planning/joint-fit-return.md new file mode 100644 index 00000000..778e9232 --- /dev/null +++ b/planning/joint-fit-return.md @@ -0,0 +1,289 @@ +--- +title: Return to a joint fit of growth and binding, with per-marker maps +status: active +filed: 2026-10-16 +area: tfmodel +revisit_when: >- + Agreed 2026-10-16. Supersedes the open steps of planning/relative-m.md and is roadmap + step 9 (planning/analysis-roadmap.md) with step 8's map measured. +related: + - planning/growth-only-relative-fit.md + - planning/analysis-roadmap.md + - planning/analysis-roadmap-summary.md + - planning/deep-coverage.md + - planning/two-stage-fit.md + - planning/relative-m.md + - planning/studies/real-data-fit/README.md + - planning/studies/m-bias-factorial/README.md + - planning/studies/two-stage-validation/README.md + - planning/studies/x-vs-binding/README.md + - planning/studies/within-tube-m/README.md + - planning/studies/relative-m/README.md + - src/tfscreen/tfmodel/generative/components/growth/linear.py + - src/tfscreen/tfmodel/generative/components/theta/hill_geno.py + - src/tfscreen/tfmodel/model_orchestrator.py +--- + +## Review, 2026-10-18 + +Step 6's first run and a review of the planning files and git log +(`planning/findings-ledger.md`) changed the footing of this plan: + +- The 2 uM agreement in "The arc" item 5 was in a gauge built from wt's own + binding endpoints, which absorbs any affine level and scale difference + between binding and growth. In the absolute model the difference + reappears (step 6, run 1: every binding offset negative). +- The arc starts on 2026-09-26 and misses July's work: about 20 in-library + binding anchors already pinned m then, and lambda already ran away under + a model conflict. +- The binding-to-growth link has failed three times (the 2026-09 joint fit, + the 2026-10-01 "no map" with M42I's inverted K, and step 6 run 1), so + binding cannot yet set theta's absolute scale. + +Steps 6-8 are paused until the link is decided (the ledger's Open item 1). +The simulation results (step 5) stand as a test of the model under its own +assumptions. + +## Why + +We split binding from growth on 2026-09-26 because the joint fit asserted +more than the data could support, and step 8 of the roadmap then found "no +map" between growth and binding. Three weeks of work since have removed +most of the reasons for that verdict and found the pieces the original plan +said would be needed. Absolute occupancy is also what the congression +correction (the partition-function rules) and the long-term goal (learning +thermodynamic ensembles from these data) need. This plan brings binding +back, at its proper weight, with the map we have now measured. + +## The arc + +**1. The split (2026-09-26).** The joint fit asserted that in vitro binding +equals in vivo occupancy and that growth is linear in it, and its default +binding weight (`N_growth_rows / N_binding_rows`) made binding count as much +as the whole growth dataset, so any mismatch distorted growth +(`planning/growth-only-relative-fit.md`). The plan: fit a wt-relative X from +growth alone (3f628bf7 the idea, 98fb6db0 growth-only models, 5820c19f +`hill_relative`), then measure the map from X to binding (roadmap step 8: +identity, affine, monotone f, with a global IPTG scale s; spikes and bulk +apart), then rejoin (step 9: learnable f and s, binding at weight 1). It +kept one X shared by both markers (decision D5) and named the test: "across +conditions it is testable (kan and 4CP responses should be affine in each +other per genotype)". + +**2. "No map" (to 2026-10-05).** The first real relative fits +(`planning/studies/real-data-fit/`, c5de9f24) found growth curves broader +than binding (n 0.6-1.2 against 1.6-2.5), M42I's K on the wrong side of +wt's, a common mutant floor near X -0.5, and no shared protein +concentration or monotone map carrying binding onto growth. Step 8 was +recorded as "no map". The tube-offset mode (27109e26) and the full-size +simulation (bf4a1821: X undercovers at depth, |m| 20-40% too large, k off +along the k/dk_geno slide) followed. + +**3. The m bias (2026-10-09 to 10-12).** Anchoring k and m on the wt +monoculture failed (34b8143a): the fit preferred an m about 40% too steep. +The m-bias study (7fcef80c) found the cause: each double pulls m a little +and the pulls add, so the full-library MAP compresses X about 1.47-fold. +The fix is the two-stage fit: k and m from wt, spikes and singles (with the +congression mixture), pinned for the full library (58b83b28, b8be0d2b; the +real X stretches a uniform 1.47). So the "no map" fits had a compressed X. + +**4. The two-stage command and its validation (2026-10-13 to 10-15).** +`tfs-fit-two-stage` (3ce04cc5) reproduced the study, but its k-shift +refits failed: they leave the MAP and slide along the dk_geno population +(7d01721a). The marker-consistency flag fired on 10.6% of deep real +genotypes against 1.9% on simulations, and pheS alone compressed X against +kanR, even with wt's curve held (b6655671). + +**5. Binding again (2026-10-15).** With the new binding data (nine +genotypes, 2 and 20 uM repressor dimer, normalized to day-of controls) and +the two-stage X: **kanR-alone X matched 2 uM binding with no correction** +(rms 0.12 against 0.30 on 2026-10-01), pheS did not +(`planning/studies/x-vs-binding/`). + +**6. Why pheS disagreed (2026-10-15 to 10-16).** In order +(`planning/studies/within-tube-m/`, `planning/studies/relative-m/`): +- within-tube contrasts (tube totals cancel) disagreed with stage 1's m in + both markers, but freeing the totals did not move m (e5501aba): not OD600; +- a log-linear IPTG term shared by every genotype helped kanR a little; + a free one was unidentified (aea7f6f4, d8060513); +- the markers read occupancy through differently shaped maps: kanR + switch-like (n at 4), pheS graded (n about 1), wt alike (6aa9ae4b). This + is D5's test, and it fails: one shared linear X blends a steep readout + with a broad one, which alone can make "growth broader than binding"; +- one row of pheS tubes was swapped (3d5fbc43; the lab agrees it is + probable; `tfs-check-tubes` 76ce30d3, `tfs-correct-tubes` f15f5b4c); +- a logistic warp of kanR's growth in occupancy lowers the loss by 8,000 + nats and is nearly linear on a simulation (d540edfc, a1670702); +- relative information alone cannot fix the scale of X: each estimate of m + rests on a different anchor (tube totals, priors and Hill shape, or weak + binders being flat), and they differ 2.5x in kanR (ba1752ba, 9a0a2f80, + b86c0ecb). Binding at 2 uM supports the larger kanR m. + +## What we know now + +- kanR growth is close to linear in 2 uM binding theta for the spikes, + once X is not compressed. +- pheS growth is a graded, differently shaped function of the same + occupancy; one shared linear map cannot serve both markers. +- The scale of the growth variable needs an external anchor; binding is + the natural one. +- Still open: M42I grows 0.3-0.4 below its 2 uM binding at low IPTG; wt's + IPTG midpoint matches 20 uM binding while the mutants' levels match 2 uM + (an in vivo IPTG scale s, or an effective protein concentration). + +## Carry over + +- The count likelihood and the staged MAP. +- The two-stage fit for the doubles' m bias (stage 1: wt, spikes and + singles; stage 2: full library with stage 1's growth parameters pinned). +- Per-marker maps from occupancy to growth instead of one shared linear X + (`linear`'s `m_warp`). +- The corrected tube table (`tfs-correct-tubes`) and the tube QC + (`tfs-check-tubes`). +- Binding at weight 1: each binding observation counts as one observation. + The default `N_growth_rows / N_binding_rows` goes. + +## Design + +**theta.** Absolute occupancy per genotype from a Hill curve in IPTG +(`hill_geno`), shared by both markers and observed by binding. + +**Growth.** g = k_c + dk_g + m_c f_c(theta) per condition, with f_c the +per-marker map: start from `m_warp`'s logistic warp (f(0) = 0, f(1) = 1), +for both selection markers, now identifiable because binding fixes theta. +Whether kanR needs a warp at all is a result, not an assumption. + +**Binding.** theta_obs at 2 uM against the genotype's theta at an IPTG +scaled by a global s (in vivo IPTG = s x nominal), weight 1. A per-genotype +deviation between in vivo and in vitro (a hierarchical offset on the +binding genotypes' logit theta or log K, with a learned scale) lets a +genotype like M42I disagree without moving the rest. Binding sets the +scale of theta and the maps; the 200,000 growth genotypes set their own +curves. + +**Congression.** With absolute theta, the partition-function rules +(`homodimer`, the physical default) return to stage 1's mixture. + +**Two stages.** Stage 1 (wt, spikes, singles, binding) fits k, m, the +maps, s and the deviation scale; stage 2 (full library) pins them. The +k/dk_geno slide and the intervals for k and m are revisited once m is +anchored by binding (the k-shift refits failed and are not reused). + +## Steps + +1. [x] **Agree this plan** (user, 2026-10-16). +2. [x] **Binding input.** `tfs-process-binding` (2026-10-16): the lab's + table to `binding_df` at one protein concentration (default 2 uM), + replicate means, `theta_std` from a pooled replicate SD. On the + 2026-10-15 table, 2 uM gives 110 points over 7 genotypes (D88A and + V95N were measured only at 20 uM). Only wt is replicated there (five + days), so the pooled SD (0.13, 31 dof) is wt's day-to-day scatter, about + three times the point-to-point scatter within one curve. A day effect + is shared by a whole curve, so treating the points as independent + overstates how well a curve's level is known; step 4(b)'s per-genotype + deviation is where that shared error should go. At 20 uM the pooled SD + is 0.059 (150 dof). +3. [x] **Weight 1** (2026-10-16). `binding_weight` None now resolves to 1; + an explicit weight is kept. Stored configs carry their resolved weight, + so none change; the count-likelihood, congression-calibration, + svi-overconfidence and real-data-fit studies already passed 1. +4. [ ] **Model pieces**, each with tests. + - (a) [x] The global IPTG scale s between binding and growth + concentrations (2026-10-16): `hill_geno`'s + `theta_growth_log_conc_shift_scale` > 0 samples log s; growth sees the + curve at s times nominal IPTG, binding at nominal. Tested on the smoke + data through MAP, Laplace, extraction and growth prediction, single + and mixture. `tfs-predict-theta` reports the curve in binding units; + an in vivo option can follow if step 6 wants it. + - (b) [x] The per-genotype in vivo/in vitro deviation for binding + genotypes (2026-10-16): `theta_binding_noise: genotype_offset`, one + additive theta offset per binding genotype, constant over IPTG, + tau * z_g with a learned tau. Additive in theta because a curve's + day-normalization error and M42I's disagreement are level shifts; a + log K deviation can follow if step 6's residuals show midpoint shifts. + - (c) [x] `m_warp` and the mixture's partition-function rules with + `hill_geno` (2026-10-16): nothing refused them; the warp acts on each + cell class's theta after the mixture combines plasmids. + `tests/tfscreen/tfmodel/generative/test_joint_warp.py` traces every + piece together and checks a near-linear warp reproduces the unwarped + model through the mixture; `tfs-extract-params` now writes + `warp_log_a`/`warp_x0` per warped condition. End to end on the smoke + data (`--growth_priors` with an `m_warp` column, homodimer mixture, + s, offsets): MAP, Laplace, extraction, chunked prediction. +5. [ ] **Simulation first.** Add a per-condition occupancy warp to the + simulator's growth linkage (it has linear, power and saturation, no + warp), simulate kanR switch-like and pheS graded with binding for the + spikes at known s and deviations, and check stage 1 recovers m, the + maps, s and theta, and that binding at weight 1 does not distort growth. + - [x] Simulator keys (2026-10-16): `warp_a`/`warp_x0` on a linear + growth entry, `growth_titrant_scale`, `binding_data.genotype_offset_sd`, + with truth files. + - [ ] The study (`planning/studies/joint-fit-sim/`): joint and null + simulations from the full-size realistic config, fitted with the full + and plain joint stage 1 (three runs). + Run 1 (2026-10-17): on the joint simulation the full model recovers + kanR's m (2%) and warp and s (16%) and halves the singles' theta error + of the plain model; on the null simulation it finds a wrong mode (the + same steep warp on both markers, s 0.25, large offsets, theta off by + 0.3). A warp shared by both markers is pinned only by the binding + genotypes, which the offsets loosen. Next: is that a trap or a + degeneracy (null fitted with the plain model; compare losses). + Runs 2-3 (2026-10-17): it was a trap; the full model started from + the plain fit's MAP recovers the null (near-linear warps, s covering + 1, offsets covering 0) and the joint truth (kanR's switch, s, the + offsets). Open before step 6: m 15-25% steep even with the true model, + and theta intervals covering 14-17%. + Runs 4-7 (2026-10-17): the m bias is an affine squeeze of the + singles' theta (k absorbs the constant, m the slope) that only binding + genotypes resist, pushed by the logit population prior and by the + congression mixture's approximation. Twenty binding singles cut it from + 14% to 6% on the no-congression simulation (weak pooling, 14% to 11%; + both, 5%). Experimental implication: binding on about twenty singles + spanning theta. Still open: the remaining 5%, the congression part + with binding singles, and theta coverage (0.38). + Run 8: with congression, twenty binding singles halve the bias (21% to + 12%), leaving the mixture's approximation as the largest piece; fifty + singles take the no-congression bias only to 5%; the full recipe on + the joint truth with twenty gets kanR's m within 1%, pheS's within 7%, + s covered. Next: the mixture's co-resident draws (`congression_sets`). + Run 9: 75 co-resident sets instead of 16 take the congression null's + bias from 12% to 9% with twenty binding singles; 430 sets only to 8.6%, + against 6% without congression. The remaining 3 points are the + mixture's form, not its draws. Many sets run the Laplace Hessian out of + GPU memory. +6. [ ] **Real stage 1.** wt, spikes and singles with binding at 2 uM, the + swapped row corrected; report m against the within-tube values, the + maps, s, the deviations (M42I), and binding residuals per genotype; + compare likelihoods with and without each piece. + Set up 2026-10-18 (`planning/studies/joint-fit-real/`): plain, then + full from plain, 75 co-resident sets, the current 2 uM binding (seven + genotypes), the swapped row dropped. Expect m 17-21% steep and theta + intervals too narrow (joint-fit-sim runs 8-10); rerun when the updated + binding (and any binding singles) arrive. + Run 1 (2026-10-18): the 2 uM binding sits below growth's theta for every + genotype (offsets -0.07 to -0.28, all negative) and wt's binding + midpoint is about 10x lower in IPTG than growth's; s (0.53) does not + absorb it, lambda runs to 0.8-2.1, and m and the warps are not + interpretable. The binding-to-growth link at 2 uM needs more than s and + per-genotype offsets. +7. [ ] **Real stage 2 and intervals.** Full library pinned; theta + intervals; the marker-consistency flag rate against the simulations'. +8. [ ] **Into `tfs-fit-two-stage`.** A joint mode replacing the relative + one's k shift; docs. + +## Open questions + +- Binding at 2 uM only, or both concentrations through a binding model in + protein concentration (which the thermodynamic-ensemble goal will want)? + Start with 2 uM. +- s alone, or s plus an effective protein concentration? Start with s. +- The binding data are still being replicated; when do they freeze for + step 6? +- Bulk-only binding genotypes (G65H, V95N) carry congression; fit them + through the mixture like any bulk genotype. + +## Out of scope + +- The selection-onset transient (`planning/offset-mode-growth-transition.md`). +- Thermodynamic ensembles (long term; this plan supplies the absolute + theta they need). diff --git a/planning/offset-mode-growth-transition.md b/planning/offset-mode-growth-transition.md new file mode 100644 index 00000000..23da28bb --- /dev/null +++ b/planning/offset-mode-growth-transition.md @@ -0,0 +1,121 @@ +--- +title: The tube-offset mode and the growth transition at the switch to selection +status: idea +filed: 2026-10-05 +area: tfmodel +revisit_when: >- + After pipeline plan steps 6 and 7 (the simulator's raw formats, the + documented chain and the offset diagnostic), as part of the science + prioritization in pipeline step 8 (user, 2026-10-05). Its simulation + steps should use the raw-format path. Keep the step list current while + it is worked. +related: + - planning/studies/staged-map/README.md + - planning/experiment-pipeline.md + - planning/analysis-roadmap.md + - src/tfscreen/tfmodel/inference/staged_map.py + - src/tfscreen/tfmodel/generative/components/growth_transition/ + - src/tfscreen/tfmodel/generative/components/sample_offset/level.py + - src/tfscreen/simulate/growth/transition_linkage.py +--- + +## Why + +The staged MAP (`tfs-fit-model --stage_offsets`, pipeline plan step 3) was +built to keep a level-offset fit out of what we called the offset trap, a +mode where the tube offsets carry the population's growth. Run 2 of +`planning/studies/staged-map/`, with every MAP run to convergence on the +exact full-batch loss, showed that the trap is not an optimizer artifact. +It is the best point found so far of the model's objective. + +- `ref_refit`, the hand-chained reference fit run to convergence, ended + 7.9e4 nats (log joint) above the staged MAP and 4.5e5 above where it + started, and it got there by moving into the offset mode: offsets up to + -4.2, 10-25 prior SDs each at `sigma_fixed` 0.17, with growth k and m + unphysical (kanR+kan k 0.036, m -0.050; about 0.01 elsewhere). +- The offsets are not noise. In both selection conditions they are a smooth + function of IPTG, with opposite signs (kanR selection about +2 at 0 mM to + -3 to -4 at 1 mM; pheS selection -1 to +1.9), nearly the same at all three + time points, and near 0 in the no-selection conditions. +- The staged MAP's offsets are small and patternless (SD 0.21) and its k + uniform (about 0.027). It reaches that point only because it stops in a + local optimum. A longer run, a better optimizer or a multi-start would + leave it. + +An offset that is constant in time and structured by IPTG and selection is +a population-level shift set before the first sampled time point. IPTG is +added at `-t_pre` (user, 2026-10-05): the pre-growth exists so the cells +express their markers before selection starts, so the shift is not +"marker not yet expressed". It points at the selection onset instead: a +transient when the selection agent arrives (a kanamycin lag or die-off of +low-resistance cells, a delay before 4CP takes effect) whose size depends on +the marker level, hence on IPTG. The `instant` growth transition cannot +produce such a transient, so the offsets supply it. This is roadmap step 7b +arriving from the real data. + +Pipeline plan step 3 is closed as machinery: the staged MAP and +exact-loss convergence work, and the staged fit is the physical one. This +plan carries the model problem. Until it is solved, a longer or better fit +can leave the physical basin; pipeline step 7's tube-offset diagnostic +flags a fit that does. + +## Done when + +- One growth-transition model (or `instant` with an explicit reason) makes + the physical mode the best-scoring one on the dev data: a cold fit, the + staged fit and a refit from the offset mode all end at the same optimum, + with tube offsets small and without IPTG structure. +- The same holds on a simulation with a known selection-onset transient, + and `instant` is kept where the simulation has none. +- Roadmap step 7b's purge is done for the models not kept. + +## Steps + +1. [ ] **Where the offset mode's likelihood comes from** (local, existing + runs). Extend `planning/studies/staged-map/score_map.py` to report the + count log-likelihood by tube and by genotype class, and compare + `ref_refit` with `staged_auto2`. If the 9e4-nat gain sits in the + selection tubes, spread across the library and concentrated at the + first sampled time point, the transient explanation stands; if it sits + in a few genotypes or tubes, look there instead. +2. [ ] **Simulation with a known transient.** Simulate a small library with + a selection-onset transient (`two_pop` or `baranyi` in + `simulate/growth/transition_linkage.py`) whose size depends on theta. + Fit `instant` with level offsets: does the IPTG-structured offset mode + appear and score best? Fit the matching transition: does it vanish? A + control simulation with no transient should keep `instant`. +3. [ ] **Dev-data grid.** Transition `instant`, `two_pop` and one more if + step 2 argues for it, each fit three ways: staged, cold, and refit from + `ref_refit`'s point (`--init_from`). Score every end point with + `score_map.py`; report offsets by tube design and k/m. About 9 runs of + about 4 h. Fix `two_pop`'s fallback for `g_pre - g_sel - k_trans <= 0` + first if it is a candidate (roadmap 7b). +4. [ ] **Decide and purge** (roadmap 7b, D12, D13). Keep `instant` and at + most one transition model; remove the rest from the fit, the simulator, + examples, grid `auto` enumeration, tests, docs and CLAUDE.md. +5. [ ] **Confirm the staged MAP under the chosen model.** Rerun the + staged-map comparison: the staged MAP should reach the best-scoring + mode, and cold and refit starts should not beat it. Then compare spike + K and n between converged fits. +6. [ ] **Revisit the growth-only defaults** (moved from pipeline step 2): + the held log(n), X_low, X_delta and log K population SDs and the offset + SD 0.17, under the chosen transition model. + +## Also open + +- **The slow tail.** Every converged MAP stopped on the `loss_rtol` floor + (61 nats per window on the dev data) while still descending about 40 + nats per window at step size 1e-6, after 1-1.2 M steps (4 h). The floor + sets where they stop. If that matters for the decision above, a final + full-batch second-order polish (L-BFGS on `full_batch_loss`) is the + proper fix; decide after step 3 shows whether end points move. +- **The offsets' own freedom.** An i.i.d. N(0, 0.17) prior cannot stop + offsets from absorbing any pattern shared by every genotype in a tube, + because each tube's likelihood outweighs its one prior term by orders of + magnitude. If no transition model removes the pattern, constrain the + offsets structurally instead (no IPTG or time structure) or let the + population model (roadmap step 6) carry the tube totals. + +## Out of scope + +- The orchestrator (pipeline plan step 5), deferred. diff --git a/planning/per-sample-level-offset.md b/planning/per-sample-level-offset.md new file mode 100644 index 00000000..b97bb0b8 --- /dev/null +++ b/planning/per-sample-level-offset.md @@ -0,0 +1,64 @@ +--- +title: Per-sample (tube-level) ln_cfu offset in the growth likelihood +status: idea +filed: 2026-09-24 +area: tfmodel +revisit_when: >- + After censored floor observations are in (step 3.5 of the congression + plan), or sooner if theta coverage stays well below nominal once the + floor bias is fixed. +related: + - planning/analysis-roadmap.md + - planning/congression-physics-plan.md + - planning/count-likelihood.md + - planning/studies/congression-calibration/README.md + - src/tfscreen/tfmodel/generative/observe/growth.py + - src/tfscreen/tfmodel/generative/components/growth_noise/normal_kt.py + - src/tfscreen/tfmodel/generative/components/sample_offset/normal.py + - src/tfscreen/process_raw/counts_to_lncfu.py +--- + +**Context:** Diagnosing the congression calibration grid (2026-09-24). On +well-measured simulated rows (more than 100 reads) the error in `ln_cfu` was +0.38 ln units, almost all of it shared by every genotype in a sample: +per-sample SD 0.374, within-sample 0.054. It is the simulator's per-tube +growth noise (`tube_noise_sigma` x t). Real data add a second shared term: +every genotype's `ln_cfu` in a tube carries the same plating (`sample_ln_cfu`) +error, and the per-tube denominator review found a PCR component too. + +The model treats these as independent per-row noise: `growth_noise` +(`normal_kt`) adds `sigma_k` in quadrature to each row's `ln_cfu_std`, and +`sample_ln_cfu_std` is folded into each row's `ln_cfu_var` in +`counts_to_lncfu`. A shared offset seen as N independent errors gives N-fold +too much information about everything that differs between tubes, and too +little about what is shared. Bulk theta 95% coverage was about 0.65 in every +arm of the grid, `single` included; this is one candidate cause. + +**Idea:** Add one latent level offset per sample, `o_s`, shared by all +genotypes in the tube: `ln_cfu_obs[g, s] ~ ln_cfu_pred[g, s] + o_s + e[g, s]`. +Its prior width comes from the known shared sources (plating SD, tube growth +noise over the elapsed time, a PCR term); the per-row noise keeps only the +counting part. `sigma_k` then shrinks to what is genuinely per-row. + +**Why not now:** The detection-floor bias (censoring) comes first; it is a +concrete bias the mixture exploits. This one is a calibration improvement +whose size in real data is not yet measured. + +**What it would take:** + +- Superseded in design (2026-09-26, `planning/analysis-roadmap.md`, "Reads + and totals" and step 6): tube growth noise cancels from read frequencies, + so it belongs in an OD600 likelihood around a smooth population curve, not + on every genotype. `sample_offset` keeps only a per-tube composition + offset (PCR, calling) with a time-independent prior (roadmap D2). The + "plating" SD below is really the OD-to-CFU error; CFU comes from OD600. +- `counts_to_lncfu` to write the counting variance and the plating variance + separately instead of summing them into `ln_cfu_var`. +- `o_s` is a per-sample latent: mini-batch safe as a global site, like the + condition parameters. +- Interaction with `ln_cfu0` and the k/dk_geno slide: `o_s` at the first + timepoint trades against `ln_cfu0`; the prior has to keep it identified. +- The sim to test it: the calibration grid, comparing theta coverage with and + without the offset. +- It is the `o_s` of `planning/count-likelihood.md`; building it first makes + that project smaller. diff --git a/planning/plasmid-segregation-low-copy.md b/planning/plasmid-segregation-low-copy.md new file mode 100644 index 00000000..63fc668c --- /dev/null +++ b/planning/plasmid-segregation-low-copy.md @@ -0,0 +1,42 @@ +--- +title: Plasmid segregation ("curing") of multiple transformants in low-copy designs +status: idea +filed: 2026-09-13 +area: experiment +revisit_when: >- + When designing or analyzing the next experiment, which moves to a low-copy + plasmid. +related: + - planning/congression-physics-plan.md + - src/tfscreen/simulate/selection_experiment.py +--- + +**Context:** The congression model assumes co-transformed plasmids persist in +a cell for the whole experiment. That is reasonable for the current high-copy +plasmids over roughly 3 to 7 doublings. The next experimental design switches +to a low-copy plasmid. + +**Idea:** Two plasmids with the same origin are incompatible: under no +selection that distinguishes them, a lineage drifts toward carrying only one +type, faster at low copy number. Two consequences: + +1. **Modeling:** congression effects would fade over time, so the congressed + fraction and each cell's variant shares `x_g` become functions of the + number of generations. The observable-level mixture in the congression plan + would need time-dependent class weights. +2. **Design tradeoff:** extra rounds of replication before selection would + "cure" multiple transformants toward single-genotype cells, but outgrowth + also costs library diversity (bottlenecks, drift, growth-rate bias against + slow genotypes). There should be an optimal number of pre-selection + doublings that balances curing against diversity. + +**Why not now:** The current dataset uses high-copy plasmids, so the +persistence assumption holds. + +**What it would take:** +- A segregation model for plasmid copy number n per cell (for example random + partitioning at division), giving the probability a co-transformed lineage is + still mixed after g doublings. +- Simulation: extend `_sim_transform`/`_sim_growth` with per-cell share drift. +- Design calculation: congression remaining vs. library diversity retained, as + a function of pre-selection doublings, for candidate copy numbers. diff --git a/planning/relative-m.md b/planning/relative-m.md new file mode 100644 index 00000000..d71af725 --- /dev/null +++ b/planning/relative-m.md @@ -0,0 +1,144 @@ +--- +title: Take m from relative information only (no tube totals in X) +status: idea +filed: 2026-10-15 +area: tfmodel +revisit_when: >- + Proposed 2026-10-15 for the user's review. If agreed it becomes the + plan for replacing the two-stage fit's k shift + (planning/two-stage-fit.md step 4). +related: + - planning/studies/within-tube-m/README.md + - planning/studies/two-stage-validation/README.md + - planning/studies/x-vs-binding/README.md + - planning/two-stage-fit.md + - planning/offset-mode-growth-transition.md + - src/tfscreen/tfmodel/generative/components/sample_offset/level.py + - src/tfscreen/tfmodel/generative/observe/growth_counts.py + - src/tfscreen/tfmodel/analysis/shared_shift.py +--- + +## Why + +The within-tube study (`planning/studies/within-tube-m/`, runs 1-3) found +that the fit's growth slopes disagree with the part of the data that does +not depend on tube totals: + +- Against flat pure spikes in the same tube, wt's swing from 0 to 1 mM + IPTG is -0.005 per minute in kanR+kan and +0.018 in pheS+4CP. Stage 1 + pins -0.0148 and +0.0070. +- Across 903 deep singles the pheS swing over the kanR swing is -2.2 to + -2.5 (wt -3.9, wide); stage 1's ratio is -0.47. wt sits with the + library, so the problem is the fit's m, not wt. +- Stage 1 does not reproduce the within-tube contrast (predicted 0.004 in + pheS against 0.018 observed); on a simulation it does. + +The fit's absolute scale comes from the OD600-derived tube totals, which +tie each condition's k and m to the population's growth. A conversion +from OD600 to cells that drifts with IPTG or cell state under selection +would bias m, and the monoculture (also OD-based) with it. Even with that +map measured, OD600 is a noisy input. X does not need it. + +## The idea + +The count likelihood is `log mu = ln_sample_reads + ln_cfu_pred - +sample_ln_cfu + offset_tube`. With the per-tube level offset free (a flat +prior), it absorbs each tube's total, and the likelihood keeps only the +relative abundances of genotypes within each tube. Then: + +- **k drops out.** Every tube is one time point, so `k (t_pre + t_sel)` is + common to every genotype in it and the tube's offset absorbs it. k can be + pinned anywhere; X does not depend on it. +- **m is still identified** through the gauge: X_wt goes from 1 at 0 IPTG + to 0 at 1 mM, so wt's contrast with the rest of the library sets m. This + is the within-tube statistic, used by the fit itself. +- **dk_geno stays identified** relative to wt (dk_wt = 0) through the + no-selection tubes. +- **A growth shift shared by every genotype in a tube** (the IPTG term of + run 2) is absorbed by the offsets, so the log-linear titrant k is not + needed. +- **k's uncertainty no longer reaches X**, so the shared k shift and its + refits (which failed, `planning/studies/two-stage-validation/`) go. + m's uncertainty replaces it, and needs no refits: with the data's + contrasts fixed, scaling m by (1 + e) scales every X_g - X_wt by + 1 / (1 + e), so the shared shift per draw is -e (X_g - X_wt) at each + concentration, through the existing `--shared_shift`. +- **The OD600 tables become optional** for X. They still give absolute + growth rates. + +Everything needed for the first test exists: `sample_offset: level` with +`sigma_fixed` large holds the offsets' prior nearly flat, and +`--growth_priors` with `k_pinned` = 1, `m_pinned` = 0 pins k and leaves m +free. + +## What could go wrong + +- **Precision.** m now rests on contrasts against wt. wt's own within-tube + swing has an SE of about 0.002-0.004 per minute (10-40% of m), though the + fit also uses every other genotype's contrast with wt. The jackknife will + say what m's SE is. If it is too wide, binding-anchored spikes could add + scale information later. +- **The offset mode.** Freed offsets went badly before + (`planning/offset-mode-growth-transition.md`: IPTG-structured offsets, + k and m unphysical). There k was free and the full library's doubles + pulled m (the 1.45x bias). Here k is pinned and stage 1 has no doubles, + so the offsets are meant to carry the totals. If m still runs away (the + old fit reached -0.05 in kanR), the relative information is not what we + think, and step 1 shows it. +- **The dk_geno population direction** that stage 2's Laplace held (the + population location against wt) may stay soft. Watch the held + directions. +- **Simulations have consistent totals,** so relative and absolute fits + should agree there; they test that relative-only is unbiased and say + what it costs in width, not that it fixes an OD bias. A simulation with + an IPTG-dependent OD bias would test that directly (optional, step 5). + +## Steps + +1. [x] **Relative stage 1** (no code). Run 1 (2026-10-15, + `planning/studies/relative-m/`): the seeds pass, but on the real data + m does not move (pheS/kanR -0.49 against -0.47 absolute and -2.2 + within tube); the fit discards pheS's IPTG response because the X + shared by both markers cannot follow their different IPTG curves. + Steps 2-6 wait on a per-marker midpoint test. Setup as planned: The two-stage fit's stage 1 with + `sigma_fixed=10` and k pinned at the current stage 1's values, m free, + on the real data, seed 1 and seed 2 (3 runs of about 30 minutes). + Pass: + - seeds: m within about 5% of the truth, as the absolute stage 1 is; + - real: m near the within-tube values (kanR+kan about -0.005, pheS+4CP + about +0.012 to +0.018, ratio about -2.3); + - the within-tube posterior-predictive check (`ppc_within_tube.py`) + reproduces the observed contrast; + - no held direction in the Laplace beyond what stage 1 has now. + Also report the offsets by tube design (they now carry the totals and + any shared IPTG term). +2. [ ] **m's precision.** The stage 1 jackknife (30 subsamples) on the + relative stage 1, real data: m's SE and its direction across the two + markers. +3. [ ] **Relative stage 2** on seed 1, seed 2 and the real data, k and m + pinned at the relative stage 1, offsets free. Compare with run 1 of the + two-stage validation: + - seeds: deep-double X coverage (run 1, absolute: 0.90 / 0.76 without + the k shift); + - real: X against binding at 2 uM for kanR alone and pheS alone (both + should now match if m was the problem), and the marker-consistency + flag rate (10.6% deep in run 1; it should fall toward the + simulations' 1.9%). +4. [ ] **m's uncertainty in X.** The analytic scale shift from step 2's + jackknife through `--shared_shift`; coverage on the seeds. +5. [ ] *(Optional)* **A simulation with an IPTG-dependent OD bias**, if + the simulator's `od600` block can be given one: the absolute fit should + fail and the relative one should not. +6. [ ] **Fold into `tfs-fit-two-stage`** if steps 1-4 pass: a relative + mode (free offsets, k pinned, m jackknife, scale shift) that replaces + the k shift; retire `kshift`; update `planning/two-stage-fit.md`. + +Independent of these, the bench test: OD600 against plate counts under +each selection across IPTG. It would explain the absolute fits' m and +give absolute growth rates, but X would not depend on it. + +## Out of scope + +- The full-library m bias from the doubles (stage 2 keeps m pinned). +- The selection-onset transient (`planning/offset-mode-growth-transition.md`), + unless step 1's offsets point at it. diff --git a/planning/release-0.5.0.md b/planning/release-0.5.0.md new file mode 100644 index 00000000..c5f99820 --- /dev/null +++ b/planning/release-0.5.0.md @@ -0,0 +1,244 @@ +--- +title: Close out the 0.5.0 release (X with honest intervals, runnable by the lab) +status: active +filed: 2026-10-19 +area: release +revisit_when: >- + Decisions settled by the user, 2026-10-19. Keep the checklist current; + 0.5.0 and its 0.5.x bugfix releases are what Natalie's manuscript uses. +related: + - planning/two-stage-fit.md + - planning/deep-coverage.md + - planning/relative-m.md + - planning/findings-ledger.md + - planning/studies/two-stage-validation/README.md + - planning/studies/relative-m/README.md + - CHANGELOG.md +--- + +## Goal + +Someone in the lab other than Mike (Natalie first) takes a screen from +per-tube counts to per-genotype X curves with reasonable 95% intervals, +using one documented command and no study scripts. The release is that +route, validated, documented and tagged. + +**The deliverable is X, not theta.** X is the wt-relative occupancy-like +variable of `hill_relative` (wt X = 1 at the lowest IPTG, 0 at the +highest). Comparing X with binding happens after the fit and is not part +of the release. + +## Where things stand (2026-10-19) + +- **The route.** `tfs-fit-two-stage` (`planning/two-stage-fit.md`). + Stage 1 fits wt, the spikes and the singles with the congression + mixture (`max` rule). Stage 2 fits the full library with every + condition's k and m pinned. A jackknife of stage 1 gives k's error, and + a marker-consistency flag runs at the end. Validation run 1 reproduced + the study's stage 1, jackknife, stage 2 and flag + (`planning/studies/two-stage-validation/`). +- **Coverage.** On the simulations, deep doubles' 95% X coverage without + the k shift is 0.90 / 0.76 (seed 1, 1e4-1e5 / >1e5 reads) and + 0.91 / 0.81 (seed 2). The study's hand-built shift reached 0.94 / 0.91. +- **The k-shift step does not work as built.** Its refits leave stage 2's + MAP and slide along the dk_geno population against wt (real X +0.32, + the same with k unchanged). So the command's shift measures where the + optimizer stopped. It got 0.915 / 0.858. +- **Congression in X** needs the `max` rule. The partition-function rules + cannot be used with X, and stage 2 is `single` anyway. +- **The kanR warp** (`m_warp`) is real on the real stage 1: about 8,000 + nats for two parameters, and nearly linear on the simulation. It closes + little of the pheS gap. A pheS warp collapsed to a step. Warping both + markers was not tested. The warp is not carried through + `tfs-fit-two-stage`: its pin table holds only k and m. +- **Monoculture rates.** The only monoculture data are wt's k and m + (`2025-10-02_control-experiments.xlsx`). As k/m priors the data + rejected them (`planning/deep-coverage.md` step 1). They are now a + check only: the real stage 1's kanR is within 4% of them, and pheS's m + is half. No mutant monoculture dk_geno data are in the dev data. The + `base_growth_df` input exists and the two-stage YAML accepts it, but it + has never been run with `hill_relative`. +- **Docs.** `docs/source/fitting.rst` (last edited 2026-10-05) describes a + single `hill_relative` fit. The two-stage route appears only in + `cli.rst`. The quickstart is about simulations. +- **Tests.** The full suite passed on 2026-10-18 (6,109 passed). + `__version__` already reads 0.5.0. The last tag is v0.4.4. + +## Decisions (user, 2026-10-19) + +1. [x] **k's uncertainty in X.** Repair the shift first: start the two + refits at the final step size with the dk_geno population location + held at stage 1's (ledger open item 4), then one validation round. If + that fails, ship without the shift and report the no-shift coverage + (0.90 / 0.76-0.81). Replacing it with k and m free under the jackknife + covariance is out of scope. + Outcome (2026-10-20): the repair failed validation round 2 (the shift a + third or less of the true one on simulations; + `planning/studies/two-stage-validation/`, run 2), so the release ships + without it (user: 0.76-0.90 coverage is fine for now). + `two_stage.kshift` defaults to false and skips the jackknife too. +2. [x] **The kanR warp is out of 0.5.0.** Its gain (about 8,000 nats on + the real stage 1) is recorded for 0.6. Stage 2's X stays validated + without it. +3. [x] **Drop pheS+4CP replicate 2, t_sel 82.** Data: the IPTG tubes look + reversed with the 0 tube in place (the tube labeled 0.1 uM is a 1 mM + tube, the one labeled 1 uM a 30 uM tube), but the 10 uM slot does not + fit and the low tubes' order cannot be recovered + (`planning/studies/relative-m/`, run 7). The lab found no mechanism + for a three-way swap, and Natalie rules out a flip of the IPTG tubes + with the 0 tube in place. The row goes in a corrections file as an + exclusion, with that evidence. +4. [x] **The lambda prior** stays at the measured value (0.38 +- 0.15). + The docs say it is required and where it comes from. +5. [x] **Usable genotypes.** The docs state one default read-depth + threshold and one policy for the consistency flag. Set both from + validation round 2's coverage by depth and the flag's no-signal rate. + Set 2026-10-20: 1,000 reads (seed 1: slope 0.88 and RMSE 0.17 at + 1e3-1e4 reads, against 0.39 and 0.54 below); flagged genotypes are + reported separately, not dropped. +6. [x] **kanR's m scale** (`planning/studies/relative-m/`, run 6: the + absolute m against the within-tube m, about 2.5x apart). The two-stage + fit takes m from the tube totals, which binding at 2 uM supports and + the weak binders' flat kanR response does not. Decided: ship the + tube-totals m and state the range as a limitation in the + interpretation section. + +## Code + +6. [x] Implement decision 1 in `tfs-fit-two-stage` and its tests. + Done 2026-10-19: `kshift_step_size` (1e-5), the dk_geno population + held at stage 2's MAP (not stage 1's: holding stage 1's would change + the validated stage 2), and a k-unchanged control refit with + `kshift/drift.csv`. Validation round 2 (item 12) is set up in + `planning/studies/two-stage-validation/`, run 2. +7. [x] The dev data's corrections file: exclude pheS+4CP replicate 2, + t_sel 82 (decision 3). Document + `tfs-correct-tubes` -> `tfs-process-counts` as the route, since the + two-stage fit reads the processed growth table. + Done 2026-10-19: `planning/dev-data/tube_corrections.yaml` (gitignored, + C10) excludes the row's six pheS+4CP tubes (the pheS-4CP control tubes + of that replicate and time stay), checked with `tfs-correct-tubes` (118 + tubes to 112). `planning/studies/real-data-fit/fit/run_recipe.srun` runs + `tfs-correct-tubes` before `tfs-process-counts` (`CORRECTIONS`, default + `../../tube_corrections.yaml`). The recipe page (item 15) documents the + route. +8. [x] Refuse or warn on the experimental joint-fit pieces in a two-stage + config: `theta_growth_log_conc_shift_scale`, `genotype_offset` and a + binding table. Mark them experimental in their docstrings. + Done 2026-10-19: `read_two_stage_config` refuses `binding_df`, theta + noise other than `zero`, the joint-fit and warp priors in `set_priors`, + and an `m_warp` column with any 1 in `growth_priors` (stage 1 would fit + the warp and stage 2's pins drop it). `genotype_offset`, hill_geno's + titrant shift and linear's `m_warp` say they are experimental. +9. [x] Make the outputs easy to read: one top-level table per genotype + and concentration with X quantiles, read depth, `in_training_data` and + the consistency flag, so users never open the step directories. + Done 2026-10-19: the `summary` step writes `{out_dir}/tfs_x.csv` + (also `k_shift`, `num_mutations` and reads per library). On run 1's + real outputs: 217,945 genotypes, 1.7 M rows, about 40 s to count the + reads. A depth flag waits on decision 5's threshold. +10. [x] Turn epistasis on X on in the recipe (`predict_epistasis`), or + document how to run `tfs-predict-epistasis --shared_shift` afterward. + Natalie's paper needs it. + Done: the screen page's YAML sets `predict_epistasis: true` and says + when to (the command's default stays false). With the k shift off + there is no `--shared_shift` to document. +11. [x] Pin or floor the dependency versions that broke CI (pandas 3). + Check a fresh install on the cluster. + Done locally 2026-10-20 (`planning/studies/fresh-install/`): floors in + `pyproject.toml` at the maintainer's passing environment (no caps; CI + tests the newest) and a `cuda12` extra for GPU jax on Linux; a fresh + venv on the newest releases (pandas 3.0.6, numpy 2.5.4, jax 0.12.0) + passed the full suite. Install docs (README, quickstart, screen page) + updated. The cluster install works (user, by hand, 2026-10-20). No + constraints file in the repo (user): the manuscript records its own + versions (item 23). CI's 3.11 jobs then failed (jax 0.11 has no 3.11 + wheel), so tfscreen needs Python 3.12+ and CI tests 3.12-3.14. + +## Validation + +12. [x] Validation round 2 with decision 1 in: seed 1, seed 2 and the + real data through the command. Done 2026-10-20: the repair failed + (shift too small on the simulations), so decision 1's fallback + applies; seed 1's coverage by depth without the shift is recorded in + `planning/studies/two-stage-validation/`, run 2. +13. [x] A dry run by Natalie on the dev data from a clean install, + following only the docs. Every question she asks becomes a docs fix. + Done 2026-10-22 to 2026-10-24: the user and Natalie both ran it. Docs + fixes: run processing on a compute node, `consistency_library` is a + convention, run the submit script or the command (not both), and the + growth-table section rewritten as numbered steps (the inputs list no + longer implies a corrections file already exists). +14. [x] `two-stage-fit.md` step 4 marked done, and the plan's status set. + Done 2026-10-24: steps 4 and 8 done; the plan stays active for steps + 5 and 7 (the design-sweep tools, after the release). + +## Docs + +15. [x] A real-data recipe page: counts, `tfs-process-counts`, + `tfs-check-tubes` and `tfs-correct-tubes`, `tfs-configure-model`'s + inputs, `tfs-fit-two-stage`, the outputs. Include cost and wall time, + and an example SLURM script with the jackknife as an array. +16. [x] An interpretation section: + - what X is, and why it is not binding (the affine map; the M42I + case); + - what the intervals include: k's shift, if decision 1 lands; + - what they leave out: stage 1's 2-6% m bias, the pheS gap, + congression error per genotype, and kanR's m scale (decision 6); + - that shallow genotypes are prior-dominated; + - what the consistency flag means. +17. [x] `fitting.rst`: make the two-stage route the default for + growth-only screens. Mark the single-MAP `hill_relative` fit, the + joint absolute fit and the thermodynamic components as advanced or + experimental. +18. [x] `README.rst` and `quickstart.rst`: point to the recipe page. + Drafted 2026-10-19: `docs/source/screen-analysis.rst` ("Analyzing a + screen") covers 15 and 16, with `examples/tfmodel/two_stage_submit.sh`, + `two_stage_step.srun` and `growth_priors_loose.csv`; `fitting.rst` + points to it and marks the joint model experimental (17); the index, + quickstart and README point to it (18). Updated 2026-10-20 for the + fallback: no k shift, the 1,000-read threshold, the coverage-by-depth + table, and k's uncertainty listed among what the intervals leave out. + +19. [x] Regenerate `cli.rst`, and update CLAUDE.md. + Done 2026-10-19: `cli.rst` was current (its test passes). CLAUDE.md: + the relative-X section names the two-stage fit as the route (it said + the choice was open), and the planning section leads with this plan. + `pipeline.rst` points to the screen page and to `tfs-check-tubes` / + `tfs-correct-tubes` (it sent tube cleanup to the lab's own script). + +## Release mechanics + +20. [x] CHANGELOG: fold `[Unreleased]` into the existing `[0.5.0]` + section and change its date (2026-10-03) to the release date. 0.5.0 + was never tagged or shared in the lab (user, 2026-10-19): the bump + came before the two-stage work, so the release stays 0.5.0 and + `__version__` is already right. Add an upgrade note: the removed + commands and the configs that no longer load. + Done 2026-10-24: `[0.5.0] - 2026-10-24` opens with a new summary, + then the entries since 2026-10-03, then "Changes up to 2026-10-03" + (the earlier summary and entries); `[Unreleased]` is empty. The + upgrade notes are in the release notes draft + (`.claude/2026-10-24_release-notes-v0.5.0.md`). If the tag date + differs, change the section's date. +21. [x] Full test suite, with `--runslow` and the smoke tests. Run lint. + Done 2026-10-24: 6,145 passed with `--runslow` (fresh venv, newest + releases, `-n 6`); smoke tests 45 passed; fatal lint clean. +22. [ ] Tag v0.5.0 and write GitHub release notes. Install that tag on + the cluster. +23. [ ] Record the commit and version Natalie's analysis uses in her + manuscript folder, with the environment's `pip freeze`. + +## Out of scope (after 0.5.0) + +- The official analysis for the paper, including X against binding + (`planning/studies/x-vs-binding/`) and the M42I re-prep. +- Thermodynamic models with a floating protein concentration (NSF). +- Experimental design: `tfs-build-sim-config` and the scoring CLI + (`two-stage-fit.md` steps 5 and 7), and + `planning/design-noise-sweep.md`. + - The kanR warp (decision 2), the relative-m plan, + the k/m-under-jackknife-prior alternative (decision 1), the joint fit + (`planning/joint-fit-return.md`, paused), the X population SDs held for + shallow genotypes (`deep-coverage.md` step 4), and mutant monoculture + dk_geno data as `base_growth_df`. diff --git a/planning/studies/congression-calibration-convergence/README.md b/planning/studies/congression-calibration-convergence/README.md new file mode 100644 index 00000000..6a25b8d7 --- /dev/null +++ b/planning/studies/congression-calibration-convergence/README.md @@ -0,0 +1,287 @@ +# Congression calibration: SVI convergence review + +**Question.** Four rounds of diagnosis on the step 3.5 calibration grid +(detection floor, binding weight, unanchored k/dk_geno slide, ELBO/Jensen) +each explained the mixture fits' endpoints, and none of the fixes changed +them. Are those endpoints converged fits at all? If they are not, what does +the mixture actually do during training? + +**Decision it feeds.** What to do next in step 3.5 of +[`planning/congression-physics-plan.md`](../../congression-physics-plan.md): +whether to keep diagnosing the mixture model, or fix the optimization first. +The routes forward below are proposals, not decisions. + +## Summary + +- **None of the 3.5 fits converged**, `single` included. The stop rule + measures loss change relative to the loss at the start of SVI, which is + ~500 times the final loss, so runs stop while the loss is still falling + 20 to 30% per 1000 epochs and the guide scales are still shrinking. +- **The mixture's "k drift" is an early excursion that was still unwinding + at the stop.** In the high-variance start of SVI (guide scales at prior + width, nu collapsing to ~1.3), the mixture's `growth_k` overshoots the + truth by +0.12 at lambda 0 and +0.05 at lambda 1, then comes back + slowly. `single`, on the same data, holds `growth_k` at the truth + throughout. +- **The simulated library is dominated by wt**, which is 39% of the + co-resident pool, so most congressed classes are "genotype + wt". wt is + also the only genotype with a pinned dk_geno. +- Nothing here points at the congression physics or the mixture's design. + The earlier diagnoses should be re-judged on converged fits. + +## How to run + +The scripts read a grid pulled from the cluster (`fetch.sh` in +`planning/studies/congression-calibration/`), by default the anchored grid +(`grid_anchor.yaml`) inside that study directory. From this directory, with +tfscreen importable: + +```bash +python trace_checkpoints.py # runs 0001, 0002, 0005 +python coresident_pool.py # run 0002 +GRID=/path/to/grid python trace_checkpoints.py run_0002 run_0003 +``` + +Both print to stdout and write nothing. `trace_checkpoints.py` takes about +20 s on a laptop CPU. + +## Inputs + +- Anchored grid (`planning/studies/congression-calibration/grid_anchor.yaml`, + set up on c7b228d, run 2026-09-24): binding weight 1, prefit + `--k_scale_ceiling 0.005`, lambda 0 and 1.0, wide dk_geno spread, seed 1. + Run 0001 is `single`, 0002 the mixture with the matched lambda prior at + simulated lambda 0 (fitted lambda ~0.004), 0005 the mixture with the matched + prior at lambda 1.0. Runs 0001 and 0002 were fit to the same simulated data. +- Each run's `checkpoints/*_checkpoint.pkl` (every 1000 epochs), + `tfs_fit_model_checkpoint.pkl`, `tfs_fit_model_losses.txt`, + `tfs_sim_growth_parameters.csv`, `tfs_configure_library.csv`, + `tfs_sim_library.csv` and `tfs_sim_presplit.csv`. + +## Commit + +Scripts written and run on 2d86049 (2026-09-25). + +## Results + +### 1. The stop rule fires on the start-up transient + +`RunInference._update_loss_deque` +(`src/tfscreen/tfmodel/inference/run_inference.py`) defines the metric as + +``` +|mean(old window) - mean(new window)| / (loss_start - loss_best) +``` + +with `loss_start` taken once, when the first two 10-epoch windows of SVI +fill. At that point the component guide's per-genotype scales are still at +their initial 1.0 and the initial values carry 0.1 jitter, so `loss_start` is +~8e7. The runs end near 1.5e5 to 2.3e5: + +| run | `loss_start` | `loss_best` | stop threshold on the 10-epoch change (tolerance 1e-4) | +|---|---|---|---| +| 0001 `single` | 7.64e7 | 2.29e5 | 7,620 | +| 0002 mixture, lambda 0 | 8.93e7 | 1.68e5 | 8,910 | +| 0005 mixture, lambda 1 | 8.91e7 | 1.34e5 | 8,900 | + +The threshold is effectively "the loss has fallen by 99.99% of its initial +value," not "the loss has stopped changing." At the stop the smoothed loss is +still falling 20 to 30% per 1000 epochs (run 0002: 3.6e5 at epoch 7000, +2.5e5 at 8000, 1.9e5 at 9000). + +The original 54-run grid (default binding weight) ended at losses of 9e6 to +1.2e7, even further from converged. + +### 2. The pre-MAP warm-up is close to a no-op + +`_run_map` (`src/tfscreen/tfmodel/scripts/fit_model_cli.py`) sets the +learning-rate schedule to decay from `adam_step_size` to +`adam_final_step_size` (1e-3 to 1e-6) over `max_num_epochs`, and the pre-MAP +passes `max_num_epochs=pre_map_num_epoch` (1000). The step size therefore +collapses within the warm-up. Adam moves a parameter by roughly the step +size per step, so about 0.15 in total over the warm-up. The pre-MAP loss +goes from 6.5e7 to 5.3e7, against ~1.5e5 at the end of SVI. SVI starts far +from the mode, with prior-width guide scales (`dk_geno_offset_scales` and the +other per-genotype scales initialize to 1.0). + +### 3. Parameter trajectories + +`trace_checkpoints.py`. `growth_k` for kanR+kan (truth 0.0107) in the three +runs; guide scale, nu, sigma_k and loss from run 0002 (runs 0001 and 0005 +follow the same pattern). nu and sigma_k are guide medians; "dk scale" is +the median per-genotype `dk_geno_offset_scales`. + +| epoch | `single` | mixture, λ 0 | mixture, λ 1 | dk scale | nu | sigma_k | loss | +|---|---|---|---|---|---|---|---| +| 1000 | 0.007 | 0.012 | -0.003 | 1.00 | 19.6 | 0.04 | 6.0e7 | +| 2000 | 0.003 | 0.096 | 0.051 | 0.93 | 7.5 | 0.10 | 1.9e7 | +| 3000 | 0.013 | 0.125 | 0.057 | 0.85 | 3.0 | 0.25 | 5.0e6 | +| 4000 | 0.016 | 0.135 | 0.046 | 0.78 | 1.3 | 0.57 | 2.0e6 | +| 5000 | 0.011 | 0.128 | 0.042 | 0.65 | 1.3 | 1.08 | 9.7e5 | +| 6000 | 0.010 | 0.128 | 0.039 | 0.50 | 2.1 | 1.15 | 5.5e5 | +| 7000 | 0.011 | 0.118 | 0.039 | 0.36 | 3.5 | 1.06 | 3.6e5 | +| 8000 | | 0.107 | 0.039 | 0.26 | 5.2 | 0.91 | 2.5e5 | +| 9000 | | 0.094 | 0.034 | 0.20 | 6.4 | 0.77 | 1.9e5 | +| stop | 0.009 (7982) | 0.092 (9234) | 0.030 (9876) | 0.18 | 6.6 | 0.74 | 1.8e5 | + +The other three conditions move the same way (run 0002 at the stop: 0.090, +0.109, 0.115 against truths 0.015, 0.021, 0.029), and `dk_geno`'s hyper-shift +mirrors them (0.024 at epoch 1000, -0.169 at 4000, -0.079 at the stop). +Lambda in run 0005 also overshoots and returns: 1.02, 1.27 (epoch 2000), 0.95 +(8000), 0.98 at the stop. + +What the table shows: + +- Between epochs 1000 and 5000 the fit escapes the high guide variance by + inflating the noise: nu falls from ~20 to ~1.3 (Cauchy-like tails) and + sigma_k rises from 0.04 to ~1.1 ln units. `single` does the same (nu 1.1 at + epoch 5000). +- In that phase nothing holds the k/dk_geno slide. Under near-Cauchy tails, + wt's pinned dk_geno costs only logarithmically to abandon, and the prefit's + tight k prior (scale 0.005; ~1,400 log units at the run 0002 peak) is small + next to a loss of 1e6 to 1e7. +- `single` wanders by about ±0.01 and ends at the truth. The mixture's + `growth_k` jumps within the first 1000 epochs of SVI and peaks near epoch + 4000. It then returns at about 0.01 per 1000 epochs as the guide scales + shrink and nu recovers, and was still falling when the run stopped. + +This also accounts for the earlier findings from the endpoints. Under the +mixture's own joint density, `single`'s solution beat the mixture's endpoint +by ~18,000, and the mean gradient at that endpoint pointed k back down +(`diag_basin.py`, `diag_gradient.py`). That is what a fit stopped +mid-recovery looks like, not a real mode or an overshoot. + +### 4. Why the mixture's excursion is larger (hypothesis, untested) + +In the high-variance phase, a guide draw often puts a genotype's dk_geno deep +in the lognormal's negative tail, so its clean class dies in that draw. Its +congressed classes rescue the draw, because dilution halves the focal dk. +Most congressed classes contain wt (result 5), whose dk_geno is pinned at 0. +Sliding k up by C and every other dk_geno down by C leaves clean classes +unchanged but speeds up every wt-containing class by C/2, which makes the +rescue more effective and improves the expected log-likelihood under the +guide. + +The excursion sizes fit this: rescue classes weighted ~0.2% (lambda 0.004) +need a larger boost than classes weighted 42% (lambda 1), and the excursion +is +0.12 against +0.05. This is the Jensen/ELBO idea in a concrete form. It +acts in the start-up phase, when guide scales are 0.8 to 1.0. `diag_elbo.py` +evaluates at the endpoints, where the scales are 0.15 to 0.26 and the effect is +much weaker. + +### 5. The simulated library is dominated by wt + +`coresident_pool.py`: + +| quantity | value | +|---|---| +| wt share of the co-resident pool | 0.392 (next largest: 0.023) | +| wt slots per genotype, `congression_sets` [12, 3, 1] | 8.2 of 21 | +| wt entries in `single-1` / `single-2` / `double-1-2` | 30 of 46 / 36 of 67 / 1080 of 3082 | +| wt share of presplit cells (true ln_cfu0), replicate 1 / 2 | 0.61 / 0.22 | + +`LibraryManager._prepare_indiv_lib_blocks` +(`src/tfscreen/genetics/library_manager.py`) treats every codon of a tile as +a site, and a non-degenerate codon's only "mutant" is its wt codon. The study +config's `degen_sites` has one degenerate codon in tile 1's 31 codons and two +in tile 2's 37, so each single sub-library gets one wt entry per +non-degenerate codon. For example, `single-1` has 30 wt entries plus the 16 +NNT codons at site 42. The simulator and the fit share the enumerator, so +this is not a sim/fit mismatch. It does make this simulation unrepresentative +of a library whose tiles are fully degenerate. + +**Resolved 2026-09-25 (bff2c51):** it was a bug. A site is now a codon whose +designed sequence differs from wt. Fully degenerate designs (the real +library, the examples) enumerate as before; this study config goes from +46/67/3082 sequences (30/36/1080 wt) to 16/32/512 (0/1/0 wt), and wt's +co-resident share from 0.39 to 0.003. Every grid in the calibration study +was simulated and fit under the old enumeration (consistently on both +sides); new grids from this config simulate a different library. + +### 6. Status of the earlier diagnoses + +| diagnosis (study README) | status | +|---|---| +| Detection floor | The floor bias itself (0 reads: +1.26 ln) is a data-side fact and stands. The "congressed classes are worth 36,700" measurement was taken at an unconverged endpoint. | +| Binding weight | Weight 251 times a binding SD of 0.001 dominating the objective is a property of the objective and stands. Its role in the mixture's drift is undetermined; the weight-1 fits drifted the same way, as the transient predicts. | +| Unanchored slide and Student-t | Partly right: the slide is loose during the transient because nu collapses. "The mixture abandons wt by 13.6 ln units" is a mid-recovery snapshot. | +| ELBO/Jensen (`diag_elbo.py`) | Plausible as the driver of the early excursion (result 4), but testing it at the endpoints measures a much weaker effect. | +| Grid-level numbers (lambda pulled up, bulk theta coverage 0.65, mixture theta RMSE above `single`'s, `single`'s -0.02 k drift) | All from unconverged fits, `single`'s baseline included. None are yet properties of the models. | + +## Is the congression architecture at fault? + +Probably not: + +- The forward model is verified: lambda -> 0 reproduces `single` to 1e-6. +- The estimator is accurate at the true parameters (step 3.0, + `planning/studies/congression-estimator/`). +- The matched-prior lambda 1 fit recovered lambda (0.98). + +Two structural points to keep in view: + +- **Growth data barely inform lambda in the measured regime.** Congression + there is ~0.05 ln RMS, and `single`'s errors did not change with the + simulated lambda. The plan already sets such parameters from biology. +- **A mixture of exponentials is dominated at late times by its fastest + class.** That mimics the detection-floor bias in dying genotypes and makes + a genotype's likelihood bimodal (clean-dominated vs congressed-dominated). + Inference is most fragile there. Look at it once fits converge. + +## Routes forward (proposals) + +In rough order of cost: + +1. **Resume the anchored runs to real convergence.** No new simulations. A + resumed fit re-bases the stop rule: `run_optimization` resets + `loss_start`, so the denominator becomes the resumed run's own + improvement. From a *copy* of a run directory (the resume overwrites + `tfs_fit_model_checkpoint.pkl` and `tfs_fit_model_losses.txt`): + + ```bash + tfs-fit-model tfs_configure_config.yaml --seed 1 \ + --checkpoint_file tfs_fit_model_checkpoint.pkl \ + --convergence_tolerance 0.0001 + ``` + + Keep `--max_num_epochs` at its default. It also sets the step-size decay + horizon, and the optimizer's step count carries over from the checkpoint, + so shortening it drops the step size abruptly. Against the resumed run's + smaller improvement, the loss noise will probably keep the stop rule from + firing, so expect the full 100,000 epochs: about 20 min on the cluster + GPU (these runs took ~12 s per 1000 epochs). Then run + `trace_checkpoints.py` on the copy. This settles "unconverged" vs "a real + pathology" for runs 0001, 0002 and 0005. +2. **Fix the start and the stop** (code changes): + - run the pre-MAP to convergence, with a step-size schedule that does not + collapse within the warm-up; + - start the component guide's per-genotype scales small (for example 0.01 + to 0.1, like AutoNormal's `init_scale`) rather than at prior width; + - measure convergence against the current loss or parameter movement, + not the start-up transient. +3. **Profile lambda with MAP.** Fix lambda on a grid, optimize everything + else to convergence, on the lambda 0 and lambda 1 data. This answers "does + the data pull lambda up?" without SVI. +4. **Test self-consistency before realism.** Fit data generated by the + numpyro model itself (fixed known parameters), `single` vs mixture at + lambda 0. This separates inference from sim/fit mismatch. Then add the + realism back one piece at a time: per-tube noise, pseudocount floor, + binding. +5. **Consider pinning lambda** as a constant by default, with lambda as a + latent an option. This follows the plan's rule for parameters that only a + lambda series could identify. +6. **Re-judge the earlier diagnoses on converged fits** (result 6). +7. **Library.** Decide whether the wt enumeration is intended, and use a + simulation library with a realistic wt share. Check the real library's wt + share of the co-resident pool. + +`grid_controls.yaml` (2d86049) is worth holding until item 2 is done: + +- Its MAP arms use the same stop rule (MAP `loss_start` ~6.5e7) and will + likely stop early too. +- Its `fixed0` arm (lambda ~1e-6) still has congressed classes, at weight + ~5e-7, which a large enough early excursion could use. Drift there would + not by itself mean a bug in the mixture code. + +**Practice.** Before interpreting any fit, run `trace_checkpoints.py` (or +equivalent) on its `checkpoints/` and confirm that the key parameters have +stopped moving. diff --git a/planning/studies/congression-calibration-convergence/coresident_pool.py b/planning/studies/congression-calibration-convergence/coresident_pool.py new file mode 100644 index 00000000..c975a2e4 --- /dev/null +++ b/planning/studies/congression-calibration-convergence/coresident_pool.py @@ -0,0 +1,73 @@ +""" +Composition of the co-resident pool the mixture draws congressed classes +from, and wt's share of simulated cells. + +The pool is the bulk share (pool_fraction * bulk_fraction) of genotypes with +growth data, __unknown__ excluded, as in +ModelOrchestrator._coresident_pool. Also reports how many wt entries each +sub-library enumerates (tfs_sim_library.csv) and wt's share of presplit +cells per replicate. + +Run from this directory. GRID names a pulled grid directory (default: the +anchored grid inside the congression-calibration study); the argument is a +run-number prefix (default: run_0002). The pool depends only on the library, +so any run of the grid gives the same answer. +""" + +import glob +import os +import sys + +import numpy as np +import pandas as pd +import yaml + +HERE = os.path.dirname(os.path.abspath(__file__)) +GRID = os.environ.get("GRID", os.path.join( + HERE, "..", "congression-calibration", "congression_calibration_anchor")) +RUN = sys.argv[1] if len(sys.argv) > 1 else "run_0002" + +run_dir = glob.glob(os.path.join(GRID, RUN + "_*"))[0] + + +def path(name): + return os.path.join(run_dir, name) + + +lib = pd.read_csv(path("tfs_configure_library.csv")) +lib["genotype"] = lib.genotype.astype(str) +grown = set(pd.read_csv(path("tfs_sim_growth.csv"), + usecols=["genotype"]).genotype.astype(str)) +lib = lib[lib.genotype.isin(grown) & (lib.genotype != "__unknown__")].copy() +lib["share"] = lib.pool_fraction * lib.bulk_fraction +lib["share"] /= lib.share.sum() + +print(f"=== {os.path.basename(run_dir)}") +print("Co-resident pool, largest shares:") +print(lib.sort_values("share", ascending=False).head(6)[ + ["genotype", "pool_fraction", "bulk_fraction", "share"]].to_string(index=False)) + +sets = yaml.safe_load(open(path("tfs_configure_config.yaml"))) +sets = sets["components"]["congression_sets"] +slots = sum(num_sets * (n + 1) for n, num_sets in enumerate(sets)) +wt = lib.set_index("genotype").share["wt"] +print(f"\nwt share {wt:.3f}: {slots * wt:.1f} of {slots} co-resident slots per " + f"genotype are wt (congression_sets {sets})") + +sim_lib = pd.read_csv(path("tfs_sim_library.csv")) +sim_lib["genotype"] = sim_lib.genotype.astype(str) +totals = sim_lib.groupby("library_origin").degeneracy.sum() +wt_rows = sim_lib[sim_lib.genotype == "wt"].set_index("library_origin") +print("\nwt entries per sub-library (enumerated codon combinations):") +for origin, row in wt_rows.iterrows(): + print(f" {origin:12s} {row.degeneracy:6d} of {totals[origin]:6d} " + f"({row.degeneracy / totals[origin]:.2f})") + +presplit = pd.read_csv(path("tfs_sim_presplit.csv")) +presplit["genotype"] = presplit.genotype.astype(str) +presplit["cfu"] = np.exp(presplit.ln_cfu_0_true) +keys = ["replicate", "condition_pre"] +share = (presplit[presplit.genotype == "wt"].groupby(keys).cfu.sum() + / presplit.groupby(keys).cfu.sum()) +print("\nwt share of presplit cells (true ln_cfu0):") +print(share.round(3).to_string()) diff --git a/planning/studies/congression-calibration-convergence/trace_checkpoints.py b/planning/studies/congression-calibration-convergence/trace_checkpoints.py new file mode 100644 index 00000000..e4a89f7a --- /dev/null +++ b/planning/studies/congression-calibration-convergence/trace_checkpoints.py @@ -0,0 +1,114 @@ +""" +Trace a fit's guide parameters across its epoch checkpoints, to see whether +they had stopped moving when SVI declared convergence. + +For each run, one row per checkpoint (checkpoints/*_checkpoint.pkl every 1000 +epochs, then the final tfs_fit_model_checkpoint.pkl): the per-condition +growth_k guide loc, dk_geno's hyper-shift loc and median per-genotype offset +guide scale, the guide medians of nu, sigma_k and lambda, and the smoothed +loss. A truth row comes from tfs_sim_growth_parameters.csv. The stop line +gives the convergence bookkeeping stored in the final checkpoint and the +10-epoch loss change below which a tolerance of 1e-4 (run.srun) stops. + +Run from this directory. GRID names a pulled grid directory (default: the +anchored grid inside the congression-calibration study); the arguments are +run-number prefixes (default: run_0001 run_0002 run_0005). The default runs +take about 20 s on a laptop CPU. +""" + +import glob +import os +import sys +import warnings + +warnings.filterwarnings("ignore") + +import dill +import numpy as np +import pandas as pd + +from tfscreen.tfmodel.configuration_io import read_configuration +from tfscreen.tfmodel.inference.run_inference import RunInference + +HERE = os.path.dirname(os.path.abspath(__file__)) +GRID = os.environ.get("GRID", os.path.join( + HERE, "..", "congression-calibration", "congression_calibration_anchor")) +RUNS = sys.argv[1:] or ["run_0001", "run_0002", "run_0005"] +TOLERANCE = 1e-4 + + +def smoothed_loss(run_dir): + losses = pd.read_csv(os.path.join(run_dir, "tfs_fit_model_losses.txt"), + header=None, names=["epoch", "loss", "change"]) + losses = losses[pd.to_numeric(losses.epoch, errors="coerce").notna()] + losses = losses.astype(float).set_index("epoch").loss + return losses.rolling(50, center=True, min_periods=1).median() + + +def condition_order(run_dir): + priors = pd.read_csv(os.path.join(run_dir, "tfs_configure_priors.csv")) + k = priors[priors.parameter == "growth.condition_growth.k_loc"] + return list(k.sort_values("flat_index").condition_rep) + + +def guide_median(params, name): + """Median of a LogNormal guide site with loc parameter ``name``.""" + if name not in params: + return np.nan + return float(np.exp(np.ravel(np.asarray(params[name]))[0])) + + +def trace(run_dir): + cwd = os.getcwd() + os.chdir(run_dir) + try: + orch, init_params = read_configuration("tfs_configure_config.yaml") + ri = RunInference(orch, 0) + files = sorted(glob.glob("checkpoints/*_checkpoint.pkl")) + files.append("tfs_fit_model_checkpoint.pkl") + loss = smoothed_loss(".") + conditions = condition_order(".") + + rows = [] + for f in files: + svi, state = ri.restore_svi_from_checkpoint( + f, init_params=init_params) + p = svi.get_params(state) + with open(f, "rb") as handle: + stored = dill.load(handle) + epoch = stored["current_step"] // ri._iterations_per_epoch + row = {"epoch": epoch} + k = np.ravel(np.asarray(p["condition_growth_k_locs"])) + row.update({f"k[{c}]": v for c, v in zip(conditions, k)}) + row["dk_shift"] = float(np.ravel(p["dk_geno_hyper_shift_loc"])[0]) + row["dk_scale_med"] = float(np.median(p["dk_geno_offset_scales"])) + row["nu"] = guide_median(p, "growth_nu_loc") + row["sigma_k"] = guide_median(p, "growth_noise_sigma_k_loc") + row["lam"] = guide_median(p, "transformation_lam_loc") + row["loss"] = float(loss.iloc[np.argmin(np.abs(loss.index - epoch))]) + rows.append(row) + + truth = pd.read_csv("tfs_sim_growth_parameters.csv") + truth = truth.set_index("condition_rep").growth_k + truth_row = {"epoch": "truth"} + truth_row.update({f"k[{c}]": truth[c] for c in conditions}) + rows.append(truth_row) + + denom = stored["loss_start"] - stored["loss_best"] + stop = (f"stop: loss_start {stored['loss_start']:.3g}, loss_best " + f"{stored['loss_best']:.3g}; tolerance {TOLERANCE:g} stops once " + f"the 10-epoch change in mean loss is below " + f"{TOLERANCE * denom:,.0f}") + finally: + os.chdir(cwd) + return pd.DataFrame(rows), stop + + +if __name__ == "__main__": + pd.set_option("display.width", 250) + for run in RUNS: + run_dir = glob.glob(os.path.join(GRID, run + "_*"))[0] + df, stop = trace(run_dir) + print(f"\n=== {os.path.basename(run_dir)}") + print(df.to_string(index=False, float_format=lambda v: f"{v:.4g}")) + print(stop) diff --git a/planning/studies/congression-calibration/.gitignore b/planning/studies/congression-calibration/.gitignore new file mode 100644 index 00000000..7419ba81 --- /dev/null +++ b/planning/studies/congression-calibration/.gitignore @@ -0,0 +1,11 @@ +# Grid outputs (tens of GB) and the calibration summary; see README.md. +# Every grid is set up as congression_calibration_/. +congression_calibration*/ +calib/ +diagnosis/*.csv +fetch.sh +# Written by tfs-setup-sim-grid next to a grid set up here; not inputs. +grid_summary.json +inputs/ +# Renamed copies of old grid runs +*.old/ diff --git a/planning/studies/congression-calibration/README.md b/planning/studies/congression-calibration/README.md new file mode 100644 index 00000000..f009a984 --- /dev/null +++ b/planning/studies/congression-calibration/README.md @@ -0,0 +1,773 @@ +# Congression calibration study + +**Question.** Fit to data from the step 2 simulator, is the observable-level +congression mixture (`transformation: mixture`) calibrated, and does it beat +`single` where congression matters? In particular: + +- Does the fit recover lambda? +- Are theta and the per-genotype parameters calibrated for bulk genotypes + *without* binding data, whose theta is informed only by their growth? +- What does fitting the mixture cost when there is no congression + (lambda = 0), and how much does a misspecified lambda prior hurt? + +**Decision it feeds.** Step 3.5 of +[`planning/congression-physics-plan.md`](../../congression-physics-plan.md): +whether the mixture is ready to use on real data. The simulator and the fit +use the same cell rules (max theta, dilution dk), so this checks the fit's +machinery given matched physics, not whether the rules are biologically +right. The mixture-vs-single effect sizes are specific to the max rule; +re-run the grid when step 4 changes the rule. + +## Design + +Base config: [`simulate_config.yaml`](simulate_config.yaml), from +`examples/simulate-and-analyze/` (483 genotypes, hill_mut theta, instant +growth transitions, kanR and pheS selections) with the real design's +`library_mixture` and 20 in-library binding anchors (`library_binding`). +Purity: 474 bulk genotypes, 3 pure spikes (D88A, H74A/K84L, +M42I/H74A/K84L), and 6 mixed (wt and the spiked singles, `bulk_fraction` +0.99 to 1.0; M42I/H74A and M42I/K84L, 0.74). + +[`grid.yaml`](grid.yaml): 3 x 2 x 3 x 3 = 54 runs. + +| axis | levels | +|---|---| +| simulated lambda | 0 (null), 0.357 (measured), 1.0 (stress) | +| dk_geno spread (`dk_geno_hyper_loc`) | -3.5 (wide, SD 0.018/min), -7.0 (tight, SD 0.0006/min, the measured regime) | +| seed (simulation and fit) | 1, 2, 3 | +| fit (`fit`, `lam_prior`) | `single`/`off`; `mixture`/`matched` (prior at the simulated lambda, 0.01 when it is 0, SD 0.05); `mixture`/`measured` (prior 0.357 +/- 0.05 whatever the truth) | + +At simulated lambda 0.357 the matched and measured fits are the same fit; +their results should agree up to GPU nondeterminism (a reproducibility +check). + +Each run ([`run.srun`](run.srun)): `tfs-simulate` -> `tfs-configure-model` +(the simulate-and-analyze components; `--transformation_model` and +`--transformation_lambda` from the fit arm) -> `tfs-prefit-calibration` -> +`tfs-fit-model` (SVI) -> `tfs-sample-posterior` -> `tfs-extract-params` -> +`tfs-predict-theta` -> `tfs-predict-growth` -> `tfs-summarize-fit`. + +## How to run + +On the cluster, from a scratch directory, with this repository checked out. +`tfs-setup-sim-grid` copies `hill_params.csv` into the grid's `inputs/` +directory and each run refers to it as `../inputs/hill_params.csv`, so the grid +directory can be moved as a unit after setup: + +```bash +tfs-setup-sim-grid /path/to/tfscreen/planning/studies/congression-calibration/grid.yaml \ + --out_prefix congression_calibration +for d in congression_calibration/run_*/; do (cd "$d" && sbatch run.srun); done +``` + +When the runs finish: + +```bash +tfs-summarize-calibration congression_calibration --out_prefix calib/congression \ + --baseline fit=single lam_prior=off \ + --facet_by transformation_poisson_lambda dk_geno_hyper_loc +``` + +Arms average over `seed` (the default `--replicate_keys`). + +## What to look at + +- `lam`: posterior against the simulated value, by lambda and prior. +- theta (`theta_test`) and per-genotype parameters by `purity` x + `has_binding`: coverage, calibration bias, width, RMSE; paired deltas + against `single`. +- The lambda = 0 arm: the mixture's cost when nothing is congressed. +- The `measured` prior at lambda 0 and 1.0: robustness to a wrong prior. + +## Commit + +Grid set up and run on `6e4735e` (2026-09-24). + +## Results (grid.yaml, 2026-09-24) + +All 54 runs finished and SVI converged. `tfs-summarize-calibration` outputs +are in `calib/` (not committed). The grid does not yet answer its question, +for three reasons: + +1. **The k/dk_geno slide is unanchored.** In every run the `growth_k` error + mirrors the `dk_geno` error, and the spread of the `dk_geno` errors is + small (0.001 to 0.01/min): each genotype's dk is recovered up to one + global constant. Nothing pins it here: `hierarchical_geno` does not pin + wt, there is no `base_growth` data, and the prefit's Hessian-derived + `k_scale` hit its 0.1 ceiling. `single` fits drift -0.02 to 0 in k, + mixture fits +0.02 to +0.07 (true baselines 0.01 to 0.03). `dk_geno` and + `growth_k` calibration are therefore meaningless in this grid. +2. **Congression is below the noise floor of this design.** `single`'s + errors do not change with the simulated lambda (growth RMSE 0.469 at + lambda 0 vs 0.471 at 1.0, tight dk; bulk theta without binding 0.180 vs + 0.181), although 42% of a bulk genotype's cells are congressed at + lambda 1.0. +3. **Lambda is pulled up whatever the truth.** With the 0.357 +/- 0.05 + prior, posterior medians were 0.46 to 0.66 at a true lambda of 0, + 0.46 to 0.60 at 0.357, and 0.44 to 0.63 at 1.0. With the matched prior at + 1.0: 1.00 to 1.28. Seed 3 is highest in every arm. + +Also: the mixture's theta RMSE is slightly worse than `single`'s in every +arm (0.20 to 0.22 vs 0.18, bulk genotypes), including lambda 0 with the +matched prior, where the fitted lambda is ~0.004. Bulk theta 95% coverage is +~0.65 in every arm, `single` included (a baseline overconfidence, separate +from congression). Binding genotypes are recovered well everywhere (theta +RMSE 0.015). + +## First diagnosis: the detection floor (2026-09-24; superseded, see below) + +Scripts: [`diagnosis/`](diagnosis/) (run from that directory against the +pulled grid). Runs 0001 (`single`) and 0002 (mixture, matched prior, true +lambda 0, fitted lambda 0.004) were fit to the same data. + +- The mixture's forward model is right: at `single`'s posterior-median + latents with lambda = 1e-8 it reproduces `single`'s `growth_pred` to 1e-6. +- The mixture fit leans on its congressed classes anyway: switching them off + at its own latents costs 36,700 in growth log-likelihood. A class weighted + 0.2% can only matter that much by outgrowing the clean class by ~6 ln + units. The fit gets there through dk dilution: slow genotypes (true + dk_geno -0.02 to -0.05) are fit at ~-0.12, and their congressed cells, + with dk averaged against a co-resident, carry the late timepoints. +- What those classes fit is the read-count floor. `ln_cfu` uses a + pseudocount of 1 (`process_raw/counts_to_lncfu.py`, the same path as real + data), so a dying genotype's late timepoints stall above the truth: + + | reads | mean error vs truth (ln) | reported SD | + |---|---|---| + | 0 | +1.26 | 1.00 | + | 1-2 | +0.62 | 0.71 | + | 3-5 | +0.22 | 0.45 | + | 6-20 | +0.07 | 0.29 | + + A single exponential cannot bend upward; a dying clean class plus a slower + congressed class can. Genotypes relying on congressed classes have zero + reads in 29% of their rows, against 6% for the rest. +- This accounts for the grid results: more congressed weight fits the floor + better (lambda pulled up everywhere), the dk shifts move the fits along + the unanchored slide, and the mixture spends flexibility on the artifact. + `single` is biased by the same points, less visibly. +- Real data share the pseudocount path, so a mixture fit to real data would + do the same. + +The 0.4 ln-unit error on well-measured rows is the simulator's per-tube +growth noise (`tube_noise_sigma` x t; per-sample SD 0.374, within-sample +0.054), which `growth_noise` is meant to absorb. Not a data problem. + +## Masked re-run (grid_masked.yaml, 2026-09-24): floor not the driver + +`grid_masked.yaml` dropped growth rows with fewer than 5 reads (`min_counts` +in `run.srun`) and refit grid runs 0001-0003 and 0037-0039. The cluster +reproduced the original simulations exactly. Comparison +([`diagnosis/compare_masked.py`](diagnosis/compare_masked.py)), true lambda 0: + +| | full data | masked | +|---|---|---| +| lambda, measured prior (0.357 +/- 0.05) | 0.55 | 0.53 | +| k offset, mixture (matched prior) | +0.062 | +0.065 | +| bulk theta RMSE, mixture vs `single` | 0.267 vs 0.219 | 0.270 vs 0.220 | + +Lambda 1.0 behaved the same. The mixture still leans on its congressed +classes (switching them off costs 21,100 in growth log-likelihood, against +36,700 unmasked). The floor bias is real and censoring is still worth doing, +but it does not drive the mixture's behavior. + +## Second diagnosis: the binding weight (2026-09-24; superseded, see below) + +Scripts: [`diagnosis/diag_basin.py`](diagnosis/diag_basin.py) and +[`diagnosis/diag_sites.py`](diagnosis/diag_sites.py) (runs 0001/0002). The +joint log density of the two lambda-0 fits differs almost entirely in the +binding likelihood: + +| term | mixture fit - `single` fit | +|---|---| +| binding likelihood, as weighted in the fit | +1,044,800 | +| growth likelihood | -37,600 | +| everything else | about -1,100 | + +`ModelOrchestrator` scales the binding likelihood by `binding_weight`, +which defaults to growth rows / binding rows (about 54,300 / 216 = 251 +here), and the simulated binding data have `theta_std` 0.001. Together they +dominate the objective; `single` cannot fit them (residuals ~15 sigma, +theta ~0.015). The mixture gives the fit a way out: pushing a genotype's own +dk_geno far down removes its clean cells from its trajectory, its congressed +classes (dk averaged with a co-resident) carry its growth, and its theta is +free to follow binding. The 20 in-library binding genotypes gain 1,430; the +4 spiked ones gain 2,730, mostly H74A (2,020), through hill_mut mutation +effects shared with bulk genotypes carrying H74 changes. This accounts for +lambda being pulled up, the slide, and the worse bulk theta and growth. +Not a mixture bug: the weight also bends `single` fits, less visibly. + +Background (user, 2026-09-24): the weight was added when, with ~200,000 +growth genotypes and ~10 binding genotypes, fits ignoring the binding data +won. That points to a conflict between the modalities over shared +parameters (m, k, shared mutation effects) rather than a volume problem; +the binding assay has since moved closer to cellular conditions. The +binding SDs used on real data are the SE of a Hill curve fitted to the +anisotropy points, which counts 9 correlated values as independent and +leaves no room for differences between assay and cell. + +## Binding weight 1 (grid_bw1.yaml, 2026-09-24): weight not the driver + +The same 6 simulations refit with `--binding_weight 1` +([`diagnosis/compare_masked.py`](diagnosis/compare_masked.py), which now +takes the follow-up grids as arguments). True lambda 0: + +| | original | masked | weight 1 | +|---|---|---|---| +| lambda, measured prior (0.357 +/- 0.05) | 0.55 | 0.53 | 0.47 | +| k offset, mixture (matched prior) | +0.062 | +0.065 | +0.080 | +| bulk theta RMSE, mixture | 0.267 | 0.270 | 0.247 | +| bulk theta RMSE, `single` | 0.219 | 0.220 | 0.227 | + +Theta 95% coverage improved in every arm at weight 1 (0.71 to 0.75, from +0.61 to 0.70), so the weight matters for calibration, but the mixture's +behavior survived. At weight 1 the mixture fit is worse than `single` on +both binding and growth, so no trade-off explains it. + +## Third diagnosis: the unanchored slide (2026-09-24; superseded, see below) + +- **Optimization, not the objective.** Under the mixture model's own joint + density, `single`'s solution at lambda 0.004 scores ~18,000 log units + better than where the mixture's SVI stopped + ([`diagnosis/diag_basin.py`](diagnosis/diag_basin.py), bw1 grid). +- **The drift happens in SVI, after the MAP warm-up.** After the warm-up all + fits have sensible `growth_k` (truth 0.011 to 0.029); the matched-prior + mixture then climbs to 0.089 to 0.112, with the between-condition + differences collapsing, and every `dk_geno` falls by ~0.09 (`ln_cfu0` + barely moves, so it is not a per-genotype trap). +- **No gradient bug.** The mean SVI gradient (64 draws) at `single`'s + solution is the same for both models (kanR+kan `growth_k`: -1.90e6 vs + -1.86e6), as is the exact joint-density gradient + ([`diagnosis/diag_gradient.py`](diagnosis/diag_gradient.py)). Both pull k + up from `single`'s point and back down from the mixture's: `single` + stopped short, the mixture overshot. +- **The only anchor is weak.** `hierarchical_geno` pins wt's `dk_geno` to 0, + and it stays 0 in both fits. The mixture fit abandons wt instead: it + predicts wt's `ln_cfu` ~13.6 ln units too high (`single`: +1.3). The + growth likelihood is Student-t (fitted nu ~9 `single`, ~6.4 mixture), + whose cost grows only logarithmically: that miss costs ~2,000 log units + here, against ~44,000 under a Normal. +- **The intended anchor never engaged.** The prefit's per-condition k prior + is meant to pin the slide, but its Hessian-based `k_scale` hit the 0.1 + ceiling in every run. + +## Anchored grid (grid_anchor.yaml, 2026-09-25) + +Binding weight 1 plus a tight prefit k prior (`--k_scale_ceiling 0.005`, +prefit `k_scale` 0.005). True lambda 0: + +| | original | weight 1 | weight 1 + tight k prior | +|---|---|---|---| +| k offset, `single` | -0.015 | -0.007 | -0.0004 | +| k offset, mixture (matched prior) | +0.062 | +0.080 | +0.083 | +| lambda, measured prior (0.357 +/- 0.05) | 0.55 | 0.47 | 0.45 | +| bulk theta RMSE, mixture vs `single` | 0.267 vs 0.219 | 0.247 vs 0.227 | 0.261 vs 0.230 | + +The prior fixed `single`; the mixture drifted ~17 prior SDs past it. Two +further endpoint checks were inconclusive or negative: the ELBO at the two +solutions ([`diagnosis/diag_elbo.py`](diagnosis/diag_elbo.py)) and the +population arrays the mixture looks co-residents up in, which match each +genotype's own values exactly +([`diagnosis/diag_population.py`](diagnosis/diag_population.py)). + +## Controls (grid_controls.yaml, 2026-09-25) + +Lambda-0 simulation, anchored settings. All three were still improving at +the stop. + +| run | k offset | fitted lambda | final loss | +|---|---|---|---| +| mixture, MAP | +0.0003 | 0.0047 | 6.5e4 | +| `single`, MAP | -0.0013 | | 6.2e4 | +| mixture, lambda pinned ~1e-6, SVI | +0.016 | 1e-6 | 1.7e5 | +| mixture, matched prior, SVI (anchored grid) | +0.083 | 0.004 | 1.8e5 | + +Without SVI's guide draws the mixture lands where `single` does; MAP also +reaches much lower loss. The pinned-lambda SVI fit drifts less but still +drifts. + +## Review: the fits never converged (2026-09-25) + +An independent review +([`../congression-calibration-convergence/`](../congression-calibration-convergence/README.md)) +found that none of the 3.5 fits converged, `single` included, and it +supersedes the endpoint diagnoses above. Verified here: + +- The stop rule (`RunInference._update_loss_deque`) divides the loss change + by the improvement since the start of SVI, whose loss is ~500 times the + final loss. Runs stop while the loss is still falling: run 0002 of the + anchored grid went 5.5e5, 3.6e5, 2.5e5, 1.9e5 over epochs 6000-9000 and + stopped at 9234. "SVI run converged" in the logs did not mean converged; + the diagnoses above took it at face value. +- The pre-MAP warm-up's step size decays to 1e-6 within its own 1000 + epochs, so SVI starts far from the mode with prior-width guide scales. +- Epoch checkpoints show `single` holding k at the truth while the + mixture's k jumps early (to +0.12 near epoch 4000, with nu ~1.3) and + returns slowly; it was cut off mid-return. The controls agree: under MAP + the mixture matches `single`. +- The simulated co-resident pool is 39% wt (the library enumerator adds a + wt entry per non-degenerate codon); being checked separately. + +What stands from the earlier sections: the read-count floor bias (0 reads ++1.26 ln) and the binding weight's dominance of the objective, as facts about +the data and the objective. Their effect on converged fits is open. + +## Resumed to convergence (resume.srun, 2026-09-25) + +Anchored runs 0001 (`single`, lambda 0), 0002 (mixture, matched prior, +lambda 0) and 0005 (mixture, matched prior, lambda 1.0), resumed from their +checkpoints for the full default 100,000 epochs. Traced with +`../congression-calibration-convergence/trace_checkpoints.py`: all three +stop moving after ~70,000 epochs, at losses of 5.2e4 to 5.4e4 (1.7e5 to +1.9e5 at the old stops). The resumed stop rule never fired, as expected. + +| growth_k | kanR+kan | kanR-kan | pheS+4CP | pheS-4CP | +|---|---|---|---|---| +| truth | 0.0107 | 0.0154 | 0.0214 | 0.0286 | +| `single`, lambda 0 | 0.0102 | 0.0139 | 0.0212 | 0.0286 | +| mixture, lambda 0 | 0.0109 | 0.0145 | 0.0220 | 0.0294 | +| mixture, lambda 1 | 0.0120 | 0.0170 | 0.0226 | 0.0295 | + +| run | fit | lambda (95%) | k offset | dk offset | bulk theta RMSE | bulk theta 95% coverage | +|---|---|---|---|---|---|---| +| 0001 `single`, lambda 0 | stopped | | -0.0004 | +0.0020 | 0.230 | 0.75 | +| | converged | | -0.0005 | +0.0014 | 0.129 | 0.66 | +| 0002 mixture, lambda 0 | stopped | 0.004 (0.003-0.005) | +0.083 | -0.095 | 0.261 | 0.70 | +| | converged | 0.043 (0.039-0.048) | +0.0002 | +0.0003 | 0.131 | 0.66 | +| 0005 mixture, lambda 1 | stopped | 0.98 (0.92-1.04) | +0.021 | -0.031 | 0.227 | 0.73 | +| | converged | 1.30 (1.29-1.32) | +0.0013 | -0.0029 | 0.173 | 0.55 | + +- **The transient explanation is confirmed.** Converged, the lambda-0 + mixture matches `single` (k, dk_geno, theta RMSE 0.131 vs 0.129). The k + drift, the abandoned wt and the worse theta were artifacts of stopping + early. +- **Converging halves the theta error** (bulk, no binding: 0.23 to 0.13). +- **Coverage got worse** as the guide scales shrank (0.66, and 0.55 for the + lambda-1 mixture): converged mean-field SVI is overconfident here. +- **Lambda is still pulled up, now with tight intervals that exclude the + truth:** 0.043 at a true 0, and 1.30 at a true 1.0 under a 1.0 +/- 0.05 + prior (six prior SDs). Open. +- Growth-noise nu at convergence: 61 (`single`), 28 (mixture lambda 0), + 6.5 (mixture lambda 1). The lambda-1 mixture needs much heavier tails. + +`single` on the lambda-1 data (run 0004), resumed the same way, also +converged (parameters flat after ~70,000 epochs). Against the lambda-1 +mixture (run 0005), same data: + +| lambda 1, converged | `single` (0004) | mixture (0005) | +|---|---|---| +| final loss | 5.59e4 | 5.40e4 | +| dk_geno RMSE | 0.0105 | 0.0049 | +| growth_m RMSE | 0.0012 | 0.0009 | +| bulk theta RMSE | 0.155 | 0.173 | +| log_hill_K RMSE | 1.03 | 1.17 | +| bulk theta 95% coverage | 0.58 | 0.55 | +| lambda | | 1.30 (truth 1.0) | + +The mixture fits the data better and halves the dk_geno error (the 3.0 +study's main congression channel, dk dilution) but does not recover theta: +`single`'s theta RMSE rises from 0.129 at lambda 0 to 0.155 at lambda 1, +and the mixture's is 0.173. The overestimated lambda may be +over-correcting theta; the lambda profile tests that. + +These grids used the library enumeration from before bff2c51 (wt 39% of the +co-resident pool; see the convergence study, result 5). Simulation and fit +shared it, so these comparisons stand; new grids from `simulate_config.yaml` +simulate a different library. + +## Baseline and profile grids on 5bf5b78: the gradient clip (2026-09-25) + +All 13 runs of `grid_baseline.yaml` and `grid_profile.yaml` ran to 100,000 +steps without converging, and the lambda-1 seed-1 fits (baseline 0005/0006, +profile 0.8-1.4) jumped to 4-5 times their best loss and stayed there +through three step-size cuts. Neither is a convergence-rule problem at +root; both come from `ClippedAdam`'s elementwise gradient clip at 1.0 +(`adam_clip_norm`, older than 5bf5b78). + +- **The per-step loss is quantized**: 41k + n x 243k (n = 0-5) on run 0005. + Each quantum is one binding point of M42C/K84N (from `hill_params.csv`, so + in every simulation): true `hill_n` 24.6, K between the 0.001 and 0.003 + grid points, measured with SD 0.001. A guide draw that shifts the step + across a grid point flips theta 0.99 <-> 0.007 there (`diagnosis/diag_clip.py`, + part 1). +- **Every run carries these penalties at a steady share of steps**, old code + and new: ~5% (the resumed anchored runs), 5-15% (seed 2), 27-32% (lambda-0 + seed 1), 46-89% (lambda-1 seed 1). The "blow-ups" are the 250-step block + medians flipping to the penalized level once the share passes 50%; guide + parameters moved by at most ~0.4 across the jump. +- **The share is set by the clip.** Gradients here are 1e3-1e6 per element, + so every element is clipped every step and Adam follows each draw's + gradient sign; it settles where the signs balance, not where the mean + gradient vanishes. At checkpoint 40000 of run 0005, mutation 1's + `theta_d_logit_delta_offset_locs` has mean gradient -7.9e3 +/- 0.9e3 (away + from the penalty) and mean clipped gradient -0.04; for + `theta_d_logit_low_offset_locs` the clip reverses the sign (-1.2e3 raw, + +0.14 clipped). The balance point does not depend on the step size (48% + at 1e-3 and at 1e-6), so the cuts locked it in. +- **Resumed from run 0005's checkpoints** (`diag_clip.py` part 3, local CPU): + +| from | clip | step size | penalized share per 2000 steps | mean loss | +|---|---|---|---|---| +| step 8000 (before the jump) | 1.0 | 1e-3 | 0.17, 0.42, 0.57, 0.77, 0.97 | 96k -> 485k | +| step 8000 | none | 1e-3 | 0.01, 0.00 | 47k -> 38k | +| step 40000 (after) | none | 1e-3 | 0.21, 0.07, 0.04, 0.02, 0.00 | 102k -> 40k | +| step 40000 | 1.0 | 1e-4 | 0.48 throughout | ~200k | + +- **Fresh fits of run 0005's data** (`tfs-fit-model`, CPU): unclipped, the + penalties vanish by step 8000 and SVI **converges at step 46,000** (3 cuts, + loss 38.5k, below the clipped run's best of 41k). With the clip and the old + start (`--pre_map_num_epoch 0`) it settles at 8-11% penalized (mean loss + 68k), not 48%: 5bf5b78's pre-MAP start only picked a worse clipped + equilibrium on this simulation. +- **The non-stop (failure 1) is the same clip, plus a yardstick problem.** + At 1e-6 the flagged locations crept in one direction at ~10% of Adam's top + speed for 60k steps (clipped dynamics), measured against guide SDs that had + collapsed to ~4e-4 (hill_mut `theta_sigma_d_*`, `theta_epi_tau`). The + collapse itself is the mean-field ELBO (it happens unclipped too; the raw + and clipped gradients agree on shrinking), so those reported widths are + not honest posterior SDs. Unclipped, the worst movement at 1e-6 was + 0.02-0.03 SDs and the rule stopped the run. + +Fixed in the optimizer (CHANGELOG, "Gradient clipping is off by default"): +plain Adam by default (`--adam_clip_norm` opts back in); at the final step +size a window whose mean sits more than 3 robust SDs above its median (rare +penalties the median hides) is not a plateau; posterior SDs used to measure +parameter movement are floored at 1% of the prior SD. Every result on these +two grids, and any clipped fit with this binding set, is biased and should +be rerun. Separately, binding noise 0.001 on step-like curves is much tighter +than real assays; to be loosened in the simulation config. + +## Next + +The convergence work has landed (5bf5b78: noise-referenced stop rule, +step-size cuts on stalls, SVI started from a converged pre-MAP with guide +scales capped at 0.1; `--convergence_tolerance` is gone and `run.srun` / +`resume.srun` no longer pass it). New grids use it and simulate the +corrected library (bff2c51). + +1. Why lambda is pulled up on converged fits: profile lambda (fixed lambda + on a grid, everything else converged) on the lambda-0 and lambda-1 data, + then test candidate causes one at a time (the read-count floor, which + curves dying trajectories the way a mixture does; realized vs design + library composition in the simulator's transformation, including + `lib_assembly_skew_sigma`; the fixed co-resident sets). +2. Coverage: why converged fits are overconfident (guide family, the + binding weight). + +Set up (2026-09-25), anchored settings (binding weight 1, prefit +`--k_scale_ceiling 0.005`), corrected library (483 genotypes as before; +wt's co-resident share 0.0026, from 0.39): + +- `grid_baseline.yaml`: lambda 0 and 1.0 x {`single`, mixture with the + matched prior} x seeds 1 and 2 (8 runs). Does the new stop rule end runs + cleanly, and do the converged results above hold on the corrected + library? +- `grid_profile.yaml`: the mixture with lambda held at 0.6, 0.8, 1.0, 1.2, + 1.4 (`lam_prior: fixed`, prior SD 0.1% of the value) on the lambda-1, + seed-1 simulation (5 runs). Loss against lambda says whether the data + prefer lambda > 1; theta error against lambda says whether theta is + recovered at the true lambda. + +## Baseline and profile grids (2026-09-25) + +Both on the noise-referenced convergence code (5bf5b78) and the corrected +library (bff2c51). **The stop rule never fired: all 13 runs ran to the +100,000-step cap** ("SVI run has not yet converged"). After three 10x +step-size cuts (to 1e-6) the parameter criterion still reads hill_mut's +hierarchical scales as moving by 0.3-0.5 posterior SDs; their guide SDs +have collapsed to 0.001-0.005, so noise-level movement counts. + +**Blow-ups on one simulation.** On the lambda-1, seed-1 simulation, both +`single` (baseline 0005) and the mixture (0006), and four of the five +profile runs (lambda fixed at 0.8-1.4), reach a good loss (4-6e4) and then +jump ~5x at full step size between steps 8,000 and 14,000 (profile run +0003: one step at epoch 8,750 took it from 7.2e4 to 2.8e5 with guide +parameters moving at most ~0.4). The step-size cuts then lock the damage in. +Flagged parameters: hill_mut's `theta_sigma_d_*` scales. Not +congression-specific (`single` does it too) and not seen before 5bf5b78 on +the analogous runs. The profile grid is therefore uninterpretable: only the +lambda-0.6 run avoided the blow-up. Both failures went to the convergence +work as a separate task. + +**The stable pairs** (losses ended within ~2% of their best, so usable with +that caveat): + +| | lambda (95%) | dk_geno RMSE | bulk theta RMSE | bulk theta 95% coverage | +|---|---|---|---|---| +| lambda 0, seed 1: `single` / mixture | - / 0.155 (0.151-0.160) | 0.0023 / 0.0017 | 0.133 / 0.121 | 0.82 / 0.83 | +| lambda 0, seed 2: `single` / mixture | - / 0.149 (0.145-0.154) | 0.0021 / 0.0017 | 0.116 / 0.101 | 0.79 / 0.82 | +| lambda 1, seed 2: `single` / mixture | - / 0.98 (0.97-0.99) | 0.0077 / 0.0023 | 0.106 / 0.121 | 0.76 / 0.81 | + +- At lambda 1 the mixture recovers lambda (0.98 at a true 1.0; 1.30 on the + old library) and cuts the dk_geno error ~3x; bulk theta is still slightly + worse than `single`'s. +- At lambda 0 the mixture still finds lambda ~0.15 (matched prior centered + at 0.01), yet its theta and dk_geno are slightly better than `single`'s. + What the congressed classes absorb there is open (the read-count floor is + one candidate). +- Coverage is better than on the old code (0.76-0.83, from 0.55-0.66). + +Next: wait for the convergence fixes, then re-run the profile grid and more +seeds of the baseline. + +## Realism fixes before the re-run (2026-09-26) + +The blow-ups traced in part to unrealistic binding data: SD 0.001 on +theta, and near-step curves (the simulator drew Hill coefficients up to 270; +50 of 483 genotypes above 4, including 5 of the 20 in-library binding +anchors). From here on the study config uses binding noise 0.025 (the +experimental theta error, user) and `theta_sim_priors.max_hill_n: 4.0` +(new; larger draws are set to 4, so ~10% of genotypes sit exactly at 4). +The optimizer was also updated. Grids after this date are not comparable +with earlier ones. + +The model's default binding weight (growth rows / binding rows) is also to +be lowered, but that is a change to the model for real fits; it is left +for its own step. These grids run at `--binding_weight 1` regardless. + +## Re-run with realistic data and the fixed optimizer (2026-09-26) + +`grid_baseline.yaml` and `grid_profile.yaml` again, with binding noise +0.025, `max_hill_n` 4 and the updated optimizer. **All 13 runs converged** +under the stop rule (40,000-48,000 steps), each ending at its best loss; no +blow-ups. `tfs-summarize-calibration` outputs: `calib/baseline_*`. + +Baseline (bulk theta and parameters; 95% intervals for lambda): + +| lambda, seed | fit | lambda | dk_geno RMSE | log_hill_K RMSE | bulk theta RMSE | bulk theta coverage | loss | +|---|---|---|---|---|---|---|---| +| 0, 1 | `single` | | 0.0023 | 0.72 | 0.108 | 0.79 | 35,874 | +| 0, 1 | mixture | 0.131 (0.128-0.134) | 0.0015 | 0.67 | 0.089 | 0.82 | 35,153 | +| 0, 2 | `single` | | 0.0024 | 0.72 | 0.097 | 0.80 | 32,480 | +| 0, 2 | mixture | 0.113 (0.110-0.116) | 0.0013 | 0.71 | 0.079 | 0.86 | 32,116 | +| 1, 1 | `single` | | 0.0092 | 0.94 | 0.125 | 0.58 | 37,837 | +| 1, 1 | mixture | 0.942 (0.936-0.948) | 0.0032 | 0.75 | 0.104 | 0.81 | 36,285 | +| 1, 2 | `single` | | 0.0080 | 0.65 | 0.084 | 0.77 | 29,437 | +| 1, 2 | mixture | 0.803 (0.796-0.810) | 0.0024 | 0.61 | 0.074 | 0.87 | 28,182 | + +Paired against `single` (`calib/baseline_paired_summary.csv`, theta, mean +of 2 seeds), the gain is in bulk genotypes without binding data: + +| lambda | stratum | delta theta RMSE | delta 95% coverage | +|---|---|---|---| +| 0 | bulk, no binding | -0.018 | +0.05 | +| 0 | with binding / spike / mixed | ~0 | small | +| 1 | bulk, no binding | -0.015 | +0.16 | +| 1 | with binding / spike / mixed | ~0 | small | + +Profile (mixture, lambda fixed, lambda-1 seed-1 data): + +| lambda held at | 0.6 | 0.8 | 1.0 | 1.2 | 1.4 | +|---|---|---|---|---|---| +| loss | 36,319 | 35,812 | 36,294 | 35,870 | 36,013 | +| bulk theta RMSE | 0.111 | 0.107 | 0.105 | 0.102 | 0.102 | +| dk_geno RMSE | 0.0044 | 0.0034 | 0.0032 | 0.0041 | 0.0052 | + +- **The mixture now beats `single` where congression is real** (lambda 1): + bulk theta RMSE 0.104 vs 0.125 and 0.074 vs 0.084, dk_geno error cut + ~3x, coverage 0.81-0.87 vs 0.58-0.77, and a better fit. The earlier + "mixture worse on theta" results came from unconverged or unstable fits. +- **Lambda is recovered roughly, with overconfident intervals:** 0.94 and + 0.80 at a true 1.0; intervals ~+/-0.007 exclude the truth. At a true 0 + the mixture still finds 0.11-0.13, yet fits better than `single` on + theta, dk_geno and coverage there too, so its congressed classes absorb + something real in the data (the read-count floor is the leading + candidate). +- **The profile is flat within the loss noise** (differences of a few + hundred, not monotone): these data barely pin lambda between 0.6 and 1.4. + Theta error changes little across that range (0.111 to 0.102), and + dk_geno error is lowest near the true lambda. So a misestimated lambda + costs little in theta. +- Coverage remains below nominal for both fits (0.78-0.87), the mixture's + less so. + +## Confirming round (grid_measured.yaml, grid_seed3.yaml, 2026-09-26) + +Same settings as the re-run above (`max` rule on both sides). **All 10 runs +converged** (38,000-48,000 steps). Outputs: `calib/measured_*`, and +`calib/pooled_*` for the baseline grid plus seed 3 (runs linked into +`calib/pooled_all/`). + +Measured regime (lambda 0.357, prior 0.357 +/- 0.05): + +| seed | fit | lambda (95%) | dk_geno RMSE | bulk theta RMSE | bulk theta coverage | loss | +|---|---|---|---|---|---|---| +| 1 | `single` | | 0.0057 | 0.112 | 0.69 | 32,654 | +| 1 | mixture | 0.266 (0.262-0.269) | 0.0025 | 0.083 | 0.83 | 31,733 | +| 2 | `single` | | 0.0056 | 0.145 | 0.53 | 31,228 | +| 2 | mixture | 0.439 (0.434-0.444) | 0.0017 | 0.110 | 0.64 | 29,604 | +| 3 | `single` | | 0.0064 | 0.106 | 0.72 | 34,176 | +| 3 | mixture | 0.347 (0.341-0.353) | 0.0026 | 0.068 | 0.85 | 32,850 | + +Seed 3 at lambda 0 and 1: + +| lambda | fit | lambda (95%) | dk_geno RMSE | bulk theta RMSE | bulk theta coverage | loss | +|---|---|---|---|---|---|---| +| 0 | `single` | | 0.0023 | 0.103 | 0.69 | 33,228 | +| 0 | mixture | 0.119 (0.116-0.123) | 0.0018 | 0.086 | 0.71 | 32,675 | +| 1 | `single` | | 0.0088 | 0.099 | 0.56 | 25,878 | +| 1 | mixture | 0.656 (0.650-0.662) | 0.0037 | 0.078 | 0.78 | 24,694 | + +Paired against `single` (bulk genotypes without binding data; mean +/- SE +over 3 seeds): + +| lambda | delta theta RMSE | delta 95% coverage | delta dk_geno RMSE | +|---|---|---|---| +| 0 | -0.018 +/- 0.000 | +0.04 +/- 0.01 | -0.0008 | +| 0.357 | -0.034 +/- 0.003 | +0.13 +/- 0.01 | -0.0036 | +| 1 | -0.017 +/- 0.004 | +0.18 +/- 0.04 | -0.0056 | + +Genotypes with binding data are unchanged (delta theta RMSE ~0). + +- **The mixture beats `single` in all 9 pairs** on fit (loss), bulk theta + RMSE, coverage and dk_geno. The gain is largest in the measured regime. +- **Lambda: roughly right on average, badly overconfident per run.** At + 0.357 the three seeds give 0.27, 0.44 and 0.35 (mean 0.35); at 1.0 they + give 0.94, 0.80 and 0.66; at 0 they give 0.13, 0.11 and 0.12. The + seed-to-seed spread (SD ~0.09 at 0.357, ~0.14 at 1.0) is 30-50 times + the reported posterior SD (~0.003), and seed 3 at 1.0 sits 7 prior SDs + below its matched prior. Theta is recovered well regardless, as the flat + profile predicted. +- Coverage is still below nominal (mixture 0.64-0.86). + +Step 3.5 is closed on these results. Open items are recorded in the plan. + +## Step 4 changes that affect this study (2026-09-26) + +The fit's default congression theta rule is now `homodimer` and the default +activity `fixed`. `simulate_config.yaml` keeps `congression_theta_rule: max` +so the recorded grids stay reproducible, and `run.srun` now passes the fit +the rule the simulation used (`fit_theta_rule` overrides it, for cross-rule +grids). The confirming round (`grid_measured.yaml`, `grid_seed3.yaml`) was +set up before this change and runs `max` on both sides. + +`grid_crossrule.yaml` (set up 2026-09-26) is the step 4 validation: simulate +with `max` or `homodimer`, then fit each simulation with `single` and with +the mixture under each rule, at lambda 0.357 and 1.0 x seeds 1-2 (24 runs). +Does the right rule recover theta, and what does the wrong one cost? The +summary commands are in the grid file's header. + +## Cross-rule grid (grid_crossrule.yaml, 2026-09-26) + +**All 24 runs converged**, and each fit used the rule it was set up with +(`max` and `homodimer` fits of the same data stop at the same step only +because the windows are 2,000 steps). Outputs: `calib/crossrule_*` +(against `single`), `calib/crossrule_vs_max_*` (homodimer against max on +the same simulation). + +Bulk genotypes without binding data (theta RMSE, 95% coverage, loss): + +| lambda | simulated | seed | `single` | mixture, `max` | mixture, `homodimer` | +|---|---|---|---|---|---| +| 0.357 | `max` | 1 | 0.112, 0.69, 32,610 | 0.083, 0.83, 31,727 | 0.076, 0.85, 31,675 | +| 0.357 | `max` | 2 | 0.144, 0.53, 31,240 | 0.108, 0.65, 29,468 | 0.104, 0.69, 29,411 | +| 0.357 | `homodimer` | 1 | 0.122, 0.69, 32,739 | 0.091, 0.81, 31,824 | 0.083, 0.82, 31,770 | +| 0.357 | `homodimer` | 2 | 0.147, 0.53, 31,169 | 0.108, 0.67, 29,694 | 0.105, 0.63, 29,581 | +| 1 | `max` | 1 | 0.125, 0.58, 37,829 | 0.104, 0.81, 36,292 | 0.117, 0.79, 35,861 | +| 1 | `max` | 2 | 0.084, 0.77, 29,421 | 0.074, 0.87, 28,173 | 0.071, 0.86, 28,159 | +| 1 | `homodimer` | 1 | 0.125, 0.55, 37,729 | 0.095, 0.83, 36,202 | 0.101, 0.82, 36,241 | +| 1 | `homodimer` | 2 | 0.087, 0.76, 29,479 | 0.086, 0.84, 28,251 | 0.075, 0.85, 28,171 | + +Homodimer fit minus max fit, same simulation (mean +/- SE over 2 seeds): + +| lambda | simulated | delta theta RMSE | delta coverage | delta dk_geno RMSE | fitted lambda (max / homodimer) | +|---|---|---|---|---|---| +| 0.357 | `max` | -0.006 +/- 0.001 | +0.03 | 0.0000 | 0.27, 0.44 / 0.27, 0.47 | +| 0.357 | `homodimer` | -0.006 +/- 0.003 | -0.01 | 0.0000 | 0.26, 0.43 / 0.27, 0.45 | +| 1 | `max` | +0.005 +/- 0.008 | -0.01 | 0.0000 | 0.94, 0.80 / 0.98, 0.85 | +| 1 | `homodimer` | -0.003 +/- 0.008 | +0.00 | -0.0001 | 0.88, 0.77 / 0.91, 0.84 | + +- **The rule is not identifiable from these data, and the wrong one costs + nothing measurable.** Theta, dk_geno and coverage differ between the two + mixture fits by less than the seed-to-seed spread whichever rule + simulated the data. The homodimer fit has the lower loss in 7 of 8 + pairs, including all 4 simulated with `max`, by 14-430, within the + loss noise seen in the lambda profile. +- **Both mixture fits beat `single` in all 16 pairs** (bulk theta RMSE + -0.010 to -0.041, coverage +0.12 to +0.18, dk_geno error ~2.5x lower). +- The homodimer fit puts lambda slightly higher (by 0.01-0.07), closer to + the truth at 1.0. Partition-function mixing pulls a strong binder's + theta down in a congressed cell, so matching the same data takes a + little more congression than `max` needs. +- The simulated data do differ by rule (`ln_cfu` differs by 0.1 on average + between the `max` and `homodimer` simulations of the same seed; this + includes resampled read counts), but not in a way the fit can pin. + +`homodimer` stays the default (the physics, the user's call in step 4); the +choice between the rules waits on native mass spec, not on screen data. + +## Step 5: dk rule sensitivity (grid_dkrule.yaml, set up 2026-09-26) + +The fit and simulator now offer the soft-min dk family (`dilution`, +`softmin` with `congression_dk_alpha`, `min`). `run.srun` fits with the +simulation's dk rule and alpha unless `fit_dk_rule`/`fit_dk_alpha` are +set. `grid_dkrule.yaml` simulates with `dilution` or `min` and fits each +simulation with `single` and the mixture under each rule (homodimer theta +rule on both sides), at the wide dk spread (worst case) at lambda 0.357 and +1.0 and the tight, realistic spread at 0.357, x seeds 1-2 (36 runs). Does +the dk rule matter, and does it matter at a realistic dk spread? Summary +commands are in the grid file's header. + +### Results (2026-09-26) + +**All 36 runs converged**, each fit with its intended dk rule. Outputs: +`calib/dkrule_*` (against `single`), `calib/dkrule_vs_dilution_*` (min +against dilution on the same simulation). Means over seeds 1-2 (bulk theta +without binding data; lambda per seed): + +| design | simulated | fit | bulk theta RMSE | coverage | dk_geno RMSE | lambda | +|---|---|---|---|---|---|---| +| wide dk, lambda 0.357 | dilution | `single` | 0.135 | 0.60 | 0.0057 | | +| | | mixture, dilution | 0.094 | 0.75 | 0.0023 | 0.27, 0.47 | +| | | mixture, min | 0.140 | 0.61 | 0.0055 | 0.13, 0.34 | +| | min | `single` | 0.115 | 0.71 | 0.0023 | | +| | | mixture, dilution | 0.090 | 0.75 | 0.0018 | 0.11, 0.19 | +| | | mixture, min | 0.121 | 0.70 | 0.0021 | 0.15, 0.44 | +| wide dk, lambda 1 | dilution | `single` | 0.103 | 0.68 | 0.0086 | | +| | | mixture, dilution | 0.088 | 0.83 | 0.0028 | 0.91, 0.84 | +| | | mixture, min | 0.107 | 0.68 | 0.0080 | 0.83, 0.76 | +| | min | `single` | 0.103 | 0.76 | 0.0024 | | +| | | mixture, dilution | 0.084 | 0.82 | 0.0050 | 0.47, 0.38 | +| | | mixture, min | 0.116 | 0.78 | 0.0017 | 1.02, 0.89 | +| tight dk, lambda 0.357 | dilution | `single` | 0.065 | 0.70 | 0.00025 | | +| | | mixture, dilution | 0.064 | 0.78 | 0.00032 | 0.55, 0.90 | +| | | mixture, min | 0.065 | 0.76 | 0.00028 | 0.56, 0.97 | +| | min | `single` | 0.065 | 0.71 | 0.00026 | | +| | | mixture, dilution | 0.064 | 0.79 | 0.00034 | 0.54, 0.89 | +| | | mixture, min | 0.065 | 0.76 | 0.00029 | 0.55, 0.96 | + +- **At a realistic (tight) dk spread the dk rule does not matter:** theta, + dk_geno and loss agree between the two rules to within 0.001 RMSE and + 30 loss units, whichever rule simulated the data. The mixture does not + beat `single` on theta there either (-0.001); it does improve coverage + (+0.06 to +0.08). +- **At the wide spread, the dilution fit gives the best theta whichever + rule simulated the data** (0.084-0.094 vs 0.103-0.135 for `single`), + and it also has the lower loss on min-simulated data at lambda 0.357 + (by 360-480). The min fit gains nothing on theta over `single`, even + when min is the truth (0.121 vs 0.115; 0.116 vs 0.103 at lambda 1). +- **dk_geno and lambda need the right rule.** On min-simulated data at + lambda 1, the min fit recovers lambda (1.02, 0.89) and gives the best + dk_geno (0.0017); the dilution fit puts lambda at 0.47 and 0.38 and its + dk_geno error is twice `single`'s. On dilution-simulated data the + dilution fit cuts dk_geno error ~3x and the min fit does not. So the + min fit is consistent with its own data: no sign of a fitting bug. +- **What the dilution mixture's theta gain comes from.** It does not + track congression: it is as large on min-simulated data (where + congressed cells cannot outgrow a slow genotype) as on dilution data, + about the same at lambda 0 in the baseline grid (-0.018) as at lambda 1 + (-0.017), and absent at the tight dk spread. It needs slow genotypes and + a rule that lets congressed cells outgrow them. The read-count floor fits + that description (a dying genotype's late timepoints stall above the + truth, and only a faster-growing class can bend its trajectory up), but + this is an inference from the pattern, not yet a test. The dk_geno gain, + by contrast, grows with lambda and follows the right rule: that part is + congression. +- **Lambda at the tight spread is pulled well above the truth** (0.54 to + 0.97 at a true 0.357, against a 0.357 +/- 0.05 prior), identically under + both rules. With little dk spread only theta mixing carries congression, + and the fit leans on it; worth understanding before trusting a fitted + lambda on real data (where the spread is tight). + +`dilution` stays the default: at a realistic spread the choice does not +matter, and at a wide one no rule does better on theta. diff --git a/planning/studies/congression-calibration/diagnosis/compare_masked.py b/planning/studies/congression-calibration/diagnosis/compare_masked.py new file mode 100644 index 00000000..8f6bac85 --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/compare_masked.py @@ -0,0 +1,60 @@ +""" +Compare follow-up runs (grid_masked.yaml: rows with < 5 reads dropped; +grid_bw1.yaml: binding_weight 1) with their twins in grid.yaml (same +simulated data). Run from the study directory; arguments name the follow-up +grid directories (default: congression_calibration_masked). +""" +import glob +import json +import sys + +import numpy as np +import pandas as pd + +PAIRS = {"0001": "0001", "0002": "0002", "0003": "0003", + "0004": "0037", "0005": "0038", "0006": "0039"} + + +def summarize(d): + c = json.load(open(f"{d}/combo.json")) + s, t = c["simulate"], c["template"] + out = dict(lam_true=s["transformation_poisson_lambda"], + arm=f"{t['fit']}/{t['lam_prior']}") + lam = glob.glob(f"{d}/summary/*_params_lam.csv") + if lam: + r = pd.read_csv(lam[0]).iloc[0] + out["lam_q50"], out["lam_q025"], out["lam_q975"] = r["q0.5"], r["q0.025"], r["q0.975"] + k = pd.read_csv(glob.glob(f"{d}/summary/*_params_growth_k.csv")[0]) + dk = pd.read_csv(glob.glob(f"{d}/summary/*_params_dk_geno.csv")[0]) + out["k_off"] = (k["q0.5"] - k["ref"]).mean() + out["dk_off"] = (dk["q0.5"] - dk["ref"]).mean() + out["dk_rmse_centered"] = ((dk["q0.5"] - dk["ref"]) - out["dk_off"]).pow(2).mean() ** 0.5 + + th = pd.read_csv(glob.glob(f"{d}/summary/*_theta_corr_test.csv")[0]) + lib = pd.read_csv(glob.glob(f"{d}/*_sim_library.csv")[0]) + binding = set(pd.read_csv(glob.glob(f"{d}/*_sim_binding.csv")[0]).genotype.astype(str)) + spiked = set(lib.loc[lib.library_origin == "spiked", "genotype"].astype(str)) + th["genotype"] = th.genotype.astype(str) + bulk_nobind = th[~th.genotype.isin(spiked | binding)] + lo, hi = bulk_nobind["q0.025"], bulk_nobind["q0.975"] + out["theta_rmse"] = ((bulk_nobind["q0.5"] - bulk_nobind["ref"]) ** 2).mean() ** 0.5 + out["theta_cov95"] = ((lo <= bulk_nobind.ref) & (bulk_nobind.ref <= hi)).mean() + return out + + +GRIDS = sys.argv[1:] or ["congression_calibration_masked"] +rows = [] +for m, o in PAIRS.items(): + runs = [("original", glob.glob(f"congression_calibration/run_{o}_*")[0])] + runs += [(g.replace("congression_calibration_", ""), glob.glob(f"{g}/run_{m}_*")[0]) + for g in GRIDS] + for tag, d in runs: + r = summarize(d) + r["data"] = tag + rows.append(r) + +df = pd.DataFrame(rows) +cols = ["lam_true", "arm", "data", "lam_q025", "lam_q50", "lam_q975", + "k_off", "dk_off", "dk_rmse_centered", "theta_rmse", "theta_cov95"] +pd.set_option("display.width", 220) +print(df[cols].sort_values(["lam_true", "arm", "data"]).round(4).to_string(index=False)) diff --git a/planning/studies/congression-calibration/diagnosis/diag_basin.py b/planning/studies/congression-calibration/diagnosis/diag_basin.py new file mode 100644 index 00000000..b4ffc70d --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_basin.py @@ -0,0 +1,57 @@ +""" +Is the mixture fit's solution better than single's solution under the +mixture model (joint log density, priors included)? And where does the +mixture's better binding fit come from? Run from this directory; GRID +selects the grid (default: the unmasked one). +""" +import sys +sys.argv = [sys.argv[0]] +from diag_lam0 import * +import pandas as pd +from numpyro.infer.util import log_density + +(d1, o1, s1, st1), (d2, o2, s2, st2) = load("run_0001"), load("run_0002") +lat1, _ = median_latents(o1, s1, st1) +lat2, _ = median_latents(o2, s2, st2) +data = full_batch(o2) + + +def joint(orch, lat): + lp, _ = log_density(seed(orch.jax_model, 0), (), dict(data=full_batch(orch), priors=orch.priors), lat) + return float(lp) + + +x = dict(lat1); x["transformation_lam"] = lat2["transformation_lam"] +print("mixture model joint log density:") +print(" at mixture fit %.1f" % joint(o2, lat2)) +print(" at single fit (same lam) %.1f" % joint(o2, x)) +y = dict(lat2); y.pop("transformation_lam") +print("single model joint log density:") +print(" at single fit %.1f" % joint(o1, lat1)) +print(" at mixture fit %.1f" % joint(o1, y)) + +# Binding log-lik per genotype at each fit +def binding_by_geno(orch, lat): + tr = trace(substitute(seed(orch.jax_model, 0), data=lat)).get_trace( + data=full_batch(orch), priors=orch.priors) + site = tr["binding_obs"] + lp = site["fn"].log_prob(site["value"]) + if site.get("mask") is not None: + lp = lp * site["mask"] + return np.asarray(lp) + +b1 = binding_by_geno(o1, lat1); b2 = binding_by_geno(o2, lat2) +bt = o2.binding_tm +names = bt.tensor_dim_names +labels = list(bt.tensor_dim_labels[names.index("genotype")]) +gain = (b2 - b1).reshape(-1, b1.shape[-1]).sum(axis=0) if b1.shape[-1] == len(labels) else None +if gain is None: + print("binding tensor layout", names, b1.shape) +else: + lib = pd.read_csv(glob.glob(d2 + "/*_sim_library.csv")[0]) + spiked = set(lib.loc[lib.library_origin == "spiked", "genotype"].astype(str)) + df = pd.DataFrame({"genotype": [str(g) for g in labels], "binding_gain": gain}) + df["spiked"] = df.genotype.isin(spiked) + print("\nbinding log-lik gain (mixture fit - single fit), by origin:") + print(df.groupby("spiked").binding_gain.agg(["count", "sum"]).round(1)) + print(df.sort_values("binding_gain", ascending=False).head(8).round(1).to_string(index=False)) diff --git a/planning/studies/congression-calibration/diagnosis/diag_clip.py b/planning/studies/congression-calibration/diagnosis/diag_clip.py new file mode 100644 index 00000000..3e6d8e92 --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_clip.py @@ -0,0 +1,148 @@ +""" +Why the step-3.5 baseline/profile fits never converged and "blew up": the +per-step loss is 41k + n * 243k (n = draws crossing one binding step), and +ClippedAdam's elementwise clip makes Adam settle where gradient *signs* +balance, holding a large share of draws across the step. + +Three checks at one checkpoint, from inside a run directory: + + cd congression_calibration_baseline/run_0005_lam1_0_seed1_single_off + python ../../diagnosis/diag_clip.py checkpoints/0040000_checkpoint.pkl + +1. events: per-draw binding log-likelihood; which observations carry the + penalty, and the predicted theta there. +2. gradients: mean raw vs mean clipped (+/-1) gradient of the loss for the + parameters that differ most between penalized and clean draws. +3. resume: --steps optimizer steps from the checkpoint with and without the + clip (--lr, default 1e-3); share of penalized steps and mean loss per + 2000 steps. + +Needs this worktree's src on PYTHONPATH (the editable install may point at +another checkout). +""" + +import argparse + +import jax +import jax.numpy as jnp +import numpy as np +from numpyro import handlers + +from tfscreen.tfmodel.configuration_io import read_configuration +from tfscreen.tfmodel.inference.run_inference import ( + RunInference, + adam_optimizer, +) + +parser = argparse.ArgumentParser() +parser.add_argument("checkpoint") +parser.add_argument("--draws", type=int, default=40) +parser.add_argument("--steps", type=int, default=8000) +parser.add_argument("--lr", type=float, default=1e-3) +args = parser.parse_args() + +orch, init_params = read_configuration("tfs_configure_config.yaml") +ri = RunInference(orch, seed=1) +svi, state = ri.restore_svi_from_checkpoint(args.checkpoint, + init_params=init_params) +params = svi.get_params(state) +data = jax.device_put(orch.data) +# The training index (binding genotypes first, the rest shuffled); the +# library-ordered data.batch_idx pairs genotypes with the wrong rows. +batch = orch.get_batch(data, jnp.asarray(ri.model.get_random_idx())) + + +def penalized(losses): + """Steps carrying at least one step-crossing penalty (~2.4e5).""" + return losses > np.percentile(losses, 1) + 1.2e5 + + +# --- 1. events ------------------------------------------------------------- + +lp, loc = [], [] +for i in range(args.draws): + key = jax.random.PRNGKey(i) + g = handlers.trace(handlers.seed( + handlers.substitute(orch.jax_model_guide, data=params), key) + ).get_trace(priors=orch.priors, data=batch) + m = handlers.trace(handlers.replay(handlers.seed(orch.jax_model, key), g) + ).get_trace(priors=orch.priors, data=batch) + site = m["binding_obs"] + fn = site["fn"] + lp.append(np.asarray(fn.log_prob(site["value"])).ravel()) # masked + loc.append(np.broadcast_to(np.asarray(fn.base_dist.loc), + site["value"].shape).ravel()) +lp, loc = np.array(lp), np.array(loc) +obs = np.asarray(batch.binding.theta_obs).ravel() +shape = batch.binding.theta_obs.shape +print("\n1. binding log-likelihood per draw:", + np.round(lp.sum(1)).astype(int).tolist()) +for j in np.where(lp.min(0) < -1e4)[0]: + idx = tuple(int(i) for i in np.unravel_index(j, shape)) + hit = lp[:, j] < -1e4 + print(f" obs {idx} theta_obs {obs[j]:.4f}: penalized in {hit.mean():.0%} " + f"of draws, predicted theta {np.round(loc[hit, j], 3)[:4]} " + f"(clean draws ~{np.median(loc[~hit, j]):.4f})") + +# --- 2. gradients ---------------------------------------------------------- + +u = svi.optim.get_params(state.optim_state) + + +def loss(p, key): + return svi.loss.loss(key, svi.constrain_fn(p), orch.jax_model, + orch.jax_model_guide, priors=orch.priors, data=batch) + + +vg = jax.jit(jax.value_and_grad(loss)) +L, G = [], [] +for i in range(4 * args.draws): + v, g = vg(u, jax.random.PRNGKey(1000 + i)) + L.append(float(v)) + G.append(g) +L = np.array(L) +ev = penalized(L) +print(f"\n2. {ev.mean():.0%} of {L.size} draws penalized; mean loss " + f"{L.mean():.4g}, median {np.median(L):.4g}") +rows = [] +for k in u: + g = np.stack([np.asarray(x[k]).ravel() for x in G]) + if ev.all() or not ev.any(): + continue + diff = np.abs(g[ev].mean(0) - g[~ev].mean(0)) + j = int(np.argmax(diff)) + rows.append((diff[j], k, j, g[:, j].mean(), + g[:, j].std() / np.sqrt(len(g)), + np.clip(g[:, j], -1, 1).mean(), + np.mean(np.abs(g) > 1))) +rows.sort(reverse=True) +print(f" {'parameter[element]':44s} {'raw mean':>10s} {'+/- se':>8s} " + f"{'clipped':>8s} array's |grad| > 1 (clipped)") +for _, k, j, raw, se, clipped, frac in rows[:10]: + print(f" {k + f'[{j}]':44s} {raw:10.3g} {se:8.2g} {clipped:8.3f} " + f"{frac:.0%} of elements x draws") + +# --- 3. resume with and without the clip ------------------------------------- + +for clip in (1.0, None): + _, st = ri.restore_svi_from_checkpoint(args.checkpoint, + init_params=init_params) + svi.optim = adam_optimizer(args.lr, clip) + + def step(s, idx): + return svi.update(s, priors=orch.priors, + data=orch.get_batch(data, idx)) + + scan = jax.jit(lambda s, idx: jax.lax.scan(step, s, idx)) + losses = [] + for _ in range(args.steps // 250): + st, lb = scan(st, jnp.asarray(ri.model.get_random_idx(num_batches=250))) + losses.append(np.asarray(lb)) + losses = np.concatenate(losses) + ev = penalized(losses) + w = 2000 + print(f"\n3. resume, clip {clip}, step size {args.lr:g}:") + print(" penalized share per 2000:", + [round(float(ev[a:a + w].mean()), 2) for a in range(0, ev.size, w)]) + print(" mean loss per 2000: ", + [int(losses[a:a + w].mean()) for a in range(0, ev.size, w)]) diff --git a/planning/studies/congression-calibration/diagnosis/diag_elbo.py b/planning/studies/congression-calibration/diagnosis/diag_elbo.py new file mode 100644 index 00000000..e0503a0e --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_elbo.py @@ -0,0 +1,42 @@ +""" +Does the ELBO (averaged over guide draws) prefer the mixture fit's endpoint +while the joint density at median latents prefers single's solution? That +would mean guide noise, passed through the mixture's convex logsumexp and +max-theta, is what drives the SVI drift. Run from this directory; GRID +selects the grid. +""" +import sys +sys.argv = [sys.argv[0]] +from diag_lam0 import * +from numpyro.infer import Trace_ELBO + +N_DRAWS = 64 +(d1, o1, s1, st1), (d2, o2, s2, st2) = load("run_0001"), load("run_0002") +p1, p2 = s1.get_params(st1), s2.get_params(st2) +start = {k: (p1[k] if k in p1 else p2[k]) for k in p2} +elbo = Trace_ELBO(num_particles=1) + + +def mean_loss(params, orch, svi, n=N_DRAWS): + f = jax.jit(lambda p, key: elbo.loss(key, p, orch.jax_model, svi.guide, + data=full_batch(orch), priors=orch.priors)) + v = np.array([float(f(params, jax.random.PRNGKey(i))) for i in range(n)]) + return v.mean(), v.std() / np.sqrt(n) + + +for tag, params in (("single's solution (+ mixture lambda)", start), + ("mixture's own endpoint", p2)): + m, se = mean_loss(params, o2, s2) + print(f"mixture -ELBO at {tag:38s} {m:12.1f} +/- {se:.1f}") +m, se = mean_loss(p1, o1, s1) +print(f"single -ELBO at {'single solution':38s} {m:12.1f} +/- {se:.1f}") + +# Same comparison with the guide's scales shrunk toward zero (near-point +# evaluation): if the ordering flips, guide noise is what favors the drift. +def shrink(p, f=1e-3): + return {k: (v * f if k.endswith("_scale") or k.endswith("_scales") else v) + for k, v in p.items()} +for tag, params in (("single's solution, scales x1e-3", shrink(start)), + ("mixture endpoint, scales x1e-3", shrink(p2))): + m, se = mean_loss(params, o2, s2, n=8) + print(f"mixture -ELBO at {tag:38s} {m:12.1f} +/- {se:.1f}") diff --git a/planning/studies/congression-calibration/diagnosis/diag_geno.py b/planning/studies/congression-calibration/diagnosis/diag_geno.py new file mode 100644 index 00000000..a9459fc5 --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_geno.py @@ -0,0 +1,28 @@ +""" +Per-genotype growth log-likelihood gained from the congressed classes in +grid run 0002 (lambda 0, mixture, matched prior), at the fit's median latents. +Writes geno_gain.csv. Run from this directory. +""" +import os +import sys +sys.argv = [sys.argv[0]] +from diag_lam0 import * +import pandas as pd + +(d2, o2, s2, st2) = load("run_0002") +lat2, _ = median_latents(o2, s2, st2) +off = dict(lat2); off["transformation_lam"] = jnp.array(1e-8) +on_lp, off_lp = per_genotype(o2, lat2, off) +drop = (on_lp - off_lp).reshape(-1, on_lp.shape[-1]).sum(axis=0) +labels = list(o2.growth_tm.tensor_dim_labels[o2.growth_tm.tensor_dim_names.index("genotype")]) +df = pd.DataFrame({"genotype": labels, "loglik_gain_from_congressed": drop}) +par = pd.read_csv(glob.glob(d2 + "/summary/*_params_dk_geno.csv")[0])[["genotype", "q0.5", "ref"]] +par.columns = ["genotype", "dk_fit", "dk_true"] +df = df.merge(par, on="genotype", how="left").sort_values("loglik_gain_from_congressed", ascending=False) +print("total gain", df.loglik_gain_from_congressed.sum().round(1)) +print("share of gain in top 10 / top 50 genotypes:", + (df.loglik_gain_from_congressed.head(10).sum() / df.loglik_gain_from_congressed.sum()).round(2), + (df.loglik_gain_from_congressed.head(50).sum() / df.loglik_gain_from_congressed.sum()).round(2)) +print(df.head(12).round(4).to_string(index=False)) +print("\nall genotypes: dk_fit - dk_true median %.4f" % (df.dk_fit - df.dk_true).median()) +df.to_csv("geno_gain.csv", index=False) diff --git a/planning/studies/congression-calibration/diagnosis/diag_gradient.py b/planning/studies/congression-calibration/diagnosis/diag_gradient.py new file mode 100644 index 00000000..2862fe0f --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_gradient.py @@ -0,0 +1,77 @@ +""" +Is the mixture's SVI gradient biased along the k/dk_geno direction? + +Build the mixture fit's guide parameters from the single fit's solution +(keeping the mixture fit's lambda parameters), then average the ELBO +gradient over many draws with no optimization. Also the deterministic +gradient of the mixture's joint log density at single's median latents. +Run from this directory; GRID selects the grid (default: the unmasked one). +""" +import sys +sys.argv = [sys.argv[0]] +from diag_lam0 import * +from numpyro.infer import Trace_ELBO +from numpyro.infer.util import log_density + +N_DRAWS = 64 + +(d1, o1, s1, st1), (d2, o2, s2, st2) = load("run_0001"), load("run_0002") +p1 = s1.get_params(st1) +p2 = s2.get_params(st2) + +shared = sorted(set(p1) & set(p2)) +only2 = sorted(set(p2) - set(p1)) +print("mixture-only params:", only2) +start = {k: (p1[k] if k in p1 else p2[k]) for k in p2} + +data = full_batch(o2) +elbo = Trace_ELBO(num_particles=1) + + +def mean_grad(params, orch, svi, n=N_DRAWS): + def loss(p, key): + return elbo.loss(key, p, orch.jax_model, svi.guide, + data=full_batch(orch), priors=orch.priors) + g = jax.jit(jax.grad(loss)) + grads = [g(params, jax.random.PRNGKey(i)) for i in range(n)] + mean = jax.tree_util.tree_map(lambda *x: jnp.mean(jnp.stack(x), 0), *grads) + se = jax.tree_util.tree_map(lambda *x: jnp.std(jnp.stack(x), 0) / np.sqrt(n), *grads) + return mean, se + + +def show(tag, grads, keys): + mean, se = grads + print(f"\n{tag}: mean d(loss)/d(param) +/- SE (SVI moves against the sign)") + for k in keys: + m = np.ravel(np.asarray(mean[k])); s = np.ravel(np.asarray(se[k])) + print(f" {k:40s} " + " ".join(f"{a:+.3g}+/-{b:.2g}" for a, b in zip(m, s))) + + +keys = [k for k in p2 if k.startswith("condition_growth_k")] + \ + [k for k in p2 if k.startswith("dk_geno") and np.size(p2[k]) <= 4] + only2 +show("mixture model, guide at SINGLE's solution", mean_grad(start, o2, s2), keys) +show("mixture model, guide at MIXTURE's own solution", mean_grad(p2, o2, s2), keys) +k1 = [k for k in keys if k in p1] +show("single model, guide at SINGLE's solution", mean_grad(p1, o1, s1), k1) + +# Deterministic check: gradient of the joint log density wrt the k latent. +lat1, _ = median_latents(o1, s1, st1) +lat = dict(lat1) +lat["transformation_lam"] = jnp.asarray(np.exp(float(p2["transformation_lam_loc"]))) + + +def joint(l, orch): + lp, _ = log_density(seed(orch.jax_model, 0), (), + dict(data=full_batch(orch), priors=orch.priors), l) + return lp + + +for tag, orch, l in (("mixture", o2, lat), ("single", o1, lat1)): + g = jax.grad(joint)(l, orch) + kname = [k for k in g if k.startswith("condition_growth_k")] + print(f"\n{tag} joint log density at single's median latents, d/d(latent):") + for k in kname: + print(f" {k:30s}", np.round(np.ravel(np.asarray(g[k])), 1)) + dk = np.asarray(g["dk_geno_offset"]) if "dk_geno_offset" in g else None + if dk is not None: + print(f" dk_geno_offset (sum over genotypes) {dk.sum():.1f}") diff --git a/planning/studies/congression-calibration/diagnosis/diag_lam0.py b/planning/studies/congression-calibration/diagnosis/diag_lam0.py new file mode 100644 index 00000000..87d4a2ff --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_lam0.py @@ -0,0 +1,119 @@ +""" +Diagnose why mixture fits with lambda ~ 0 do worse than single on the same data. + +Runs 0001 (single) and 0002 (mixture, matched prior, lambda -> ~0.004) were +fit to identical simulated data. Evaluate both models' log-likelihood and +predictions at both fits' posterior-median latents. +""" + +import glob +import os +import sys + +import jax +import jax.numpy as jnp +import numpy as np +from numpyro.handlers import seed, substitute, trace + +from tfscreen.tfmodel.configuration_io import read_configuration +from tfscreen.tfmodel.inference.run_inference import RunInference + +GRID = os.environ.get("GRID", os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "congression_calibration")) +RUNS = sys.argv[1:] or ["run_0001", "run_0002"] + + +def load(run): + d = glob.glob(os.path.join(GRID, run + "_*"))[0] + cwd = os.getcwd() + os.chdir(d) + try: + orch, init_params = read_configuration("tfs_configure_config.yaml") + ri = RunInference(orch, 0) + svi, state = ri.restore_svi_from_checkpoint( + "tfs_fit_model_checkpoint.pkl", init_params=init_params) + finally: + os.chdir(cwd) + return d, orch, svi, state + + +def full_batch(orch): + return orch.get_batch(orch.data, jnp.arange(orch.data.num_genotype)) + + +def median_latents(orch, svi, state, n=60): + params = svi.get_params(state) + data = full_batch(orch) + guide = substitute(svi.guide, data=params) + draws = {} + for i in range(n): + tr = trace(seed(guide, i)).get_trace(data=data, priors=orch.priors) + for name, site in tr.items(): + if site["type"] == "sample" and not name.startswith("_"): + draws.setdefault(name, []).append(np.asarray(site["value"])) + return {k: jnp.asarray(np.median(np.stack(v), axis=0)) for k, v in draws.items()}, params + + +def evaluate(orch, latents): + data = full_batch(orch) + model = substitute(seed(orch.jax_model, 0), data=latents) + tr = trace(model).get_trace(data=data, priors=orch.priors) + obs = {name: float(jnp.sum(site["fn"].log_prob(site["value"]) + * (site["mask"] if site.get("mask") is not None else 1.0))) + for name, site in tr.items() + if site["type"] == "sample" and site.get("is_observed", False)} + return obs, np.asarray(tr["growth_pred"]["value"]) + + +if __name__ == "__main__": + (d1, o1, s1, st1), (d2, o2, s2, st2) = load(RUNS[0]), load(RUNS[1]) + print("single run :", os.path.basename(d1)) + print("mixture run:", os.path.basename(d2)) + + lat1, _ = median_latents(o1, s1, st1) + lat2, p2 = median_latents(o2, s2, st2) + print("\nmixture guide lam params:", + {k: np.asarray(v).round(4).tolist() for k, v in p2.items() if k.startswith("transformation")}) + print("mixture median lam:", float(lat2["transformation_lam"])) + + only1 = sorted(set(lat1) - set(lat2)); only2 = sorted(set(lat2) - set(lat1)) + print("latents only in single:", only1, " only in mixture:", only2) + + rows = [] + obs, g_s_s = evaluate(o1, lat1) + rows.append(("single model @ single fit", obs)) + obs, g_m_m = evaluate(o2, lat2) + rows.append(("mixture model @ mixture fit", obs)) + + for lam in (float(lat2["transformation_lam"]), 1e-8): + l = dict(lat1); l["transformation_lam"] = jnp.array(lam) + obs, g = evaluate(o2, l) + rows.append((f"mixture model @ single fit, lam={lam:.3g}", obs)) + diff = g - g_s_s + print(f"\ngrowth_pred mixture@single-fit(lam={lam:.3g}) - single@single-fit: " + f"max|d| {np.nanmax(np.abs(diff)):.4g}, rms {np.sqrt(np.nanmean(diff**2)):.4g}") + + l = dict(lat2); l.pop("transformation_lam") + obs, _ = evaluate(o1, l) + rows.append(("single model @ mixture fit", obs)) + + print("\nobserved-site log-likelihoods (sum):") + sites = sorted({k for _, o in rows for k in o}) + print(f"{'':48s}" + "".join(f"{s:>18s}" for s in sites) + f"{'total':>18s}") + for name, o in rows: + print(f"{name:48s}" + "".join(f"{o.get(s, np.nan):18.1f}" for s in sites) + + f"{sum(o.values()):18.1f}") + + +def per_genotype(orch, latents, off_latents): + """Growth log-lik per genotype with and without the congressed classes.""" + data = full_batch(orch) + out = [] + for lat in (latents, off_latents): + tr = trace(substitute(seed(orch.jax_model, 0), data=lat)).get_trace( + data=data, priors=orch.priors) + site = tr["growth_obs"] + lp = site["fn"].log_prob(site["value"]) + if site.get("mask") is not None: + lp = lp * site["mask"] + out.append(np.asarray(lp)) + return out diff --git a/planning/studies/congression-calibration/diagnosis/diag_population.py b/planning/studies/congression-calibration/diagnosis/diag_population.py new file mode 100644 index 00000000..20963906 --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_population.py @@ -0,0 +1,29 @@ +""" +Is the population dk_geno / activity / theta the mixture looks co-residents +up in the same as each genotype's own value? Compare the component's focal +output with population[batch_idx] at the mixture fit's median latents. +Run from this directory; GRID selects the grid. +""" +import sys +sys.argv = [sys.argv[0]] +from diag_lam0 import * + +(d2, o2, s2, st2) = load("run_0002") +lat2, _ = median_latents(o2, s2, st2) +data = full_batch(o2) +tr = trace(substitute(seed(o2.jax_model, 0), data=lat2)).get_trace(data=data, priors=o2.priors) +print("deterministic sites:", [k for k, v in tr.items() if v["type"] == "deterministic"][:40]) + +from tfscreen.tfmodel.generative.registry import model_registry +comp = model_registry["dk_geno"][o2.settings["dk_geno"]] +def run_dk(): + return comp.define_model("dk_geno", data.growth, o2.priors.growth.dk_geno, + return_population=True) +focal, pop = substitute(seed(run_dk, 0), data=lat2)() +bi = np.asarray(data.growth.batch_idx) +f = np.ravel(np.asarray(focal)); p = np.asarray(pop) +print("focal dk_geno: shape", np.asarray(focal).shape, " population:", p.shape) +print("focal - population[batch_idx]: max|d| %.3g" % np.max(np.abs(f - p[bi]))) +print("focal mean %.4f population mean %.4f" % (f.mean(), p.mean())) +if "dk_geno" in tr: + print("traced dk_geno site mean %.4f" % float(np.mean(tr["dk_geno"]["value"]))) diff --git a/planning/studies/congression-calibration/diagnosis/diag_sites.py b/planning/studies/congression-calibration/diagnosis/diag_sites.py new file mode 100644 index 00000000..d3bf78ad --- /dev/null +++ b/planning/studies/congression-calibration/diagnosis/diag_sites.py @@ -0,0 +1,27 @@ +"""Per-site joint log density of the single model at the single and mixture fits.""" +import sys +sys.argv = [sys.argv[0]] +from diag_lam0 import * +import pandas as pd +from numpyro.infer.util import log_density + +(d1, o1, s1, st1), (d2, o2, s2, st2) = load("run_0001"), load("run_0002") +lat1, _ = median_latents(o1, s1, st1) +lat2, _ = median_latents(o2, s2, st2) +lat2.pop("transformation_lam") +rows = {} +for tag, lat in (("single_fit", lat1), ("mixture_fit", lat2)): + _, tr = log_density(seed(o1.jax_model, 0), (), dict(data=full_batch(o1), priors=o1.priors), lat) + for name, site in tr.items(): + if site["type"] != "sample": + continue + lp = site["fn"].log_prob(site["value"]) + if site.get("scale") is not None: + lp = lp * site["scale"] + if site.get("mask") is not None: + lp = jnp.where(site["mask"], lp, 0.0) + rows.setdefault(name, {})[tag] = float(jnp.sum(lp)) +df = pd.DataFrame(rows).T +df["diff"] = df.mixture_fit - df.single_fit +pd.set_option("display.width", 200) +print(df.reindex(df["diff"].abs().sort_values(ascending=False).index).head(12).round(1).to_string()) diff --git a/planning/studies/congression-calibration/grid.yaml b/planning/studies/congression-calibration/grid.yaml new file mode 100644 index 00000000..85d4521e --- /dev/null +++ b/planning/studies/congression-calibration/grid.yaml @@ -0,0 +1,70 @@ +# --------------------------------------------------------------------------- +# Congression calibration grid (step 3.5 of planning/congression-physics-plan.md) +# +# tfs-setup-sim-grid grid.yaml --out_prefix congression_calibration +# for d in congression_calibration/*/; do (cd "$d" && sbatch run.srun); done +# +# 3 lambda x 2 dk spreads x 3 seeds x 3 fits = 54 runs. See README.md. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__dkloc{{ dk_geno_hyper_loc }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}" + +output_file: run.srun + + +simulate: + + # --- Simulated congression rate ------------------------------------------ + # 0 = no congression (null arm); 0.357 = measured; 1.0 = stress. + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + - transformation_poisson_lambda: 0.357 + - transformation_poisson_lambda: 1.0 + + # --- dk_geno spread -------------------------------------------------------- + # dk_geno = shift - exp(Normal(loc, scale)), wt = 0. + # wide (loc -3.5): SD 0.018/min, 5th-95th percentile -0.049 to 0.007 + # (the example config) + # tight (loc -7.0): SD 0.0006/min, mean ~0 (the measured regime, dk SD + # ~0.001/min) + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + - dk_geno_hyper_loc: -7.0 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.001 + + # --- Simulation seed (also the fit seed) ----------------------------------- + + - name: seed + variants: + - seed: 1 + - seed: 2 + - seed: 3 + + +template: + + # --- Fit ------------------------------------------------------------------ + # single: no congression model (the paired baseline). + # mixture/matched: lambda prior centered on the simulated value (0.01 when + # the simulated value is 0), SD 0.05. + # mixture/measured: lambda prior 0.357 +/- 0.05 whatever the truth, so + # lambda = 0 and 1.0 test a misspecified prior. At lambda = 0.357 it is + # the same fit as matched (a determinism check). + + - name: fit + variants: + - fit: single + lam_prior: "off" + - fit: mixture + lam_prior: matched + - fit: mixture + lam_prior: measured diff --git a/planning/studies/congression-calibration/grid_anchor.yaml b/planning/studies/congression-calibration/grid_anchor.yaml new file mode 100644 index 00000000..4a833312 --- /dev/null +++ b/planning/studies/congression-calibration/grid_anchor.yaml @@ -0,0 +1,55 @@ +# --------------------------------------------------------------------------- +# Anchored follow-up to grid.yaml (congression calibration, step 3.5) +# +# tfs-setup-sim-grid grid_anchor.yaml --out_prefix congression_calibration_anchor +# for d in congression_calibration_anchor/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Tests the slide diagnosis (README "Third diagnosis"): the k/dk_geno slide +# is held only by wt's dk_geno pin, which the Student-t growth likelihood +# makes cheap to abandon, and the prefit's k prior never engaged (its +# Hessian scale hit the 0.1 ceiling). Pin k with a tight prefit prior +# (k_scale_ceiling 0.005), at binding_weight 1. Same simulations as grid.yaml +# runs 0001-0003 (lambda 0) and 0037-0039 (lambda 1.0): wide dk, seed 1. +# 2 lambda x 3 fits = 6 runs. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__dkloc{{ dk_geno_hyper_loc }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}__bw{{ binding_weight }}__kceil{{ k_scale_ceiling }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + - transformation_poisson_lambda: 1.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + + +template: + + - name: fit + variants: + - fit: single + lam_prior: "off" + - fit: mixture + lam_prior: matched + - fit: mixture + lam_prior: measured + + - name: weight + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/grid_baseline.yaml b/planning/studies/congression-calibration/grid_baseline.yaml new file mode 100644 index 00000000..99ccffdb --- /dev/null +++ b/planning/studies/congression-calibration/grid_baseline.yaml @@ -0,0 +1,53 @@ +# --------------------------------------------------------------------------- +# Baseline on the new convergence code and corrected library (congression +# calibration, step 3.5) +# +# tfs-setup-sim-grid grid_baseline.yaml --out_prefix congression_calibration_baseline +# for d in congression_calibration_baseline/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Checks that the noise-referenced stop rule (5bf5b78) ends runs cleanly on +# its own, and whether the converged anchored results (README "Resumed to +# convergence") hold on the corrected library enumeration (bff2c51). +# Anchored settings: binding_weight 1, prefit k_scale_ceiling 0.005. +# 2 lambda x 2 fits x 2 seeds = 8 runs. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + - transformation_poisson_lambda: 1.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + - seed: 2 + + +template: + + - name: fit + variants: + - fit: single + lam_prior: "off" + - fit: mixture + lam_prior: matched + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/grid_bw1.yaml b/planning/studies/congression-calibration/grid_bw1.yaml new file mode 100644 index 00000000..8f94715c --- /dev/null +++ b/planning/studies/congression-calibration/grid_bw1.yaml @@ -0,0 +1,53 @@ +# --------------------------------------------------------------------------- +# Binding-weight follow-up to grid.yaml (congression calibration, step 3.5) +# +# tfs-setup-sim-grid grid_bw1.yaml --out_prefix congression_calibration_bw1 +# for d in congression_calibration_bw1/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Tests the binding-weight diagnosis (README "Diagnosis, revised"): the +# default binding_weight (growth rows / binding rows, ~251 here) lets the +# mixture trade growth fit for binding fit through its congressed classes. +# Refit with binding_weight = 1. Same simulations as grid.yaml runs 0001-0003 +# (lambda 0) and 0037-0039 (lambda 1.0): wide dk, seed 1, all growth rows. +# 2 lambda x 3 fits = 6 runs. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__dkloc{{ dk_geno_hyper_loc }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}__bw{{ binding_weight }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + - transformation_poisson_lambda: 1.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + + +template: + + - name: fit + variants: + - fit: single + lam_prior: "off" + - fit: mixture + lam_prior: matched + - fit: mixture + lam_prior: measured + + - name: weight + variants: + - binding_weight: 1.0 diff --git a/planning/studies/congression-calibration/grid_controls.yaml b/planning/studies/congression-calibration/grid_controls.yaml new file mode 100644 index 00000000..1ec10518 --- /dev/null +++ b/planning/studies/congression-calibration/grid_controls.yaml @@ -0,0 +1,65 @@ +# --------------------------------------------------------------------------- +# Controlled fits for the lambda-0 mixture anomaly (congression calibration, +# step 3.5) +# +# tfs-setup-sim-grid grid_controls.yaml --out_prefix congression_calibration_controls +# for d in congression_calibration_controls/run_*/; do (cd "$d" && sbatch run.srun); done +# +# In every follow-up grid the lambda-0 mixture fit ends in a solution that +# leans on congressed classes of weight ~0.2%, with k ~0.08 too high and wt +# abandoned, while single does not. Two controls on the same simulation as +# grid.yaml runs 0001-0003 (lambda 0, wide dk, seed 1), with the anchored +# settings of grid_anchor.yaml (binding_weight 1, prefit k_scale_ceiling +# 0.005): +# +# mixture / fixed0 / svi lambda pinned at ~1e-6, so the mixture is single. +# Drift here means a bug in the mixture code path +# during training. +# mixture / matched / map MAP instead of SVI. The same solution here means a +# real mode of the model; single-like means an SVI +# artifact. +# single / off / map the MAP baseline for the MAP mixture fit. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}__{{ analysis_method }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + + +template: + + - name: fit + variants: + - fit: mixture + lam_prior: fixed0 + analysis_method: svi + - fit: mixture + lam_prior: matched + analysis_method: map + - fit: single + lam_prior: "off" + analysis_method: map + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/grid_crossrule.yaml b/planning/studies/congression-calibration/grid_crossrule.yaml new file mode 100644 index 00000000..a2b9f990 --- /dev/null +++ b/planning/studies/congression-calibration/grid_crossrule.yaml @@ -0,0 +1,73 @@ +# --------------------------------------------------------------------------- +# Cross-rule validation (congression physics plan, step 4) +# +# tfs-setup-sim-grid grid_crossrule.yaml --out_prefix congression_calibration_crossrule +# for d in congression_calibration_crossrule/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Simulate with the max or the homodimer theta rule; fit each simulation +# with `single` and with the mixture under each rule (matched lambda prior). +# Lambda 0.357 (the real library) and 1.0 x seeds 1-2; otherwise the +# settings of grid_baseline.yaml. 24 runs. The simulate-max / fit-max cells +# repeat runs from grid_measured.yaml and grid_baseline.yaml, so the grid +# stands on its own. +# +# Summaries: against `single` (fit_theta_rule na), and homodimer against +# max on the same simulation: +# +# tfs-summarize-calibration congression_calibration_crossrule \ +# --out_prefix calib/crossrule \ +# --baseline fit=single lam_prior=off fit_theta_rule=na +# tfs-summarize-calibration congression_calibration_crossrule \ +# --out_prefix calib/crossrule_vs_max --baseline fit_theta_rule=max +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__sim{{ congression_theta_rule }}__seed{{ seed }}__{{ fit }}_{{ fit_theta_rule }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.357 + - transformation_poisson_lambda: 1.0 + + - name: sim_rule + variants: + - congression_theta_rule: max + - congression_theta_rule: homodimer + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + - seed: 2 + + +template: + + # fit_theta_rule "na": `single` has no theta rule (run.srun ignores it). + - name: fit + variants: + - fit: single + lam_prior: "off" + fit_theta_rule: na + - fit: mixture + lam_prior: matched + fit_theta_rule: max + - fit: mixture + lam_prior: matched + fit_theta_rule: homodimer + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/grid_dkrule.yaml b/planning/studies/congression-calibration/grid_dkrule.yaml new file mode 100644 index 00000000..312855c7 --- /dev/null +++ b/planning/studies/congression-calibration/grid_dkrule.yaml @@ -0,0 +1,84 @@ +# --------------------------------------------------------------------------- +# dk rule sensitivity (congression physics plan, step 5) +# +# tfs-setup-sim-grid grid_dkrule.yaml --out_prefix congression_calibration_dkrule +# for d in congression_calibration_dkrule/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Simulate with the dilution or the min dk rule (the two ends of the soft-min +# family); fit each simulation with `single` and with the mixture under each +# rule (matched lambda prior). Theta rule homodimer on both sides (the +# default; step 4 found the theta rule does not matter here). +# +# Designs: the study's wide dk_geno spread (SD ~0.018/min, a worst case) at +# lambda 0.357 and 1.0, and a realistic tight spread (bench: 0.024 +/- 0.001) +# at lambda 0.357 only. x seeds 1-2 = 36 runs. +# +# Summaries: against `single` (fit_dk_rule na), and min against dilution on +# the same simulation: +# +# tfs-summarize-calibration congression_calibration_dkrule \ +# --out_prefix calib/dkrule \ +# --baseline fit=single lam_prior=off fit_dk_rule=na +# tfs-summarize-calibration congression_calibration_dkrule \ +# --out_prefix calib/dkrule_vs_dilution --baseline fit_dk_rule=dilution +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__dk{{ dk_geno_hyper_loc }}__sim{{ congression_dk_rule }}__seed{{ seed }}__{{ fit }}_{{ fit_dk_rule }}" + +output_file: run.srun + + +simulate: + + # lambda and dk spread move together: wide at 0.357 and 1.0, tight at 0.357. + - name: design + variants: + - transformation_poisson_lambda: 0.357 + dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + - transformation_poisson_lambda: 1.0 + dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + - transformation_poisson_lambda: 0.357 + dk_geno_hyper_loc: -7.0 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.001 + + - name: theta_rule + variants: + - congression_theta_rule: homodimer + + - name: sim_dk_rule + variants: + - congression_dk_rule: dilution + - congression_dk_rule: min + + - name: seed + variants: + - seed: 1 + - seed: 2 + + +template: + + # fit_dk_rule "na": `single` has no dk rule (run.srun ignores it). + - name: fit + variants: + - fit: single + lam_prior: "off" + fit_dk_rule: na + - fit: mixture + lam_prior: matched + fit_dk_rule: dilution + - fit: mixture + lam_prior: matched + fit_dk_rule: min + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/grid_masked.yaml b/planning/studies/congression-calibration/grid_masked.yaml new file mode 100644 index 00000000..a6678c3f --- /dev/null +++ b/planning/studies/congression-calibration/grid_masked.yaml @@ -0,0 +1,51 @@ +# --------------------------------------------------------------------------- +# Masked follow-up to grid.yaml (congression calibration, step 3.5) +# +# tfs-setup-sim-grid grid_masked.yaml --out_prefix congression_calibration_masked +# for d in congression_calibration_masked/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Tests the detection-floor diagnosis (README "Diagnosis"): growth rows with +# fewer than min_counts reads are dropped before fitting. Same simulations as +# grid.yaml runs 0001-0003 (lambda 0) and 0037-0039 (lambda 1.0): wide dk, +# seed 1. 2 lambda x 3 fits = 6 runs. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__dkloc{{ dk_geno_hyper_loc }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}__min{{ min_counts }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + - transformation_poisson_lambda: 1.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + + +template: + + - name: fit + variants: + - fit: single + lam_prior: "off" + - fit: mixture + lam_prior: matched + - fit: mixture + lam_prior: measured + + - name: mask + variants: + - min_counts: 5 diff --git a/planning/studies/congression-calibration/grid_measured.yaml b/planning/studies/congression-calibration/grid_measured.yaml new file mode 100644 index 00000000..5af577aa --- /dev/null +++ b/planning/studies/congression-calibration/grid_measured.yaml @@ -0,0 +1,50 @@ +# --------------------------------------------------------------------------- +# Measured-regime confirmation (congression calibration, step 3.5) +# +# tfs-setup-sim-grid grid_measured.yaml --out_prefix congression_calibration_measured +# for d in congression_calibration_measured/run_*/; do (cd "$d" && sbatch run.srun); done +# +# The real library's lambda (0.357), with the matched prior (0.357 +/- 0.05, +# as on real data), {single, mixture} x seeds 1-3. Same settings as +# grid_baseline.yaml. 6 runs. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.357 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + - seed: 2 + - seed: 3 + + +template: + + - name: fit + variants: + - fit: single + lam_prior: "off" + - fit: mixture + lam_prior: matched + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/grid_profile.yaml b/planning/studies/congression-calibration/grid_profile.yaml new file mode 100644 index 00000000..a3a5270b --- /dev/null +++ b/planning/studies/congression-calibration/grid_profile.yaml @@ -0,0 +1,59 @@ +# --------------------------------------------------------------------------- +# Lambda profile on the lambda-1 simulation (congression calibration, +# step 3.5) +# +# tfs-setup-sim-grid grid_profile.yaml --out_prefix congression_calibration_profile +# for d in congression_calibration_profile/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Converged mixture fits put lambda at 1.30 on data simulated at 1.0 (README +# "Resumed to convergence"). Hold lambda fixed (lam_prior "fixed", prior SD +# 0.1% of the value) at five values and fit everything else, on the same +# simulation as grid_baseline.yaml's lambda-1 seed-1 runs. The loss against +# lambda says whether the data prefer lambda > 1; theta error against lambda +# says whether theta is recovered at the true lambda. Anchored settings as in +# grid_baseline.yaml. 5 runs. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}{{ lam_fixed }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 1.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + + +template: + + - name: fit + variants: + - fit: mixture + lam_prior: fixed + + - name: lam_value + variants: + - lam_fixed: 0.6 + - lam_fixed: 0.8 + - lam_fixed: 1.0 + - lam_fixed: 1.2 + - lam_fixed: 1.4 + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/grid_seed3.yaml b/planning/studies/congression-calibration/grid_seed3.yaml new file mode 100644 index 00000000..8214bda5 --- /dev/null +++ b/planning/studies/congression-calibration/grid_seed3.yaml @@ -0,0 +1,48 @@ +# --------------------------------------------------------------------------- +# Third seed for grid_baseline.yaml (congression calibration, step 3.5) +# +# tfs-setup-sim-grid grid_seed3.yaml --out_prefix congression_calibration_seed3 +# for d in congression_calibration_seed3/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Lambda 0 and 1.0 x {single, mixture}, seed 3; otherwise identical to +# grid_baseline.yaml. 4 runs. +# --------------------------------------------------------------------------- + +base_config: simulate_config.yaml + +run_name: "lam{{ transformation_poisson_lambda }}__seed{{ seed }}__{{ fit }}_{{ lam_prior }}" + +output_file: run.srun + + +simulate: + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + - transformation_poisson_lambda: 1.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 3 + + +template: + + - name: fit + variants: + - fit: single + lam_prior: "off" + - fit: mixture + lam_prior: matched + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/congression-calibration/hill_params.csv b/planning/studies/congression-calibration/hill_params.csv new file mode 100644 index 00000000..a3568471 --- /dev/null +++ b/planning/studies/congression-calibration/hill_params.csv @@ -0,0 +1,5 @@ +genotype,log_hill_K,hill_n,theta_high,theta_low +wt,-4.1,2,0.01,0.99 +M42I,-2.2104779610300698,1.5156149571654287,0.5054105302329879,0.9538933267956679 +H74A,-5.872705439673401,1.2001278450518955,0.05,0.7115380202706272 +K84L,-5.755115159475512,1.0593068151944411,0.1,0.856776050040223 diff --git a/planning/studies/congression-calibration/inputs/hill_params.csv b/planning/studies/congression-calibration/inputs/hill_params.csv new file mode 100644 index 00000000..a3568471 --- /dev/null +++ b/planning/studies/congression-calibration/inputs/hill_params.csv @@ -0,0 +1,5 @@ +genotype,log_hill_K,hill_n,theta_high,theta_low +wt,-4.1,2,0.01,0.99 +M42I,-2.2104779610300698,1.5156149571654287,0.5054105302329879,0.9538933267956679 +H74A,-5.872705439673401,1.2001278450518955,0.05,0.7115380202706272 +K84L,-5.755115159475512,1.0593068151944411,0.1,0.856776050040223 diff --git a/planning/studies/congression-calibration/resume.srun b/planning/studies/congression-calibration/resume.srun new file mode 100644 index 00000000..48b9702f --- /dev/null +++ b/planning/studies/congression-calibration/resume.srun @@ -0,0 +1,70 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=cong_resume +#SBATCH --output=resume_%j.out +#SBATCH --error=resume_%j.err +#SBATCH --partition=gpu +#SBATCH --time=00-02:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --ntasks-per-node=1 + +# Resume one finished grid run to real convergence, in a copy of its +# directory (step 3.5 convergence check; see +# planning/studies/congression-calibration-convergence/README.md, route 1). +# +# sbatch resume.srun +# e.g. sbatch resume.srun congression_calibration_anchor run_0002 +# +# The copy goes to _resumed/. Resuming overwrites +# tfs_fit_model_checkpoint.pkl and tfs_fit_model_losses.txt, hence the copy. +# Written for fits from before the noise-referenced stop rule (5bf5b78), +# whose stop rule fired early; the current rule stops on its own. + +# Swap lines if on cluster (module load) or local (XLA_FLAGS to use CPU) +module load cuda/12.4.1 +#export XLA_FLAGS="--xla_force_host_platform_device_count=8" + +set -e + +GRID_DIR="${1%/}" +RUN_PREFIX="${2}" +if [[ -z "${GRID_DIR}" || -z "${RUN_PREFIX}" ]]; then + echo "Usage: sbatch resume.srun " + exit 1 +fi + +SRC=$(ls -d "${GRID_DIR}/${RUN_PREFIX}"_* | head -1) +DEST="${GRID_DIR}_resumed/$(basename "${SRC}")" +mkdir -p "${GRID_DIR}_resumed" +rsync -a --exclude '*.h5' --exclude 'summary/' "${SRC}/" "${DEST}/" +cd "${DEST}" + +SEED=$(python -c "import yaml; print(yaml.safe_load(open('tfs_sim_config.yaml'))['seed'])") +METHOD=$(grep -o '^ANALYSIS_METHOD="[a-z]*"' run.srun | cut -d'"' -f2) +METHOD=${METHOD:-svi} + +echo ">>> Resume fit (${METHOD}) from ${SRC}" +tfs-fit-model tfs_configure_config.yaml \ + --seed "${SEED}" \ + --analysis_method "${METHOD}" \ + --checkpoint_file tfs_fit_model_checkpoint.pkl + +echo ">>> Sample posterior" +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl + +echo ">>> Extract parameters" +tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Predict theta" +tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Predict growth" +tfs-predict-growth tfs_configure_config.yaml tfs_posterior.h5 \ + --num_marginal_samples=500 + +echo ">>> Summarize fit" +tfs-summarize-fit . + +echo ">>> Done" diff --git a/planning/studies/congression-calibration/run.srun b/planning/studies/congression-calibration/run.srun new file mode 100644 index 00000000..d7be6a8e --- /dev/null +++ b/planning/studies/congression-calibration/run.srun @@ -0,0 +1,171 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=cong_calib +#SBATCH --output=run.out +#SBATCH --error=run.err +#SBATCH --partition=gpu +#SBATCH --time=00-02:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --ntasks-per-node=1 + +# Congression calibration study, one run (rendered by tfs-setup-sim-grid from +# planning/studies/congression-calibration/run.srun). Submit from the run +# directory: sbatch run.srun + +# Swap lines if on cluster (module load) or local (XLA_FLAGS to use CPU) +module load cuda/12.4.1 +#export XLA_FLAGS="--xla_force_host_platform_device_count=8" + +set -e + +FIT="{{ fit }}" +LAM_PRIOR="{{ lam_prior }}" +LAM_PRIOR_SD=0.05 +LAM_MEASURED=0.357 +# Growth rows with fewer reads than this are dropped before configure (0 keeps +# everything). grid_masked.yaml sets it; grid.yaml leaves the default. +MIN_COUNTS={{ min_counts | default(0) }} +# Binding likelihood weight passed to tfs-configure-model (empty = its default, +# growth rows / binding rows). grid_bw1.yaml sets it to 1. +BINDING_WEIGHT="{{ binding_weight | default('') }}" +# Ceiling on the prefit's per-condition k prior scale (empty = its default, +# 0.1). grid_anchor.yaml sets it low to pin the k/dk_geno slide. +K_SCALE_CEILING="{{ k_scale_ceiling | default('') }}" +# tfs-fit-model --analysis_method (svi or map). grid_controls.yaml sets map. +ANALYSIS_METHOD="{{ analysis_method | default('svi') }}" +# Lambda value for lam_prior "fixed" (grid_profile.yaml). +LAM_FIXED="{{ lam_fixed | default('') }}" +# Congression theta rule for the fit (mixture only). Empty = the rule the +# simulation used (tfs_sim_config.yaml's congression_theta_rule); set it to +# fit with a different rule (cross-rule grids). +FIT_THETA_RULE="{{ fit_theta_rule | default('') }}" +# Congression dk rule (and softmin alpha) for the fit (mixture only). Empty = +# the simulation's (congression_dk_rule, congression_dk_alpha). +FIT_DK_RULE="{{ fit_dk_rule | default('') }}" +FIT_DK_ALPHA="{{ fit_dk_alpha | default('') }}" + +# The simulated lambda and the seed live in tfs_sim_config.yaml. +read -r SEED LAM_SIM SIM_THETA_RULE SIM_DK_RULE SIM_DK_ALPHA < <(python -c " +import yaml +cf = yaml.safe_load(open('tfs_sim_config.yaml')) +print(cf['seed'], cf.get('transformation_poisson_lambda') or 0.0, + cf.get('congression_theta_rule') or 'homodimer', + cf.get('congression_dk_rule') or 'dilution', + cf.get('congression_dk_alpha') or '')") +FIT_THETA_RULE=${FIT_THETA_RULE:-${SIM_THETA_RULE}} +if [[ -z "${FIT_DK_RULE}" ]]; then + FIT_DK_RULE=${SIM_DK_RULE} + FIT_DK_ALPHA=${SIM_DK_ALPHA} +fi + +# --------------------------------------------------------------------------- +# 1. Simulate +# --------------------------------------------------------------------------- +echo ">>> Simulate (lambda=${LAM_SIM}, seed=${SEED})" +tfs-simulate tfs_sim_config.yaml . --seed "${SEED}" + +GROWTH_CSV=tfs_sim_growth.csv +if [[ "${MIN_COUNTS}" -gt 0 ]]; then + GROWTH_CSV=tfs_sim_growth_min${MIN_COUNTS}.csv + python -c " +import pandas as pd +df = pd.read_csv('tfs_sim_growth.csv') +kept = df[df['counts'] >= ${MIN_COUNTS}] +kept.to_csv('${GROWTH_CSV}', index=False) +print(f'Dropped {len(df) - len(kept)} of {len(df)} growth rows with counts < ${MIN_COUNTS}')" +fi + +# --------------------------------------------------------------------------- +# 2. Configure +# --------------------------------------------------------------------------- +if [[ "${FIT}" == "single" ]]; then + TRANSFORMATION_ARGS=(--transformation_model single) +else + if [[ "${LAM_PRIOR}" == "matched" ]]; then + LAM_MEAN=$(python -c "print(max(${LAM_SIM}, 0.01))") + elif [[ "${LAM_PRIOR}" == "fixed" ]]; then + # lambda held at LAM_FIXED (prior SD 0.1% of the value). + LAM_MEAN=${LAM_FIXED} + LAM_PRIOR_SD=$(python -c "print(${LAM_FIXED} * 1e-3)") + elif [[ "${LAM_PRIOR}" == "fixed0" ]]; then + # lambda pinned essentially at zero: the mixture is then single. + LAM_MEAN=1e-6 + LAM_PRIOR_SD=1e-9 + else + LAM_MEAN=${LAM_MEASURED} + fi + TRANSFORMATION_ARGS=(--transformation_model mixture + --transformation_lambda "${LAM_MEAN}" "${LAM_PRIOR_SD}" + --congression_theta_rule "${FIT_THETA_RULE}" + --congression_dk_rule "${FIT_DK_RULE}") + if [[ -n "${FIT_DK_ALPHA}" ]]; then + TRANSFORMATION_ARGS+=(--congression_dk_alpha "${FIT_DK_ALPHA}") + fi +fi + +BINDING_WEIGHT_ARGS=() +if [[ -n "${BINDING_WEIGHT}" ]]; then + BINDING_WEIGHT_ARGS=(--binding_weight "${BINDING_WEIGHT}") +fi + +echo ">>> Configure (${FIT}, lambda prior ${LAM_PRIOR}, theta rule ${FIT_THETA_RULE}, dk rule ${FIT_DK_RULE}${FIT_DK_ALPHA:+ alpha ${FIT_DK_ALPHA}})" +tfs-configure-model \ + --binding_df tfs_sim_binding.csv \ + --growth_df "${GROWTH_CSV}" \ + --presplit_df tfs_sim_presplit.csv \ + --condition_growth_model linear \ + --growth_transition_model instant \ + --ln_cfu0_model hierarchical \ + --dk_geno_model hierarchical_geno \ + --activity_model fixed \ + --theta_model hill_mut \ + --theta_rescale_model passthrough \ + --theta_growth_noise_model logit_normal \ + --theta_binding_noise_model zero \ + --growth_noise_model normal_kt \ + --growth_likelihood lncfu \ + --sample_offset_model zero \ + --library_config tfs_sim_config.yaml \ + --growth_shares_replicates \ + --epistasis \ + "${TRANSFORMATION_ARGS[@]}" \ + "${BINDING_WEIGHT_ARGS[@]}" + +# --------------------------------------------------------------------------- +# 3. Pre-fit calibration, fit, posterior +# --------------------------------------------------------------------------- +PREFIT_ARGS=() +if [[ -n "${K_SCALE_CEILING}" ]]; then + PREFIT_ARGS=(--k_scale_ceiling "${K_SCALE_CEILING}") +fi + +echo ">>> Pre-fit calibration" +tfs-prefit-calibration tfs_configure_config.yaml --seed "${SEED}" "${PREFIT_ARGS[@]}" + +echo ">>> Fit model" +tfs-fit-model tfs_configure_config.yaml \ + --seed "${SEED}" \ + --analysis_method "${ANALYSIS_METHOD}" + +echo ">>> Sample posterior" +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl + +# --------------------------------------------------------------------------- +# 4. Extract, predict, summarize +# --------------------------------------------------------------------------- +echo ">>> Extract parameters" +tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Predict theta" +tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Predict growth" +tfs-predict-growth tfs_configure_config.yaml tfs_posterior.h5 \ + --num_marginal_samples=500 + +echo ">>> Summarize fit" +tfs-summarize-fit . + +echo ">>> Done" diff --git a/planning/studies/congression-calibration/simulate_config.yaml b/planning/studies/congression-calibration/simulate_config.yaml new file mode 100644 index 00000000..53893ce4 --- /dev/null +++ b/planning/studies/congression-calibration/simulate_config.yaml @@ -0,0 +1,277 @@ +# Base simulate config for the congression calibration study (step 3.5 of +# planning/congression-physics-plan.md). Copied from +# examples/simulate-and-analyze/simulate_config.yaml, with these changes: +# - library_mixture is the real design (single-1 100, single-2 100, +# double-1-2 1000, spiked 1), as in planning/studies/congression-estimator; +# - library_binding adds 20 in-library (bulk) binding anchors; +# - transformation_poisson_lambda and the dk_geno hyperparameters are set +# per run by grid.yaml; +# - from 2026-09-26, binding noise 0.025 (the experimental theta error; was +# 0.001) and a Hill coefficient limit of 4 (theta_sim_priors.max_hill_n). +# binding_data.spiked_binding.choose_by names hill_params.csv relative to this +# file; tfs-setup-sim-grid copies it into the grid's inputs/ directory. + + +# --------------------------------------------------------------------------- +# Library genetic information +# --------------------------------------------------------------------------- + +# Reading frame 0, 1, 2. No reverse (negative) strand allowed. +reading_frame: 0 + +# Amino acid residue number for the first in-frame residue of the amplicon. +first_amplicon_residue: 27 + +# Wildtype DNA sequence of the amplicon. +wt_seq: gccagccacgtttctgcgaaaacgcgggaaaaagtggaagcggcgatggcggagctgaattacattcccaaccgcgtggcacaacaactggcgggcaaacagtcgttgctgattggcgttgccacctccagtctggccctgcacgcgccgtcgcaaattgtcgcggcgattaaatctcgcgccgatcaactgggtgccagcgtggtggtgtcgatg + +# Degenerate codon pattern for library positions. '.' = wildtype; 'nnt' = NNT +# codon (encodes all 20 amino acids + stop with reduced synonymous redundancy). +# Must be the same length as wt_seq. +degen_sites: | + .............................................nnt.............................................................................................nnt...........................nnt.......................................... +# ......nntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnnt......nntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnntnnt + + +# tiles groups contiguous degenerate positions that were cloned together. +# '.' = wildtype; any other character defines a tile block. Blocks must +# be contiguous (......111... ok; ....11.1... not ok). +tiles: | + ......111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111......222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222 + +# Flanking sequences immediately upstream and downstream of the amplicon. +expected_5p: cgcgtggtgaaccag +expected_3p: gtagaacgaagcggc + +# Which genotype classes to include in the library. 'single-x' = all +# single-mutation variants in tile x. 'double-x-y' = all pairwise +# combinations between tiles x and y. 'double-x-x' = internal +# doubles within tile x. +tile_combos: ["single-1", "single-2", "double-1-2"] + +# Control sequences (e.g. known variants) added as spikes. Must be the same +# length as wt_seq. +spiked_seqs: +- ........................................................................................................................................................................................................................ # wt +- .............................................ata........................................................................................................................................................................ # m42i +- .............................................................................................................................................gcg........................................................................ # h74a +- ...........................................................................................................................................................................ctg.......................................... # k84l +- .............................................ata.............................................................................................gcg........................................................................ # m42i/h74a +- .............................................ata...........................................................................................................................ctg.......................................... # m42i/k84l +- .............................................................................................................................................gcg...........................ctg.......................................... # h74a/k84l +- .............................................ata.............................................................................................gcg...........................ctg.......................................... # m42i/h74a/k84l +- ........................................................................................................................................................................................cc.............................. # d88a + +# --------------------------------------------------------------------------- +# Phenotype calculation +# --------------------------------------------------------------------------- + +# Registered theta component name. Any key in model_registry["theta"] is +# valid except "_simple" (calibration-only). Examples: "hill_geno", "thermo.O2_C12_K5_U0_a.PK". +theta_component: "hill_mut" + +# Pairwise epistasis (regularized horseshoe; used when library has doubles): +theta_sim_priors: + epi_tau_scale: 0.1 # HalfCauchy scale for global sparsity τ + max_hill_n: 4.0 # physical upper limit on the Hill coefficient + +# Optional: path to structural/thermodynamic data file required by PnnC and +# PddG theta components. Omit (or set to null) for all other models. +# .h5 / .hdf5 → HDF5 structural ensemble (PnnC) +# .csv → per-mutation per-structure ddG prior means (PddG) +# thermo_data: null + +# Optional: overrides for the theta component's default hyperparameters. +# Any key returned by the component's get_hyperparameters() can be overridden. +# Omit this block to use all component defaults. +# theta_priors: +# theta_tf_total_M: 3.3e-7 + +# --------------------------------------------------------------------------- +# Conditions +# --------------------------------------------------------------------------- + +# Each block defines one set of growth conditions. The Cartesian product of +# titrant_conc × t_sel gives one sample per entry. condition_pre and +# condition_sel must also appear as keys in the 'growth' section below. +condition_blocks: + - library: "kanR" + titrant_name: "iptg" + titrant_conc: [0, 0.0001, 0.001, 0.003] + condition_pre: "kanR-kan" + t_pre: 30 + condition_sel: "kanR+kan" + t_sel: [160, 180, 200] + + - library: "kanR" + titrant_name: "iptg" + titrant_conc: [0.01, 0.03, 0.1, 1.0] + condition_pre: "kanR-kan" + t_pre: 30 + condition_sel: "kanR+kan" + t_sel: [145, 165, 180] + + - library: "kanR" + titrant_name: "iptg" + titrant_conc: [0, 1.0] + condition_pre: "kanR-kan" + t_pre: 30 + condition_sel: "kanR-kan" + t_sel: [160, 180, 200] + + - library: "pheS" + titrant_name: "iptg" + titrant_conc: [0, 0.0001, 0.001, 0.003, 0.01, 0.03, 0.1, 1.0] + condition_pre: "pheS-4CP" + t_pre: 30 + condition_sel: "pheS+4CP" + t_sel: [95, 110, 125] + + - library: "pheS" + titrant_name: "iptg" + titrant_conc: [0, 1.0] + condition_pre: "pheS-4CP" + t_pre: 30 + condition_sel: "pheS-4CP" + t_sel: [95, 110, 125] + +# --------------------------------------------------------------------------- +# Growth rate parameters +# --------------------------------------------------------------------------- + +# Linear mapping from operator occupancy (theta) to growth rate k for each +# condition: k = m * theta + b. Every condition_pre and condition_sel that +# appears in condition_blocks must have an entry here, and vice versa. +growth: + kanR+kan: {m: -0.009933, b: 0.010696} + kanR-kan: {m: 0.000808, b: 0.015366} + pheS+4CP: {m: 0.006226, b: 0.021437} + pheS-4CP: {m: -0.000344, b: 0.028558} + +# Per-genotype pleiotropic growth cost is sampled from a shifted lognormal: +# dk_geno = hyper_shift - exp(Normal(hyper_loc, hyper_scale)) +# Most variants are mildly deleterious; a short right tail (at most hyper_shift) +# allows beneficial mutations. Matches the hierarchical tfmodel inference prior. +dk_geno_hyper_loc: -3.5 +dk_geno_hyper_scale: 0.5 +dk_geno_hyper_shift: 0.02 +dk_geno_zero: false + +# Per-genotype TF activity (A) — a scalar that multiplies the theta contribution +# to growth: k = b + A * m * theta. The wild-type is fixed at activity_wt=1.0 +# by inference convention. Set activity_mut_scale > 0 to draw each mutant +# genotype's log(A) independently from Normal(log(activity_wt), activity_mut_scale). +# activity_mut_scale=0 (default) gives all genotypes identical activity, matching +# the 'fixed' activity inference component. +activity_wt: 1.0 +activity_mut_scale: 0.0 + +# --------------------------------------------------------------------------- +# Experimental simulation parameters +# --------------------------------------------------------------------------- + +# Number of transformants per tile combo. Keys must match the entries in +# tile_combos (e.g. 'single-1', 'double-1-2') plus 'spiked'. +transform_sizes: + single-1: 10_000 + single-2: 10_000 + double-1-2: 1_000_000 + spiked: 1000 + +# Mixing ratio of tile combos in the pooled library. Only the ratio matters. +library_mixture: + single-1: 100 + single-2: 100 + double-1-2: 1000 + spiked: 1 + +# Sigma of the log-normal used to draw initial genotype frequencies. 0 = flat +# distribution; 1–2 is realistic for a typical transformation. +lib_assembly_skew_sigma: 1.25 + +# Each bacterium receives a number of plasmids drawn from a zero-truncated +# Poisson with this lambda. Set to 0 or null for exactly one plasmid per cell. +transformation_poisson_lambda: 0 + +# How a cell carrying several plasmids gets its growth rate. The cell's theta +# and TF activity come from congression_theta_rule ('max': the highest-theta +# plasmid sets both, at each condition); its dk_geno from congression_dk_rule +# ('dilution': share-weighted mean; also 'softmin' with congression_dk_alpha, +# or 'min'). Growth is then computed once from these cell-level values. The +# study keeps 'max' so its recorded grids stay reproducible (the default is +# now 'homodimer'); grids set the rules they test. +congression_theta_rule: max +congression_dk_rule: dilution + +# Colony-forming units per mL at the start of pre-growth. +cfu0: 6.5e7 + +# Per-tube environmental growth-rate noise (hr⁻¹). Each physical tube gets its +# own delta_k ~ Normal(0, tube_noise_sigma); the kt contribution per tube is +# delta_k * (t_pre + t_sel), shared by all genotypes in that tube. A value of +# ~0.001–0.002 hr⁻¹ is typical for well-controlled liquid cultures. +# Set to 0 or null to disable. +tube_noise_sigma: 0.002 + +# --------------------------------------------------------------------------- +# Growth transition model (optional) +# --------------------------------------------------------------------------- + +# Models the lag phase when bacteria switch from pre-selection to selection medium. +# Every condition_pre in condition_blocks must have an entry. Use model: "instant" +# for conditions with no detectable lag. Omit this block to use instant everywhere. +# +# "memory" model: tau = tau0 + k1 / (theta + k2) +# tau0 = baseline lag (minutes) +# k1 = occupancy-dependence of lag (higher = more effect) +# k2 = curvature constant (> 0; prevents division by zero) +growth_transition: + - condition_pre: "pheS-4CP" + model: "instant" + + - condition_pre: "kanR-kan" + model: "instant" + +# --------------------------------------------------------------------------- +# Data collection parameters +# --------------------------------------------------------------------------- + +# Total sequencing reads across all samples. +total_num_reads: 100_000_000 + +# Fraction of reads re-assigned to a randomly chosen genotype (index hopping). +# Set to 0 or null to disable. +prob_index_hop: 0.00 + +# Random seed for reproducibility. Set to null for a random seed each run. +seed: 12345 + +# --------------------------------------------------------------------------- +# Binding data simulation (optional) +# --------------------------------------------------------------------------- + +# If present, tfs-simulate also writes a binding CSV with per-genotype theta +# curves. + +binding_data: + # The binding assay (shared by both genotype sets). + titrant_name: iptg + titrant_conc: [0, 0.0001, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0] + noise: 0.025 # experimental theta error (user, 2026-09-26) + + # Clean, monoclonal (congression-free) controls. Pool = spiked_seqs. + # choose_by: | stratified | random + # num: optional [1, num_spiked], default all (forbidden with a file) + spiked_binding: + choose_by: hill_params.csv + + # In-library controls, drawn from the bulk (congression-affected growth). + # Pool = library genotypes NOT in spiked_seqs. Omit to disable. + # choose_by: stratified | random | + # num: required for stratified/random (forbidden with a file) + library_binding: + num: 20 + choose_by: stratified + +presplit_data: + noise: 0.00 diff --git a/planning/studies/congression-estimator/README.md b/planning/studies/congression-estimator/README.md new file mode 100644 index 00000000..32ba0fbc --- /dev/null +++ b/planning/studies/congression-estimator/README.md @@ -0,0 +1,136 @@ +# Congression estimator study + +**Question.** In the observable-level congression mixture, how should the fit +evaluate a genotype's congressed-cell term `E_S exp(G_{g,S}(t))`, the average +over random co-resident sets S of the congressed cells' exponentiated +log-growth? And do co-residents need per-origin backgrounds? + +**Decision it fed.** Step 3.0 of +[`planning/congression-physics-plan.md`](../../congression-physics-plan.md): +the fit uses fixed Monte Carlo quadrature (K per-genotype co-resident sets, +stratified by plasmid count, drawn once), with a pooled co-resident +background. See the plan's "Fit design (step 3)". + +## How to run + +From a scratch directory (outputs are written to the working directory): + +```bash +python /path/to/tfscreen/planning/studies/congression-estimator/study.py +``` + +An optional argument replaces the simulate config. Runtime is about 2.5 min +and peak memory about 1.3 GB. + +Outputs: `study_summary.csv` (one row per dk scale x estimator x subset) and +`study_by_time.csv` (RMS per growth condition and selection time). Not +committed. + +## Inputs + +- `examples/simulate/simulate_config.yaml` (default) for the library, theta + prior, growth conditions and growth transition: 843 genotypes, thermo theta + prior, memory growth transition, 48 growth conditions, t_sel 95 to 200 min. +- Overridden in the script: `library_mixture` = the real design + (single-1 100, single-2 100, double-1-2 1000, spiked 1), + `transformation_poisson_lambda` = 0.357, seed 11. +- Cell rules as in `simulate/cell_rules.py`: max theta, activity from the max + plasmid, dilution dk. + +## Method + +For each bulk genotype g and growth condition, the observable relative to +`ln_cfu0` is + +``` +L_g = log[ w_clean exp(G_clean) + sum_o w_o E_{S~o} exp(G_{g,S}) ] +``` + +with co-residents S drawn as N ~ Poisson(lambda) | N >= 1 plasmids from +sub-library o's plasmid distribution (each sub-library is transformed +separately). "Exact" enumerates N = 1 over every genotype in the sub-library +and uses Monte Carlo for N >= 2 (16% of congressed cells). dk_geno deviations +are scaled by s; at s = 1 (the sim prior) dk_geno has SD 0.068/min, with 5th +to 95th percentiles -0.13 and 0.014/min. s = 0.015 corresponds to the bench +measurement for the current library (SD about 0.001/min). + +Estimators of each congressed term, plugged into the same mixture: + +| name | estimator | +|---|---| +| `rate_avg` | `exp(E_S[G])` | +| `cumulant2` | `exp(E_S[G] + Var_S[G] / 2)` | +| `n1_only` | exact, but every congressed cell has exactly one co-resident | +| `mcK` | K fixed co-resident sets per sub-library, shared by all genotypes | +| `pooled` | exact, but co-residents from the pooled bulk library | +| `current` | today's fit: one trajectory, theta -> `E[max(theta_g, M)]` over an unweighted all-genotype background, own dk (reference) | +| `none` | no congression (reference) | + +Error = approximate minus exact `L_g`, in natural-log `ln_cfu` units, over +genotype x condition cells within 7 ln units of wt ("detectable"). `mcK` +pools 10 independent draw sets. + +## Commit + +Produced on `abec95f`; reproduced exactly on `fc0f2b9` (no simulator changes +in between). + +## Results + +RMS error (detectable): + +| estimator | s = 0 | s = 0.015 | s = 0.03 | s = 0.1 | s = 0.3 | s = 1 | +|---|---|---|---|---|---|---| +| none | 0.041 | 0.053 | 0.097 | 0.287 | 0.758 | 1.507 | +| current | 0.019 | 0.041 | 0.093 | 0.293 | 0.770 | 1.524 | +| rate_avg | 0.006 | 0.006 | 0.008 | 0.036 | 0.206 | 1.086 | +| cumulant2 | 0.000 | 0.000 | 0.001 | 0.025 | 0.509 | 12.599 | +| n1_only | 0.003 | 0.004 | 0.010 | 0.030 | 0.080 | 0.127 | +| mc1 | 0.032 | 0.043 | 0.049 | 0.104 | 0.391 | 1.113 | +| mc2 | 0.018 | 0.028 | 0.038 | 0.075 | 0.207 | 0.515 | +| mc4 | 0.016 | 0.016 | 0.032 | 0.072 | 0.181 | 0.379 | +| mc8 | 0.013 | 0.017 | 0.020 | 0.037 | 0.136 | 0.360 | +| mc16 | 0.008 | 0.011 | 0.010 | 0.028 | 0.072 | 0.187 | +| mc32 | 0.005 | 0.006 | 0.010 | 0.019 | 0.064 | 0.142 | +| pooled | 0.002 | 0.002 | 0.002 | 0.004 | 0.008 | 0.024 | + +95th percentile |error| (detectable): + +| estimator | s = 0 | s = 0.015 | s = 0.03 | s = 0.1 | s = 0.3 | s = 1 | +|---|---|---|---|---|---|---| +| none | 0.099 | 0.105 | 0.121 | 0.376 | 1.671 | 3.882 | +| current | 0.048 | 0.064 | 0.108 | 0.414 | 1.707 | 3.900 | +| rate_avg | 0.014 | 0.016 | 0.018 | 0.057 | 0.460 | 2.418 | +| mc16 | 0.021 | 0.028 | 0.020 | 0.044 | 0.147 | 0.391 | +| mc32 | 0.013 | 0.013 | 0.020 | 0.034 | 0.113 | 0.296 | +| pooled | 0.002 | 0.002 | 0.003 | 0.006 | 0.016 | 0.053 | + +Mean error (detectable) of `mcK` is within ±0.01 at s <= 0.1 and within ±0.09 +at s = 1, for every K >= 2; `rate_avg` is biased low (-0.15 at s = 0.3, +-0.76 at s = 1). + +Error grows with selection time. At s = 0.3, `rate_avg` RMS rises from 0.20 +to 0.31 over kanR t_sel 145 to 200 min, and `mc16` from 0.07 to 0.09. + +## Conclusions + +- Once dk varies, the congression effect is large (`none`), and today's fit + rule removes almost none of it: averaging theta cannot represent a mixture + of growth rates. +- `rate_avg` is adequate only for a narrow dk spread and worsens with time; + `cumulant2` fails on heavy dk tails. +- `mcK` has near-zero mean error at every spread, with RMS falling roughly as + 1/sqrt(K). It is the only estimator that holds across the dk range. +- A pooled co-resident background is adequate. Caveat: in the sim prior, + singles and doubles have similar theta/dk distributions; real doubles may be + more often broken. +- At the measured spread for the current library (s ~ 0.015), the whole effect + is modest (0.053 RMS, 0.105 p95) and every estimator except small-K MC is + accurate. + +## Limits + +- One simulated library (843 genotypes, 2 sites) and one theta prior. +- This study shares draw sets across genotypes; the adopted fit design uses + per-genotype sets, which gives the same per-genotype error with errors + independent across genotypes. diff --git a/planning/studies/congression-estimator/study.py b/planning/studies/congression-estimator/study.py new file mode 100644 index 00000000..ef479e60 --- /dev/null +++ b/planning/studies/congression-estimator/study.py @@ -0,0 +1,340 @@ +""" +Step 3.0 of planning/congression-physics-plan.md: how should the fit evaluate +the congressed class's E_S exp(G_{g,S}(t))? + +Ground truth comes from the simulator's own machinery (library_prediction on +examples/simulate/simulate_config.yaml, with the real design's +library_mixture and lambda = 0.357). For each bulk genotype g and each +growth condition (titrant, condition_pre/sel, t_pre, t_sel), the observable +(relative to ln_cfu0) is + + L_g = log[ w_clean exp(G_clean) + sum_o w_o E_{S~o} exp(G_{g,S}) ] + +with the cell rules of simulate/cell_rules.py (max theta, activity from the +max plasmid, dilution dk) and the same growth pipeline as the simulator +(growth_rate_one_condition + growth transition). Co-residents S: N ~ +Poisson(lambda) | N >= 1 plasmids drawn from origin o's plasmid +distribution (each sub-library is transformed separately). + +"Exact": N = 1 enumerated exactly over the origin's genotypes; N >= 2 by +Monte Carlo (16% of congressed cells at lambda = 0.357). + +Estimators of each congressed term (plugged into the same mixture): + rate_avg exp(E_S[G]) + cumulant2 exp(E_S[G] + Var_S[G]/2) + n1_only exact, but every congressed cell has exactly one co-resident + mcK K fixed co-resident sets per origin, shared by all focal + genotypes (common random numbers, as a fit would use) + pooled exact, but co-residents from the pooled bulk library +References: + current today's fit: one trajectory, theta -> E[max(theta_g, M)] over + an unweighted all-genotype background (spiked-origin genotypes + uncorrected), own dk and activity + none no congression at all (G_clean) + +Errors are in ln_cfu units (natural log), approx - exact. +""" + +import sys +import numpy as np +import pandas as pd + +from tfscreen.util.io import read_yaml +from tfscreen.simulate.library_prediction import library_prediction +from tfscreen.simulate.thermo_to_growth import growth_rate_one_condition +from tfscreen.simulate.selection_experiment import _compute_kt_arrays + +LAM = 0.357 +REAL_MIXTURE = {"single-1": 100, "single-2": 100, "double-1-2": 1000, + "spiked": 1} +DK_SCALES = [1.0, 0.3, 0.1, 0.03, 0.015, 0.0] +MC_N2_DRAWS = 400 # N >= 2 draws per (focal, origin) for "exact" +K_LIST = [1, 2, 4, 8, 16, 32] +K_REPEATS = 10 +DETECT = -7.0 # detectable: L_g - L_wt >= DETECT +SEED = 11 + +rng = np.random.default_rng(SEED) + +# --------------------------------------------------------------------------- +# Ground truth phenotypes and design +# --------------------------------------------------------------------------- + +# Default input: the repository's example simulate config. +_REPO = __import__("pathlib").Path(__file__).resolve().parents[3] +cfg_path = (sys.argv[1] if len(sys.argv) > 1 + else str(_REPO / "examples" / "simulate" / "simulate_config.yaml")) +cf = read_yaml(cfg_path) +cf["transformation_poisson_lambda"] = LAM +cf["library_mixture"] = dict(REAL_MIXTURE) +cf["seed"] = SEED +library_df, phenotype_df, genotype_theta_df, parameters_df = library_prediction(cf)[:4] +growth_params = cf["growth"] +theta_rescale = cf.get("theta_rescale", "passthrough") +growth_transition = cf.get("growth_transition") + +genotypes = list(parameters_df["genotype"].astype(str)) +gidx = {g: i for i, g in enumerate(genotypes)} +G = len(genotypes) + +# theta per genotype per titrant point +th = genotype_theta_df.copy() +th["genotype"] = th["genotype"].astype(str) +titr = (th[["titrant_name", "titrant_conc"]].drop_duplicates() + .sort_values(["titrant_name", "titrant_conc"]).reset_index(drop=True)) +titr["t_idx"] = np.arange(len(titr)) +th = th.merge(titr, on=["titrant_name", "titrant_conc"]) +theta_gt = np.full((G, len(titr)), np.nan) +theta_gt[th["genotype"].map(gidx).to_numpy(), th["t_idx"].to_numpy()] = th["theta"].to_numpy() + +dk0 = parameters_df["dk_geno"].to_numpy(dtype=float) +act = parameters_df["activity"].to_numpy(dtype=float) + +# growth conditions +cond = (phenotype_df[phenotype_df["replicate"] == phenotype_df["replicate"].min()] + [["library", "titrant_name", "titrant_conc", "condition_pre", + "condition_sel", "t_pre", "t_sel"]] + .drop_duplicates().reset_index(drop=True)) +cond = cond.merge(titr, on=["titrant_name", "titrant_conc"]) +C = len(cond) +cond_t = cond["t_idx"].to_numpy() + +# per-origin plasmid distributions (design, no assembly skew) +lib = library_df.copy() +lib["genotype"] = lib["genotype"].astype(str) +lib["library_origin"] = lib["library_origin"].astype(str) +lib = lib[lib["genotype"].isin(gidx)] +origins = sorted(lib["library_origin"].unique()) +bulk_origins = [o for o in origins if o != "spiked"] +probs = {} +for o in origins: + sub = lib[lib["library_origin"] == o] + p = np.zeros(G) + np.add.at(p, sub["genotype"].map(gidx).to_numpy(), sub["weight"].to_numpy()) + probs[o] = p / p.sum() +mass = {o: REAL_MIXTURE[o] * probs[o] for o in origins} +total_mass = sum(mass.values()) +in_spiked = mass["spiked"] > 0 +pooled_bulk = sum(mass[o] for o in bulk_origins) +pooled_bulk = pooled_bulk / pooled_bulk.sum() + +p_cong = 1.0 - np.exp(-LAM) +# weight of each (genotype, bulk origin) congressed class, and clean +w_orig = {o: np.divide(mass[o], total_mass, out=np.zeros(G), where=total_mass > 0) * p_cong + for o in bulk_origins} +w_clean = 1.0 - sum(w_orig.values()) +f_g = np.divide(sum(mass[o] for o in bulk_origins), total_mass, + out=np.zeros(G), where=total_mass > 0) +present = total_mass > 0 + +# Poisson(lambda) | N >= 1 split into N = 1 and N >= 2 +p_n = np.array([np.exp(-LAM) * LAM**n / np.prod(np.arange(1, n + 1)) for n in range(0, 12)]) +p1 = p_n[1] / (1 - p_n[0]) +p_n2 = p_n[2:] / p_n[2:].sum() # distribution of N given N >= 2 +n2_values = np.arange(2, 12) + +# --------------------------------------------------------------------------- +# Growth +# --------------------------------------------------------------------------- + +def kt(theta_c, act_c, dk): + """theta_c, act_c: (..., C) over growth conditions; dk: (...). -> (..., C)""" + shape = theta_c.shape + out = np.empty(shape) + dk_b = np.broadcast_to(np.asarray(dk)[..., None], shape) + for c in range(C): + row = cond.iloc[c] + th_c = theta_c[..., c].ravel() + a_c = act_c[..., c].ravel() + d_c = dk_b[..., c].ravel() + k_pre = growth_rate_one_condition(row["condition_pre"], th_c, a_c, d_c, + growth_params, theta_rescale) + k_sel = growth_rate_one_condition(row["condition_sel"], th_c, a_c, d_c, + growth_params, theta_rescale) + n = len(th_c) + v = _compute_kt_arrays(k_pre, k_sel, np.full(n, float(row["t_pre"])), + np.full(n, float(row["t_sel"])), th_c, + np.full(n, row["condition_pre"], dtype=object), + growth_transition) + out[..., c] = v.reshape(shape[:-1]) + return out + + +def cell_G(focal, co_sets, dk): + """ + focal: (F,) genotype idx. co_sets: (F, D, Nmax) co-resident idx, -1 = none. + Returns G (F, D, C) for cells {focal} + co_sets under max theta / + activity-follows-theta / dilution dk. + """ + F, D, Nmax = co_sets.shape + slots = np.concatenate([np.broadcast_to(focal[:, None, None], (F, D, 1)), co_sets], axis=2) + valid = slots >= 0 + sl = np.where(valid, slots, 0) + th_s = theta_gt[sl][..., cond_t] # (F,D,P,C) + th_s = np.where(valid[..., None], th_s, -np.inf) + best = np.argmax(th_s, axis=2) # (F,D,C) + th_cell = np.take_along_axis(th_s, best[:, :, None, :], axis=2)[:, :, 0, :] + a_s = np.broadcast_to(act[sl][..., None], th_s.shape) + a_cell = np.take_along_axis(a_s, best[:, :, None, :], axis=2)[:, :, 0, :] + n_in = valid.sum(axis=2) + dk_cell = np.where(valid, dk[sl], 0.0).sum(axis=2) / n_in + return kt(th_cell, a_cell, dk_cell) + + +def logmeanexp(x, axis, w=None): + m = np.max(x, axis=axis, keepdims=True) + if w is None: + return (m + np.log(np.mean(np.exp(x - m), axis=axis, keepdims=True))).squeeze(axis) + return (m + np.log(np.sum(w * np.exp(x - m), axis=axis, keepdims=True))).squeeze(axis) + + +def draw_sets(p, F, D, n_values, n_probs, rng): + n = rng.choice(n_values, size=(F, D), p=n_probs) + nmax = int(n.max()) + co = rng.choice(G, size=(F, D, nmax), p=p) + co = np.where(np.arange(nmax)[None, None, :] < n[..., None], co, -1) + return co + + +def congressed_terms(focal, p, dk, rng, chunk=60): + parts = [_congressed_terms(focal[i:i + chunk], p, dk, rng) + for i in range(0, len(focal), chunk)] + return {k: np.concatenate([q[k] for q in parts]) for k in parts[0]} + + +def _congressed_terms(focal, p, dk, rng): + """ + For focal genotypes and one co-resident distribution p, return + dict of per-(focal, condition) log E exp(G) for exact / n1_only, and the + moments E[G], Var[G] (exact mixture of N=1 enumeration and N>=2 MC). + """ + F = len(focal) + support = np.where(p > 0)[0] + # N = 1: enumerate + co1 = np.broadcast_to(support[None, :, None], (F, len(support), 1)).copy() + G1 = cell_G(focal, co1, dk) # (F,S,C) + w1 = p[support][None, :, None] + log_e1 = logmeanexp(G1, 1, w1) + m1 = np.sum(w1 * G1, axis=1) + s1 = np.sum(w1 * G1**2, axis=1) + # N >= 2: Monte Carlo + co2 = draw_sets(p, F, MC_N2_DRAWS, n2_values, p_n2, rng) + G2 = cell_G(focal, co2, dk) + log_e2 = logmeanexp(G2, 1) + m2 = G2.mean(axis=1) + s2 = (G2**2).mean(axis=1) + exact = np.logaddexp(np.log(p1) + log_e1, np.log(1 - p1) + log_e2) + mean = p1 * m1 + (1 - p1) * m2 + var = p1 * s1 + (1 - p1) * s2 - mean**2 + return {"exact": exact, "n1_only": log_e1, "mean": mean, "var": var} + + +def mixture(clean_G, terms_by_origin, focal): + """log[w_clean e^Gclean + sum_o w_o e^{term_o}] for focal genotypes.""" + acc = np.log(w_clean[focal])[:, None] + clean_G + for o, term in terms_by_origin.items(): + wo = w_orig[o][focal] + with np.errstate(divide="ignore"): + acc = np.logaddexp(acc, np.log(wo)[:, None] + term) + return acc + + +# today's fit rule: E[max(theta_g, M)], M = max of Poisson(lambda) draws +# from the unweighted all-genotype theta distribution (per titrant point) +def current_theta(): + grid = np.linspace(0, 1, 2001) + out = theta_gt.copy() + for t in range(theta_gt.shape[1]): + pop = np.sort(theta_gt[present, t]) + F = np.searchsorted(pop, grid, side="right") / len(pop) + P_M_le = np.exp(LAM * (F - 1.0)) # includes N = 0 + tail = 1.0 - P_M_le # P(M > t) + cum = np.concatenate([[0], np.cumsum((tail[1:] + tail[:-1]) / 2 * np.diff(grid))]) + G_tail = cum[-1] - np.interp(theta_gt[:, t], grid, cum) + out[:, t] = theta_gt[:, t] + G_tail + return out + +theta_current = current_theta() + +# --------------------------------------------------------------------------- +# Run +# --------------------------------------------------------------------------- + +focal = np.where(present & (f_g > 0))[0] +wt = gidx["wt"] +rows = [] +by_time = [] +for s in DK_SCALES: + dk = s * dk0 + clean_G = kt(theta_gt[focal][:, cond_t], np.broadcast_to(act[focal][:, None], (len(focal), C)), + dk[focal]) + per_origin = {o: congressed_terms(focal, probs[o], dk, rng) for o in bulk_origins + if np.any(w_orig[o][focal] > 0)} + pooled = congressed_terms(focal, pooled_bulk, dk, rng) + + exact = mixture(clean_G, {o: t["exact"] for o, t in per_origin.items()}, focal) + est = { + "rate_avg": mixture(clean_G, {o: t["mean"] for o, t in per_origin.items()}, focal), + "cumulant2": mixture(clean_G, {o: t["mean"] + t["var"] / 2 for o, t in per_origin.items()}, focal), + "n1_only": mixture(clean_G, {o: t["n1_only"] for o, t in per_origin.items()}, focal), + "pooled": mixture(clean_G, {o: pooled["exact"] for o in per_origin}, focal), + "none": clean_G, + } + cur_theta = np.where(in_spiked[focal][:, None], theta_gt[focal], theta_current[focal]) + est["current"] = kt(cur_theta[:, cond_t], np.broadcast_to(act[focal][:, None], (len(focal), C)), + dk[focal]) + for K in K_LIST: + errs = [] + for r in range(K_REPEATS): + terms = {} + for o in per_origin: + co = draw_sets(probs[o], 1, K, np.arange(1, 12), + p_n[1:] / p_n[1:].sum(), rng) + co = np.broadcast_to(co, (len(focal),) + co.shape[1:]).copy() + terms[o] = logmeanexp(cell_G(focal, co, dk), 1) + errs.append(mixture(clean_G, terms, focal) - exact) + est[f"mc{K}"] = errs # list of repeats + + # detectable: relative to wt's exact observable in the same condition + wt_pos = np.where(focal == wt)[0] + ref = exact[wt_pos[0]] if len(wt_pos) else np.median(exact, axis=0) + detect = (exact - ref[None, :]) >= DETECT + + for name, val in est.items(): + reps = val if isinstance(val, list) else [val - exact] + for subset, mask in (("all", np.ones_like(detect)), ("detectable", detect)): + e = np.concatenate([r[mask] for r in reps]) + rows.append({"dk_scale": s, "estimator": name, "subset": subset, + "rms": np.sqrt(np.mean(e**2)), + "mean": np.mean(e), + "p95_abs": np.quantile(np.abs(e), 0.95), + "max_abs": np.max(np.abs(e)), + "n": e.size}) + for c in range(C): + e = np.concatenate([r[detect[:, c], c] for r in reps]) + if e.size: + by_time.append({"dk_scale": s, "estimator": name, + "library": cond.loc[c, "library"], + "t_sel": cond.loc[c, "t_sel"], + "rms": np.sqrt(np.mean(e**2))}) + + # size of the congression effect itself (exact vs none), detectable + print(f"dk_scale={s}: done", flush=True) + +res = pd.DataFrame(rows) +res.to_csv("study_summary.csv", index=False) +bt = (pd.DataFrame(by_time).groupby(["dk_scale", "estimator", "library", "t_sel"]) + ["rms"].mean().reset_index()) +bt.to_csv("study_by_time.csv", index=False) + +pd.set_option("display.width", 200) +print(f"\n{len(focal)} bulk genotypes x {C} conditions; lambda={LAM}; " + f"dk_geno SD (scale 1) = {np.std(dk0[present]):.4f}/min") +print(res[res.subset == "detectable"].pivot_table(index="estimator", columns="dk_scale", + values="rms").round(3).to_string()) +print("\np95 |error| (detectable)") +print(res[res.subset == "detectable"].pivot_table(index="estimator", columns="dk_scale", + values="p95_abs").round(3).to_string()) +print("\nmean error (detectable)") +print(res[res.subset == "detectable"].pivot_table(index="estimator", columns="dk_scale", + values="mean").round(3).to_string()) diff --git a/planning/studies/count-likelihood/.gitignore b/planning/studies/count-likelihood/.gitignore new file mode 100644 index 00000000..b8072a76 --- /dev/null +++ b/planning/studies/count-likelihood/.gitignore @@ -0,0 +1 @@ +count_likelihood* diff --git a/planning/studies/count-likelihood/README.md b/planning/studies/count-likelihood/README.md new file mode 100644 index 00000000..0a6ce1e6 --- /dev/null +++ b/planning/studies/count-likelihood/README.md @@ -0,0 +1,277 @@ +# Count-likelihood study + +**Question.** Does observing read counts directly (a negative binomial with +learned dispersion, no pseudocount; `growth_likelihood: counts`) fit better +and give better-calibrated theta than observing `ln_cfu`, both on data whose +counts are Poisson and on data with the real data's extra count noise? And, +at a true lambda of 0, does the congression mixture still find lambda +around 0.12 once the read-count floor is modeled? + +**Decisions it feeds.** +- Roadmap step 7 (`planning/analysis-roadmap.md`): whether the count + likelihood becomes the default for real data. +- Congression plan step 6 (`planning/congression-physics-plan.md`, on main): + whether the mixture's theta gain at a true lambda of 0 was the read-count + floor. That step also re-runs `grid_baseline.yaml` and `grid_dkrule.yaml` + on the count likelihood; this grid covers the lambda-0 question directly. + +## Design + +Base config: `../congression-calibration/simulate_config.yaml` (483 +genotypes, hill_mut theta, instant growth transitions, kanR and pheS, 20 +in-library binding anchors), at lambda 0 and the wide dk spread, with the +anchored fit settings of that study's baseline (binding weight 1, prefit +`k_scale_ceiling` 0.005). + +[`grid.yaml`](grid.yaml): 2 noise x 3 seeds x 4 fits = 24 runs. + +| axis | levels | +|---|---| +| simulated noise | `poisson` (the simulator's old behavior: Poisson counts); `realistic` (founder sampling and demographic growth with about 300 founders per genotype per tube, `cfu0` 150,000; shared transformation; 150,000 template molecules for about 830k reads per tube with amplification CV 0.5, so count variance about 8x Poisson, the middle of the 5-18x measured on real data in `../noise-anatomy/`) | +| seed | 1, 2, 3 | +| fit | `lncfu`/`zero` (today's fit: Student-t on `ln_cfu`, `normal_kt` growth noise); `lncfu`/`level` (plus a per-tube level offset); `counts`/`level` (the count likelihood; growth noise off, as it requires); `counts`/`level`/`mixture` with the measured lambda prior (0.357 +/- 0.05) | + +Each run ([`run.srun`](run.srun)) is the congression-calibration pipeline +with `--growth_likelihood` and `--sample_offset_model` from the fit arm. + +## How to run + +On the cluster, from a scratch directory, with this repository checked out: + +```bash +tfs-setup-sim-grid grid.yaml --out_prefix count_likelihood_v4 +for d in count_likelihood_v4/run_*/; do (cd "$d" && sbatch run.srun); done +``` + +When the runs finish: + +```bash +tfs-summarize-calibration count_likelihood_v4 --out_prefix calib/count_likelihood_v4 \ + --baseline likelihood=lncfu sample_offset=zero fit=single \ + --facet_by founder_sampling +``` + +## What to look at + +- theta (`theta_test`): coverage, calibration bias, width and RMSE by + `purity` x `has_binding`, paired against `lncfu`/`zero`, separately for + the two noise levels. Under `realistic` noise the `lncfu` fits treat 5-18x + too little variance as known, so their coverage should fall; the count + fit should hold it. +- `growth_phi` and `growth_inv_r`: near 0 under `poisson`; `phi` near the + simulated multiplier (about 8) under `realistic`. +- The `mixture` arm's `lam` against the true 0. +- Convergence (every run should stop on the noise-referenced rule, not the + epoch cap). + +## Inputs + +Repository paths only: `grid.yaml`, `run.srun`, +`../congression-calibration/simulate_config.yaml` and its +`hill_params.csv`. Seeds fixed in the grid. + +## Commit + +First run: commit 775f8b7 (2026-09-26/27), results in `count_likelihood/`. +Second run: commit c65390aa (2026-09-27), results in `count_likelihood_v2/`. +Third run: commit 740ebc19 (2026-09-27), results in `count_likelihood_v3/`. +Fourth run: 722393cf (2026-09-28, inferred from the file times of its +`run.out` files; the runs do not log a commit), results in +`count_likelihood_v4/`. None of the result directories are committed. + +## Results + +**Fourth run (`count_likelihood_v4`, 2026-09-28): `ln_cfu0: hierarchical`.** +All 24 runs finished. This is the first run after the SVI start fix +(`../relative-fit/`) and the ln_cfu0 fix (`../svi-overconfidence/`). Every +earlier run fit `hierarchical_factored`, which shares each genotype's +starting abundance across the separately grown kanR and pheS libraries. + +Pooled theta (test genotypes), mean over 3 seeds, 95% coverage / RMSE, with +the third run's numbers for comparison: + +| noise | fit | v4 | v3 | +|---|---|---|---| +| poisson | lncfu / zero | 0.72 / 0.100 | 0.76 / 0.101 | +| poisson | lncfu / level | 0.74 / 0.100 | 0.56 / 0.100 | +| poisson | counts / level / single | 0.83 / 0.035 | 0.65 / 0.041 | +| poisson | counts / level / mixture | 0.82 / 0.037 | 0.65 / 0.043 | +| realistic | lncfu / zero | 0.83 / 0.109 | 0.82 / 0.113 | +| realistic | lncfu / level | 0.76 / 0.109 | 0.69 / 0.110 | +| realistic | counts / level / single | 0.84 / 0.072 | 0.79 / 0.073 | +| realistic | counts / level / mixture | 0.84 / 0.073 | 0.78 / 0.074 | + +- **Counts now wins on coverage as well as accuracy.** Under Poisson noise + it covers 0.83 against 0.72 for `lncfu`/`zero`, at a third of the RMSE. + Its coverage gain over v3 came from the two fixes. RMSE barely moved. +- **dk_geno:** counts/single covers 0.86 (poisson) and 0.71 (realistic), + against 0.53-0.55 for `lncfu`/`zero`. +- **k and m are still overconfident in every arm:** `growth_m` 95% + coverage is 0-0.58. This is the guides' shared-parameter collapse + (`../svi-overconfidence/`). +- **Lambda at a true 0** is unchanged: 0.019-0.028 (poisson), 0.056-0.063 + (realistic), with intervals that exclude 0. + +**Third run (`count_likelihood_v3`, commit 740ebc19, 2026-09-27): the +answer.** All 24 runs finished; no NaN, no wrong optimum (run_0012's theta +RMSE is 0.035, was 0.51). 16 stopped on the convergence rule. 8 reached +the 100k-epoch cap, all poisson: the loss was flat at the floor step size +and one global noise parameter whose true value is 0 (`growth_inv_r` for +counts, `growth_noise` sigma for `lncfu`/`level`) kept sliding toward its +boundary just past the movement tolerance (excess 0.05-0.08) -- a benign, +honestly reported non-convergence. + +Pooled theta (test genotypes), mean over 3 seeds: + +| noise | fit | 95% coverage | 95% width | RMSE | +|---|---|---|---|---| +| poisson | lncfu / zero | 0.76 | 0.156 | 0.101 | +| poisson | lncfu / level | 0.56 | 0.073 | 0.100 | +| poisson | counts / level / single | 0.65 | 0.067 | 0.041 | +| poisson | counts / level / mixture | 0.65 | 0.067 | 0.043 | +| realistic | lncfu / zero | 0.82 | 0.207 | 0.113 | +| realistic | lncfu / level | 0.69 | 0.139 | 0.110 | +| realistic | counts / level / single | 0.79 | 0.152 | 0.073 | +| realistic | counts / level / mixture | 0.78 | 0.154 | 0.074 | + +- **Accuracy: counts wins everywhere.** Theta RMSE falls 60% (poisson) and + 35% (realistic); dk_geno RMSE 85% (poisson) and 40% (realistic); Hill K + and n improve too (log K RMSE 0.41 against 0.68 for `lncfu`/`zero`, + poisson). By stratum the gain holds for bulk, spiked and binding-anchored + genotypes; spiked genotypes gain the most under realistic noise (RMSE + 0.13 against 0.26, coverage 0.93 against 0.50). +- **Calibration: counts is about as overconfident as `lncfu`/`zero`, not + less.** Coverage 0.79 against 0.82 (realistic), 0.65 against 0.76 + (poisson): intervals shrink with the error but not as far. The shortfall + is concentrated where the data say least: `mixed`-purity genotypes (0.41 + poisson, 0.55 realistic) and theta near saturation (0.46 poisson, 0.69 + realistic). So the remaining overconfidence is the fit's (mean-field + guide, the k pin), not the likelihood's; `growth_k` coverage is ~0 in + every arm (95% width 1e-4 against RMSE 6e-4-1.3e-3). +- **Lambda at a true 0** (prior 0.357 +/- 0.05): 0.019-0.027 (poisson), + 0.055-0.064 (realistic), against ~0.12 for the `lncfu` mixture in the + congression study. Most of that spurious lambda was the read-count floor; + the rest remains, and its intervals (width 0.002-0.006) exclude 0. The + mixture costs nothing on theta against `single`. +- **Longer runs changed little.** Run for run against v2 (early cuts), + theta RMSE moved by at most 0.0015 and 95% coverage by at most 0.03 + (poisson counts runs lost ~0.025), except run_0012, which the early cut + had broken. The pooled check mattered for robustness, not for the + typical run. +- Dispersion and per-tube offsets as in v2: poisson phi ~0.05, realistic + phi ~28 (the realistic arm is ~29x Poisson, more than intended); + `sample_offset` SD ~0.36 absorbing the simulator's tube growth noise. + +**Second run (`count_likelihood_v2`, commit c65390aa, 2026-09-27): the +count likelihood is clearly more accurate; two failures remain, neither in +the likelihood itself.** 23 of 24 runs finished; 18 stopped on the +convergence rule, 5 at the epoch cap (3 `counts`, 2 `lncfu`/`level`). + +Pooled theta (test genotypes), mean over seeds: + +| noise | fit | 95% coverage | 95% width | RMSE | +|---|---|---|---|---| +| poisson | lncfu / zero | 0.76 | 0.157 | 0.101 | +| poisson | lncfu / level | 0.56 | 0.076 | 0.100 | +| poisson | counts / level / single (2 runs) | 0.70 | 0.068 | 0.037 | +| poisson | counts / level / mixture (seeds 1, 2) | 0.66 | 0.072 | 0.047 | +| realistic | lncfu / zero | 0.81 | 0.207 | 0.114 | +| realistic | lncfu / level | 0.68 | 0.140 | 0.110 | +| realistic | counts / level / single | 0.78 | 0.153 | 0.073 | +| realistic | counts / level / mixture | 0.78 | 0.155 | 0.074 | + +- **Accuracy.** Counts cut theta RMSE by a third (realistic) to two thirds + (poisson) against either `lncfu` arm, with coverage close to `lncfu`/ + `zero` and better than `lncfu`/`level`. dk_geno RMSE also falls + (poisson single: 0.0003 against 0.002). Every arm still under-covers + (0.66-0.81 at 95%): the overconfidence is not the likelihood's. +- **Lambda at a true 0** (prior 0.357 +/- 0.05): 0.023 and 0.027 + (poisson), 0.054-0.064 (realistic), against ~0.12 for the `lncfu` + mixture in the congression study. Counts removes most of the spurious + lambda but not all, and the intervals (width ~0.01) exclude 0. The + mixture arms lose nothing against `single` on theta. +- **Dispersion.** poisson: phi 0.03-0.06, inv_r 0.0014 (Poisson, as + simulated). realistic: phi 27-29, inv_r 0.007, i.e. ~29x Poisson, + well above the ~8x the arm was designed for (real data: 5-18x); the + realistic arm is noisier than intended. +- **`sample_offset` level SD ~0.36 in every count run**, poisson included: + the base config's `tube_noise_sigma` (0.002/min over ~230 min, ~0.4 ln + units) shifts each tube's genotypes and its supplied total together, and + the model's predictions carry no tube term; the offset absorbs it, as + intended until step 6. The same explains `lncfu`/`level` under-covering: + in `lncfu`/`zero` the `normal_kt` row noise soaked up this tube noise and + with it the understated `ln_cfu` noise; the structural offset removes that + cover and the intervals halve at the same RMSE. +- **`growth_k` coverage ~0 in every arm** (95% widths ~1e-4 against RMSE + ~1e-3): the pre-fit's k pin, not the likelihood. +- **Failure 1, `run_0007` (poisson, seed 2, counts, single): NaN at step + ~12,400.** Not the likelihood: hill_mut's horseshoe local scales for the + lowest-read doubles (200-800 reads over all tubes, mostly zeros under + selection; typical doubles 3,600-200,000) widened (guide LogNormal scale + up to 11.5 on 41 elements; every other run stays at ~1.5), because beyond + saturation theta, and so the likelihood, no longer changes, while the + half-Cauchy tail is heavy. A draw of ~e^90 overflows float32 and the + epistasis term, theta, growth and the likelihood go NaN. Reproduced by + replaying from the step-12,000 checkpoint. +- **Failure 2, `run_0012` (poisson, seed 3, counts, mixture): converged to + a wrong answer** (theta RMSE 0.51; m shrunk to ~40% and 4CP's m of the + wrong sign; ELBO 5.4e5 against 2.6e5 for the same-sized seed-2 mixture; + its `single` twin on the same data is fine). The step size was cut at + step 20,000 during a noisy but steady descent: window-median losses fell + 1.5e5-1.9e5 per window, but each window's own trend was t = 1.9, 3.0, + 1.0, so three windows counted as a plateau. It also showed some lambda + widening (scales to 4.9), perhaps from the same low-read doubles. +- **Both fixed on this branch (2026-09-27).** Horseshoe: overflow-safe + `regularized_scale` and `HalfCauchy` (`components/_horseshoe.py`); a + replay of run_0007 from its step-12,000 checkpoint, which went NaN at step + 12,370 before, ran 3,000 steps clean. Convergence: before a cut or stop, + the stalled windows are pooled into one trend (`pooled_loss_trend`). + Replaying every v2 run's loss trace up to its first cut, 20 of 23 first + cuts were made mid-descent (pooled t 4.4-14.2; run_0012: 14.2) and would + now continue; the other 3 (pooled t -0.5 to 1.4) are still cut. So the + early cut was the norm, not a one-off, and the v2 fits (and the earlier + congression grids) mostly finished their fast phase at too small a step. + A third run with both fixes is needed; runs will be longer. + +**First run (775f8b7): 11 of the 12 `counts` runs failed; not usable for +the comparison.** All 12 `lncfu` runs and one `counts` run +(`poisson`/seed 2/`mixture`) finished. + +- 8 `counts` runs crashed at the fit on a NaN `k_scale` written by the + pre-fit. Cause: `RunInference.compute_hessian_sigmas` took the constrained + MAP values it was given as unconstrained, so every positive site was + evaluated at `exp(value)`; the count likelihood's `growth_phi` (MAP 49 + from the calibration model's misfit, dk_geno fixed at 0) became e^49 and + overflowed. Reproduced from `run_0003`'s saved MAP; fixed. The same bug + loosened the `lncfu` arms' pre-fit `k_scale` (0.0028 written against + k sigmas of 0.0003-0.0007 at the right point, which the 0.002 floor + now overrides), so the `lncfu` arms need the rerun too. +- 3 `counts` runs (`poisson` seeds 2 and 3) went NaN within 250 SVI steps + after the pre-MAP. Cause: the negative binomial's concentration `mu / phi` + underflows float32 for genotypes predicted near extinction (all-zero + genotypes drift there), and the gradient goes NaN. Reproduced by + replaying `run_0007`'s pre-MAP hand-off (NaN by step 250); with the + concentration floor (`c + e^-30`) the same replay runs normally. +- The one finished `counts` run (true lambda 0, prior 0.357 +/- 0.05) + found lambda 0.027 (95% 0.026-0.028): far from the prior and well below + the ~0.12 the `lncfu` mixture found at true lambda 0 in the congression + study, as the floor hypothesis predicts; one run, and the interval is + overconfident. Its theta RMSE was 0.054 against 0.10 for the `lncfu` + arms at the same noise. +- `lncfu` arms, pooled theta 95% coverage (3 seeds each): `poisson` + 0.76 (`zero`) and 0.55 (`level`); `realistic` 0.82 (`zero`) and 0.68 + (`level`). The `level` offset narrows the theta intervals by about half + at the same RMSE, so it makes `lncfu` more overconfident, not less. + `growth_k` coverage is about 0 in every arm (intervals ~1e-4 wide, RMSE + 0.001): the pre-fit's k prior dominates. +- Counts runs start SVI at a much higher ELBO than `lncfu` (about 3e8 at + the pre-MAP point here): at 1e4-1e5 reads the likelihood is sharp and the + component guide's initial location scales (0.1) cost ~(0.1 mu)^2 / var per + observation. It falls as the scales shrink; a smaller + `--guide_init_scale` may suit counts fits if this slows convergence. + **Update (2026-09-27):** it did more than slow convergence. SVI lost the + pre-MAP point and settled into other optima (`../relative-fit/`, + Results), and every arm here, `lncfu` included, started 2e6-5e8 above + its pre-MAP. The default is now 1e-4 with no jitter. The comparisons + between arms ran under the same handicap. The absolute coverage numbers + above, `growth_k` included, need a rerun. diff --git a/planning/studies/count-likelihood/grid.yaml b/planning/studies/count-likelihood/grid.yaml new file mode 100644 index 00000000..6eb2dcd6 --- /dev/null +++ b/planning/studies/count-likelihood/grid.yaml @@ -0,0 +1,93 @@ +# --------------------------------------------------------------------------- +# Count-likelihood validation (roadmap step 7, planning/analysis-roadmap.md) +# +# tfs-setup-sim-grid grid.yaml --out_prefix count_likelihood +# for d in count_likelihood/run_*/; do (cd "$d" && sbatch run.srun); done +# +# Does fitting read counts (negative binomial, no pseudocount) beat fitting +# ln_cfu, on data with Poisson-only counts and on data with the real data's +# extra count noise? And at a true lambda of 0, does the congression mixture +# still find lambda ~0.12 once the read-count floor is modeled (the open +# question from the congression plan's step 3.5)? +# +# Simulated noise: +# poisson the simulator's old behavior: deterministic growth, one +# multinomial draw of reads (count variance = Poisson). +# realistic founder sampling and demographic growth with ~300 founders +# per genotype per tube (cfu0 150,000 over 483 genotypes, as in +# the real 215k-genotype library at 6.5e7 cells), one shared +# transformation, and a template bottleneck with PCR +# jackpotting (150,000 templates for ~830k reads per tube, CV +# 0.5): count variance ~8x Poisson, the middle of the 5-18x +# measured on real data (study noise-anatomy). +# +# Fits (all with the anchored settings of the congression 3.5 baseline): +# lncfu / zero today's production fit (Student-t on ln_cfu, normal_kt) +# lncfu / level the same with a per-tube level offset +# counts / level the count likelihood with a per-tube level offset +# counts / level / mixture (measured lambda prior): the floor question +# +# 2 noise x 3 seeds x 4 fits = 24 runs. +# --------------------------------------------------------------------------- + +base_config: ../congression-calibration/simulate_config.yaml + +run_name: "{{ 'realistic' if founder_sampling else 'poisson' }}__seed{{ seed }}__{{ likelihood }}_{{ sample_offset }}_{{ fit }}" + +output_file: run.srun + + +simulate: + + - name: noise + variants: + - founder_sampling: false + - founder_sampling: true + demographic_growth: true + shared_transformation: true + cfu0: 150_000.0 + pcr_template_molecules: 150_000 + pcr_amplification_cv: 0.5 + + - name: lam + variants: + - transformation_poisson_lambda: 0.0 + + - name: dk + variants: + - dk_geno_hyper_loc: -3.5 + dk_geno_hyper_scale: 0.5 + dk_geno_hyper_shift: 0.02 + + - name: seed + variants: + - seed: 1 + - seed: 2 + - seed: 3 + + +template: + + - name: fit + variants: + - likelihood: lncfu + sample_offset: zero + fit: single + lam_prior: "off" + - likelihood: lncfu + sample_offset: level + fit: single + lam_prior: "off" + - likelihood: counts + sample_offset: level + fit: single + lam_prior: "off" + - likelihood: counts + sample_offset: level + fit: mixture + lam_prior: measured + + - name: anchor + variants: + - binding_weight: 1.0 + k_scale_ceiling: 0.005 diff --git a/planning/studies/count-likelihood/run.srun b/planning/studies/count-likelihood/run.srun new file mode 100644 index 00000000..e825a1dd --- /dev/null +++ b/planning/studies/count-likelihood/run.srun @@ -0,0 +1,183 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=count_lik +#SBATCH --output=run.out +#SBATCH --error=run.err +#SBATCH --partition=gpu +#SBATCH --time=00-02:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --ntasks-per-node=1 + +# Count-likelihood study, one run (rendered by tfs-setup-sim-grid from +# planning/studies/count-likelihood/run.srun, which is the congression- +# calibration study's run.srun plus the likelihood and sample-offset arms). +# Submit from the run directory: sbatch run.srun + +# Swap lines if on cluster (module load) or local (XLA_FLAGS to use CPU) +module load cuda/12.4.1 +#export XLA_FLAGS="--xla_force_host_platform_device_count=8" + +set -e + +FIT="{{ fit }}" +# Growth likelihood (lncfu or counts) and per-tube offset (zero or level). +LIKELIHOOD="{{ likelihood }}" +SAMPLE_OFFSET="{{ sample_offset }}" +LAM_PRIOR="{{ lam_prior }}" +LAM_PRIOR_SD=0.05 +LAM_MEASURED=0.357 +# Growth rows with fewer reads than this are dropped before configure (0 keeps +# everything). grid_masked.yaml sets it; grid.yaml leaves the default. +MIN_COUNTS={{ min_counts | default(0) }} +# Binding likelihood weight passed to tfs-configure-model (empty = its default, +# growth rows / binding rows). grid_bw1.yaml sets it to 1. +BINDING_WEIGHT="{{ binding_weight | default('') }}" +# Ceiling on the prefit's per-condition k prior scale (empty = its default, +# 0.1). grid_anchor.yaml sets it low to pin the k/dk_geno slide. +K_SCALE_CEILING="{{ k_scale_ceiling | default('') }}" +# tfs-fit-model --analysis_method (svi or map). grid_controls.yaml sets map. +ANALYSIS_METHOD="{{ analysis_method | default('svi') }}" +# Lambda value for lam_prior "fixed" (grid_profile.yaml). +LAM_FIXED="{{ lam_fixed | default('') }}" +# Congression theta rule for the fit (mixture only). Empty = the rule the +# simulation used (tfs_sim_config.yaml's congression_theta_rule); set it to +# fit with a different rule (cross-rule grids). +FIT_THETA_RULE="{{ fit_theta_rule | default('') }}" +# Congression dk rule (and softmin alpha) for the fit (mixture only). Empty = +# the simulation's (congression_dk_rule, congression_dk_alpha). +FIT_DK_RULE="{{ fit_dk_rule | default('') }}" +FIT_DK_ALPHA="{{ fit_dk_alpha | default('') }}" + +# The simulated lambda and the seed live in tfs_sim_config.yaml. +read -r SEED LAM_SIM SIM_THETA_RULE SIM_DK_RULE SIM_DK_ALPHA < <(python -c " +import yaml +cf = yaml.safe_load(open('tfs_sim_config.yaml')) +print(cf['seed'], cf.get('transformation_poisson_lambda') or 0.0, + cf.get('congression_theta_rule') or 'homodimer', + cf.get('congression_dk_rule') or 'dilution', + cf.get('congression_dk_alpha') or '')") +FIT_THETA_RULE=${FIT_THETA_RULE:-${SIM_THETA_RULE}} +if [[ -z "${FIT_DK_RULE}" ]]; then + FIT_DK_RULE=${SIM_DK_RULE} + FIT_DK_ALPHA=${SIM_DK_ALPHA} +fi + +# --------------------------------------------------------------------------- +# 1. Simulate +# --------------------------------------------------------------------------- +echo ">>> Simulate (lambda=${LAM_SIM}, seed=${SEED})" +tfs-simulate tfs_sim_config.yaml . --seed "${SEED}" + +GROWTH_CSV=tfs_sim_growth.csv +if [[ "${MIN_COUNTS}" -gt 0 ]]; then + GROWTH_CSV=tfs_sim_growth_min${MIN_COUNTS}.csv + python -c " +import pandas as pd +df = pd.read_csv('tfs_sim_growth.csv') +kept = df[df['counts'] >= ${MIN_COUNTS}] +kept.to_csv('${GROWTH_CSV}', index=False) +print(f'Dropped {len(df) - len(kept)} of {len(df)} growth rows with counts < ${MIN_COUNTS}')" +fi + +# --------------------------------------------------------------------------- +# 2. Configure +# --------------------------------------------------------------------------- +if [[ "${FIT}" == "single" ]]; then + TRANSFORMATION_ARGS=(--transformation_model single) +else + if [[ "${LAM_PRIOR}" == "matched" ]]; then + LAM_MEAN=$(python -c "print(max(${LAM_SIM}, 0.01))") + elif [[ "${LAM_PRIOR}" == "fixed" ]]; then + # lambda held at LAM_FIXED (prior SD 0.1% of the value). + LAM_MEAN=${LAM_FIXED} + LAM_PRIOR_SD=$(python -c "print(${LAM_FIXED} * 1e-3)") + elif [[ "${LAM_PRIOR}" == "fixed0" ]]; then + # lambda pinned essentially at zero: the mixture is then single. + LAM_MEAN=1e-6 + LAM_PRIOR_SD=1e-9 + else + LAM_MEAN=${LAM_MEASURED} + fi + TRANSFORMATION_ARGS=(--transformation_model mixture + --transformation_lambda "${LAM_MEAN}" "${LAM_PRIOR_SD}" + --congression_theta_rule "${FIT_THETA_RULE}" + --congression_dk_rule "${FIT_DK_RULE}") + if [[ -n "${FIT_DK_ALPHA}" ]]; then + TRANSFORMATION_ARGS+=(--congression_dk_alpha "${FIT_DK_ALPHA}") + fi +fi + +# The count likelihood carries its own per-row dispersion, so it refuses the +# extra ln_cfu-scale growth noise the lncfu arms use. +if [[ "${LIKELIHOOD}" == "counts" ]]; then + GROWTH_NOISE=zero +else + GROWTH_NOISE=normal_kt +fi + +BINDING_WEIGHT_ARGS=() +if [[ -n "${BINDING_WEIGHT}" ]]; then + BINDING_WEIGHT_ARGS=(--binding_weight "${BINDING_WEIGHT}") +fi + +echo ">>> Configure (${LIKELIHOOD}, offset ${SAMPLE_OFFSET}, ${FIT}, lambda prior ${LAM_PRIOR}, theta rule ${FIT_THETA_RULE}, dk rule ${FIT_DK_RULE}${FIT_DK_ALPHA:+ alpha ${FIT_DK_ALPHA}})" +tfs-configure-model \ + --binding_df tfs_sim_binding.csv \ + --growth_df "${GROWTH_CSV}" \ + --presplit_df tfs_sim_presplit.csv \ + --condition_growth_model linear \ + --growth_transition_model instant \ + --ln_cfu0_model hierarchical \ + --dk_geno_model hierarchical_geno \ + --activity_model fixed \ + --theta_model hill_mut \ + --theta_rescale_model passthrough \ + --theta_growth_noise_model logit_normal \ + --theta_binding_noise_model zero \ + --growth_noise_model "${GROWTH_NOISE}" \ + --growth_likelihood "${LIKELIHOOD}" \ + --sample_offset_model "${SAMPLE_OFFSET}" \ + --library_config tfs_sim_config.yaml \ + --growth_shares_replicates \ + --epistasis \ + "${TRANSFORMATION_ARGS[@]}" \ + "${BINDING_WEIGHT_ARGS[@]}" + +# --------------------------------------------------------------------------- +# 3. Pre-fit calibration, fit, posterior +# --------------------------------------------------------------------------- +PREFIT_ARGS=() +if [[ -n "${K_SCALE_CEILING}" ]]; then + PREFIT_ARGS=(--k_scale_ceiling "${K_SCALE_CEILING}") +fi + +echo ">>> Pre-fit calibration" +tfs-prefit-calibration tfs_configure_config.yaml --seed "${SEED}" "${PREFIT_ARGS[@]}" + +echo ">>> Fit model" +tfs-fit-model tfs_configure_config.yaml \ + --seed "${SEED}" \ + --analysis_method "${ANALYSIS_METHOD}" + +echo ">>> Sample posterior" +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl + +# --------------------------------------------------------------------------- +# 4. Extract, predict, summarize +# --------------------------------------------------------------------------- +echo ">>> Extract parameters" +tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Predict theta" +tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Predict growth" +tfs-predict-growth tfs_configure_config.yaml tfs_posterior.h5 \ + --num_marginal_samples=500 + +echo ">>> Summarize fit" +tfs-summarize-fit . + +echo ">>> Done" diff --git a/planning/studies/fresh-install/README.md b/planning/studies/fresh-install/README.md new file mode 100644 index 00000000..f833c77c --- /dev/null +++ b/planning/studies/fresh-install/README.md @@ -0,0 +1,65 @@ +# Fresh install + +**Question.** Does a new lab member's install of tfscreen work: a clean +clone, a new virtual environment, `pip install -e ".[cuda12]"`, on a +cluster GPU node? + +**Decision fed.** `planning/release-0.5.0.md`, items 11 and 13: whether +0.5.0's install instructions and dependency floors are right before +Natalie's dry run. + +**How to run.** On the cluster, after pulling the commit that adds this +study and pushing it to GitHub (the job clones from GitHub, not from the +cluster checkout): + +```bash +sbatch --export=ALL,OUT=/gpfs/projects/harmslab/harms/studies/fresh-install/run1 /gpfs/home/harms/tfscreen/planning/studies/fresh-install/fresh_install.srun +``` + +`OUT` must not hold `tfscreen/` or `venv/` already. Set `PY` to a Python +3.12 or newer if `python3` on the node is older, and `REF` to test a tag. + +**What it checks, in order:** the clone at `REF`; the environment +(`installed.txt` is its `pip freeze`); that jax sees the GPU (the job +fails here if pip installed CPU-only jax); the unit tests +(`unit_tests.txt`); the quickstart example on the GPU +(`quickstart/run.log`, `quickstart/out/`). + +**Inputs.** None beyond the repository. + +**Pass.** jax on the GPU, the unit tests as on CI, and the quickstart +running to its summary. + +## Results + +### Local fresh install (2026-10-20) + +A new venv on the maintainer's laptop (macOS, Python 3.13.11), +`pip install -e ".[test]"` with no constraints, took the newest releases +(pandas 3.0.6, numpy 2.5.4, jax 0.12.0, scipy 1.18.1, numba 0.68.0). +The full unit suite passed (6,127 passed, 18 skipped, `pytest -n 4`, +8.7 min). Its `pip freeze` was briefly kept as +`constraints/tfscreen-0.5.0.txt`, then removed (user, 2026-10-20: no +top-level directory for it; the manuscript records its own versions). + +A dry-run resolve of that set with `jax[cuda12]` for Linux x86-64 +(`pip install --dry-run --platform manylinux... --only-binary=:all:`) +gives 83 packages for Python 3.12, with jax, jaxlib, jax-cuda12-plugin +and jax-cuda12-pjrt all at 0.12.0 (pybktree left out of the check: it +ships only a source package, which pip does not cross-resolve). For +Python 3.11 it fails: contourpy 1.4.0, at least, has dropped 3.11. + +### Cluster fresh install (2026-10-20) + +The user wiped and reinstalled by hand in a new environment on the +cluster: the install works. `fresh_install.srun` has not been run. + +### Python 3.12-3.14 (2026-10-20) + +CI's 3.11 jobs then failed at install: jax 0.11 and newer need Python +3.12, so the new jax floor has no 3.11 wheel. tfscreen now needs Python +3.12 or newer, and CI tests 3.12, 3.13 and 3.14. A dry-run resolve of the +fresh install's versions found wheels for every package on 3.12 and 3.14, +Linux x86-64 and macOS arm64. + + diff --git a/planning/studies/fresh-install/fresh_install.srun b/planning/studies/fresh-install/fresh_install.srun new file mode 100644 index 00000000..928c6583 --- /dev/null +++ b/planning/studies/fresh-install/fresh_install.srun @@ -0,0 +1,62 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_fresh +#SBATCH --partition=gpu +#SBATCH --time=00-06:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=64G +#SBATCH --ntasks-per-node=1 + +# A fresh install of tfscreen as a new lab member would make it, on a GPU +# node (planning/release-0.5.0.md item 11; README.md here). Clones the +# repository at REF into OUT, makes a new virtual environment there, +# installs with the GPU extra, checks that +# jax sees the GPU, runs the unit tests and the quickstart example. +# +# sbatch --export=ALL,OUT=/path/to/empty/dir fresh_install.srun +# +# OUT an empty directory (required; absolute) +# REPO repository URL (default https://github.com/harmsm/tfscreen) +# REF branch, tag or commit (default main) +# PY the Python that makes the environment (default python3; >= 3.12) + +module load cuda/12.4.1 +set -e +OUT="${OUT:?set OUT to an empty directory}" +REPO="${REPO:-https://github.com/harmsm/tfscreen}" +REF="${REF:-main}" +mkdir -p "${OUT}" && cd "${OUT}" +if [ -e tfscreen ] || [ -e venv ]; then + echo "OUT must not hold tfscreen/ or venv/ already" >&2; exit 1 +fi + +echo ">>> Clone ${REPO} ${REF} $(date)" +git clone -q "${REPO}" tfscreen +git -C tfscreen checkout -q "${REF}" +git -C tfscreen log --oneline -1 + +echo ">>> Environment $(date)" +PY="${PY:-python3}" +"${PY}" -c "import sys; print(sys.version); assert sys.version_info >= (3, 12), 'tfscreen needs Python 3.12 or newer; set PY'" +"${PY}" -m venv venv +source venv/bin/activate +pip install -q --upgrade pip +pip install -q -e "tfscreen[cuda12,test]" +pip freeze --exclude-editable > installed.txt +python -c "import jax; d = jax.devices(); print('jax', jax.__version__, d); assert d[0].platform == 'gpu', 'jax does not see the GPU'" + +echo ">>> Unit tests $(date)" +(cd tfscreen && NUMBA_DISABLE_JIT=1 pytest -q -n 8 tests/tfscreen) > unit_tests.txt 2>&1 || true +tail -3 unit_tests.txt + +echo ">>> Quickstart $(date)" +cp -r tfscreen/examples/simulate-and-analyze quickstart +cd quickstart +# run.sh forces 8 virtual CPU devices for laptops; on a GPU node drop it +sed -i 's/^export XLA_FLAGS=/#export XLA_FLAGS=/' run.sh +bash run.sh simulate_config.yaml out 1 > run.log 2>&1 +tail -5 run.log +ls out | head -40 +echo ">>> Done $(date)" diff --git a/planning/studies/full-size-sim/README.md b/planning/studies/full-size-sim/README.md new file mode 100644 index 00000000..a6b400c2 --- /dev/null +++ b/planning/studies/full-size-sim/README.md @@ -0,0 +1,367 @@ +# Full-size simulation of the dev-data design + +**Question.** Does the documented chain (simulate, process counts, +configure, staged MAP, arrowhead Laplace, extract, summarize) run on a +simulation the size and shape of the real screen, with no hand step, and +how well does the growth-only relative fit recover the truth there? + +**Decisions it feeds.** +- Pipeline plan step 6's last item and step 7 + (`planning/experiment-pipeline.md`): this is the full-size simulation + that runs through the same chain as the real data, with only the data + paths changed. +- The footing for the science prioritization (pipeline step 8) and for + `planning/offset-mode-growth-transition.md`: every later model study + starts from this design, on files a user can regenerate. + +## Design + +`make_sim_config.py` builds the simulate config from the real experiment; +its docstring says what is taken from the data and what is chosen. In +short: + +- **Design:** the real tube table, 118 tubes in 3 replicates, with the real + names and per-replicate selection times. `tfs-simulate` follows it + exactly through the `design` key (added for this study), so the + simulated tube table has the real layout. +- **Library:** the real genetics and `transform_sizes` (the nine-spike + config), about 218,000 genotypes. Since 2026-10-08 the `library_mixture` + is the realized one, estimated from the real pre-split composition + (`estimate_library_mixture`; doubles 0.54, single-1 0.20, single-2 0.25, + against the design's 0.83/0.08/0.08), and wt is repeated in the simulated + `spiked_seqs` until it is 11% of the pool, as in the real one. The excess + wt is simulation-only: the fit is configured with the real library config + (`library_config.yaml`), as the real data's fit is. +- **Growth:** linear, b and m per condition from the wt monoculture rates + (the monokan rule); control conditions m = 0. +- **Population:** `cfu0` per library, calibrated so the simulated tube + totals match the real ones on average (below); total reads equal to the + real experiment's (5.1e9, 4.3e7 per tube; runs before 2026-10-08 had 3 + times that, a simulator bug), with the real share of each library's reads + unassigned (`unassigned_read_fraction`: kanR 0.66, pheS 0.22); OD600 + through the real calibration, totals estimated from it. +- **Congression:** lambda 0.357, measured. +- **Chosen:** noise (`realistic`, the default, or `poisson`), phenotypes + (hill_geno prior draws, or `--phenotype_model` from `tfs-build-empirical`), + the growth transition (`instant`; `memory` as a test arm), dk_geno's + hyperparameters and `tube_noise_sigma`. + +The fit is the real data's: growth-only `hill_relative`, level tube offsets +held at SD 0.17, log(n) population SD held at 0.5, loose k/m priors +(`growth_priors_loose.csv`), MAP (staged automatically), arrowhead Laplace. + +## How to run + +A full-size simulation is large: about 85 minutes, 10 GB resident and a +68 GB peak memory footprint on a laptop for one seed (2026-10-05), and +several GB of files. Run it on the cluster, in `planning/dev-data/` +(gitignored, not pulled), after `prep_dev_data.py` has written +`processed/`. Copy the study files there (the staged-map study's +`score_map.py` too): + +```bash +cp /planning/studies/full-size-sim/{make_sim_config.py,run_pilot.srun,run_full_size.srun,growth_priors_loose.csv} /planning/studies/staged-map/score_map.py . +``` + +First a pilot at the first-guess `cfu0`, to calibrate it and to check the +read depth against the real screen (raw count files, no growth table): + +```bash +sbatch run_pilot.srun +``` + +It runs `make_sim_config.py --out_dir pilot_s1 --seed 1`, then +`tfs-simulate` with `--no_write_growth` inside `pilot_s1`. + +Then the calibrated config and the run (same seed, so the same phenotypes): + +```bash +python make_sim_config.py --out_dir sim_realistic_s1 --seed 1 --calibrate_from pilot_s1 +``` + +```bash +cd sim_realistic_s1 && sbatch ../run_full_size.srun +``` + +A Poisson-noise arm for comparison: + +```bash +python make_sim_config.py --out_dir sim_poisson_s1 --seed 1 --noise poisson --calibrate_from pilot_s1 +``` + +```bash +cd sim_poisson_s1 && sbatch ../run_full_size.srun +``` + +## Inputs + +`planning/dev-data/processed/`: `sample_df.csv` (the tube table), +`library_config.yaml`, `od600_calibration.yaml`, +`wt_control_rate_summary.csv`. Nothing outside `planning/dev-data`. + +## Commit + +The commit that adds the simulator's `design` key. Every step writes its +provenance (`*_provenance.json`). + +## Results + +Local pilot (2026-10-05, the uncommitted working tree, seed 1, first-guess +`cfu0` 1.5e7 per tube): it ran in 85 minutes (10 GB resident, 68 GB peak +footprint) and followed the 118-tube design. The simulated tubes were +about half as dense as the real ones (median OD600 kanR 0.16 against 0.22, +pheS 0.13 against 0.26), and six pheS tubes read below the detection +threshold. Calibrating the level per library from it gave `cfu0` x1.74 for +kanR and x3.58 for pheS (mean log real/simulated total 0.55 and 1.28). + +A per-library level leaves a structured residual (log real/simulated +total, SD 0.6-0.7 over tubes), which the calibration does not try to +remove, because it is the model question itself: + +- **By IPTG.** kanR+kan: about +1 at low IPTG and -0.3 at 1 mM; pheS+4CP: + +1 rising to +2.1 at high IPTG. The simulated growth difference across + IPTG (wt monoculture rates on the theta scale) is stronger than the real + library's, most of which is mutants. +- **Over time.** Within nearly every condition, controls included, real + totals fall behind the simulation by about 0.01-0.02 per minute: the + screen populations grow slower than the wt monoculture rates say. + +Both bear on `planning/offset-mode-growth-transition.md` (the tube offsets +carried an IPTG pattern) and on roadmap step 6 (the population model), and +go to the science prioritization (pipeline plan step 8). + +Cluster runs, first attempt (2026-10-06): both arms simulated and +processed, then stopped at configure. Seed 1 drew dk_geno -0.1 per minute +for the spike H74A/K84L. At that rate the spike falls about e^-12 behind by +the first sampled tube (30 minutes of pre-growth plus about 90 of +selection), so it has no reads, and configure refused a spike without +growth data. A spike can die in a simulation, so `run_full_size.srun` now +passes `--allow_missing_spikes`. It also skips the simulation and the +processing when their outputs exist. + +Rerun from configure (2026-10-06): both arms ran through the whole chain +with no hand step (staged MAP converged at every stage; arrowhead Laplace +held 45 and 55 shared directions, all k/m/dk_geno shift combinations), and +the tube-offset diagnostic found no structure (no condition trends with +IPTG or time; offset SD 1.8 to 1.9 times the held 0.17). The recovery +numbers are not usable: **the simulated wt did not respond to IPTG**. +hill_geno's simulation sampler drew wt's phenotype like any genotype's, +and seed 1 gave it "never binds" (theta 0.0063 at 0 to 0.0060 at 1 mM). +The relative fit gauges X on wt's own change, so the mapped X truth was +about 3,600 times too wide (theta test RMSE about 2,700), and the fit +itself was forced to put a wt response where there was none. +`thermo_to_growth` now gives wt the hill_geno reference curve +(`_pin_wt_reference`; an explicit wt override still wins). Every earlier +hill_geno simulation had a perturbed wt, 12% of seeds a non-normal +category. Rerun: new directories, same pilot calibration (wt is one +spike, a negligible share of the tube totals): + +```bash +python make_sim_config.py --out_dir sim_realistic_s1b --seed 1 --calibrate_from pilot_s1 +``` + +```bash +python make_sim_config.py --out_dir sim_poisson_s1b --seed 1 --noise poisson --calibrate_from pilot_s1 +``` + +Corrected arms (2026-10-06/07, commit 8399fb6b; `sim_realistic_s1b`, +`sim_poisson_s1b`): wt carries the reference curve (0.99 to 0.01, log K +-4.1, n 2). **The chain is clean**: simulate, process, configure, staged +MAP (every stage converged on the exact loss), arrowhead Laplace (9 and 11 +held shared directions), extract, predict and summarize, with no hand +step. **The offset diagnostic is clean** (smallest BH q 0.63 realistic, +0.31 Poisson), as it should be with no selection-onset transient in the +simulator. The offsets' SD (0.31, 0.34) is about twice the held 0.17: the +simulator's tube noise is a per-tube growth-rate shift +(`tube_noise_sigma` 0.002 per minute, so ~0.4 ln units over a selection), +larger than the real fit's offsets (SD 0.20) and not a level. + +Recovery, the baseline for pipeline plan step 8 (realistic / Poisson; +genotypes 124k / 186k with growth data): + +| | realistic | Poisson | +|---|---|---| +| X: Pearson r, RMSE, mean error | 0.34, 0.94, +0.06 | 0.76, 0.37, -0.21 | +| X: 95% coverage, median width | 0.66, 0.39 | 0.29, 0.15 | +| log K: r, 95% coverage | 0.28, 0.80 | 0.45, 0.82 | +| log n: r, 95% coverage | 0.43, 0.94 | 0.66, 0.94 | +| dk_geno: r, bias, 95% coverage | 0.69, -0.007, 0.66 | 0.79, -0.011, 0.23 | +| growth k, bias per minute | +0.006 | +0.022 | +| growth m, kanR+kan / pheS+4CP (truth -0.0138 / +0.0141) | -0.0192 / +0.0187 | -0.0168 / +0.0164 | +| k and m 95% coverage | 0 | 0 | + +X coverage by genotype reads (95%; realistic / Poisson): <=100 reads +1.00 / 0.93, 1e2-1e3 0.90 / 0.62, 1e3-1e4 0.81 / 0.24, 1e4-1e5 0.61 / +0.10, >1e5 0.20 / 0.04. Coverage falls with depth, and under Poisson +noise the X error is a near-constant -0.22 at every depth: a shared error, +not per-genotype noise. The shared parameters are off and carry no +interval. k slid up with dk_geno down (the k/dk_geno slide, +"Per-condition growth priors" in CLAUDE.md; the loose priors, k SD 0.01, +allow it), |m| came out 20-40% too large, and the arrowhead Laplace held +exactly those directions (k, m, dk_geno_hyper_shift) at the MAP, so k and +m have zero-width intervals and every deep genotype inherits their error. +The relative-fit study's run 4 (small library) covered X 0.97 / 0.94 with +the same route; at full size the shared error dominates. These go to step +8: anchoring k against dk_geno (tighter or measured k priors, base-growth +or wt monoculture data), the held Schur directions, and a simulator tube +noise that matches the real offsets. + +**What limits the correlation is depth, not k and m.** The realistic arm's +overall X r of 0.34 is set by shallow genotypes. Reads below are summed +over all 118 tubes (every condition, IPTG, time and replicate, both +libraries); 1,000 reads is about 8 per tube. By genotype-concentration +point, realistic arm: + +| total reads | r | RMSE | RMSE after the best affine map | SD of truth | +|---|---|---|---|---| +| <=100 | 0.11 | 2.64 | 0.43 | 0.43 | +| 1e2-1e3 | 0.30 | 1.26 | 0.41 | 0.43 | +| 1e3-1e4 | 0.73 | 0.33 | 0.30 | 0.43 | +| 1e4-1e5 | 0.96 | 0.14 | 0.13 | 0.43 | +| >1e5 | 0.99 | 0.11 | 0.06 | 0.43 | + +Deep genotypes rank well; their error is a shared rescaling (slope 1.2, +|m| too large), which the k/m work fixes and r does not see. Below 1e3 +reads the realistic noise leaves no information on X (best affine error = +the truth's spread), and the MAP points are wild (RMSE 1.3-2.6) because +the X population SDs are not held (only log n is; `hill_relative`'s +`theta_{X_low,X_delta,log_hill_K}_hyper_scale_fixed` would shrink them). +Their intervals cover (0.996), so they are honest but uninformative. +Above 1e3 reads the realistic arm's r is 0.87 (Spearman 0.82). Report +recovery by depth, not one all-genotype r. + +**The simulated depth distribution does not match the real screen.** +Genotypes with growth data by total reads (doubles / singles): + +| total reads | realistic sim | Poisson sim | real (recipe) | +|---|---|---|---| +| 1-100 | 10,948 / 25 | 17,870 / 17 | 43,713 / 0 | +| 1e2-1e3 | 18,616 / 36 | 30,618 / 33 | 95,717 / 0 | +| 1e3-1e4 | 30,460 / 90 | 48,356 / 82 | 60,542 / 7 | +| 1e4-1e5 | 39,771 / 159 | 59,435 / 145 | 15,634 / 40 | +| >1e5 | 23,630 / 608 | 28,474 / 644 | 1,387 / 903 | +| all (incl. wt, one triple) | 124,345 | 185,676 | 217,945 | + +Median total reads per double: 11,136 / 8,348 / 480 (94 / 71 / 4 per +tube); per single 441,080 / 590,799 / 867,829. The total reads match, but +the simulation concentrates them: many doubles deep, many below the +10-read minimum and dropped (124k of 218k survive under realistic noise), +where the real reads spread thinly over nearly every double (64% of real +doubles, 139,430, have fewer than 1,000 reads). The likely cause is the +simulated pool's unevenness (`lib_assembly_skew_sigma` 1.25 and the +design's `transform_sizes` bottleneck). So: + +- The simulation is not yet a stand-in for this screen. Calibrate its + library composition to the real depth distribution (this table is the + target) before quoting its recovery; every other result depends on it. +- Most real doubles sit where the realistic arm had no information on X. + If the real counts are as overdispersed as that arm, those curves are + not usable one genotype at a time (the noise model is calibrated to the + real overdispersion, planning/studies/noise-anatomy/, so this is a + warning, not a measurement). Pooling is the lever there: held X + population SDs, and mutation-level structure (`hill_mut`) that shares + information across doubles with the same single. + +**Depth calibration (2026-10-08).** Three causes of the depth mismatch, +found by comparing the real count files with the simulation's raw files +kind by kind, and fixed: + +1. A simulator bug gave every replicate the full `total_num_reads`, so + the runs above had 3 times the configured reads (fixed in + `simulate_cli.replicate_read_totals`). +2. No unassigned reads (real: kanR 66%, pheS 22%; the new + `unassigned_read_fraction`) and the design's pool composition instead + of the realized one (doubles 83% of the pool against 54%; wt 1% against + 11%). `make_sim_config.py` now takes both from the real files. +3. Too few clones per double and too harsh a dk_geno prior. The real + initial pool is very uneven among doubles (CV 5-7, 23-30% at zero out + of a mean of 20-50 reads) yet only 2.7% of real doubles end below 10 + reads, and abundance is shared between the two separately transformed + libraries (log-count r 0.69): a wide assembly skew (sigma 2.0 for + doubles, 0.8 for singles) with ~300 clones per double (6.7e7 + transformants), which the simulator now carries cheaply by merging + identical clones into weighted rows (`_sim_transform_collapsed`). The + old dk_geno prior put 31% of genotypes below -0.03 per minute (they died; + real well-measured genotypes span about -0.018 to -0.002); scale 0.15 + instead of 1.0. With slower-growing mutants the pool grows less, so the + cfu0 calibration rose about 14-fold (2.1e8 per library). + +Check (local, seed 1, `--calibrate_from` a pilot; 14 minutes, 6.5 GB peak; +every tube detectable): + +| doubles by total reads | real | simulated | +|---|---|---| +| <10 (dropped) | 5,887 | 15,327 | +| 10-100 | 43,759 | 37,987 | +| 1e2-1e3 | 96,201 | 89,233 | +| 1e3-1e4 | 60,549 | 64,362 | +| 1e4-1e5 | 15,634 | 15,283 | +| >1e5 | 1,387 | 1,225 | + +Median double 446 real, 472 simulated (3.8 and 4.0 reads per tube); median +single 868k and 623k (10th-90th percentile 166k-3.2M and 171k-2.2M). Read +shares by kind match within a few points, except wt: 4% (kanR) and 12% +(pheS) of real reads against 12% and 23% simulated. Real wt falls from 11% +of the initial pool to those shares, so it grows slower than the library +average in the screen; the simulated wt does not. Left: the dropout tail +(7% against 2.7%; a skew of 1.9 instead of 2.0 fitted the dropout better +in the composition fit) and the wt's growth. The tube totals' residual +after the cfu0 level is now mostly control against selection conditions +(control tubes -1.2 kanR, -0.7 pheS in log real/simulated). + +**Calibrated arms on the cluster (2026-10-08/09, `sim_realistic_s2`, +`sim_poisson_s2`, downloaded to the gitignored +`planning/studies/full-size-sim-3/`).** Both ran end to end; every MAP +stage converged on the exact loss; the arrowhead Laplace held 9 and 10 +shared directions; the tube offsets are unstructured (smallest BH q 0.45 +and 0.21; SD 0.32 and 0.35, about twice the held 0.17, from the +simulator's per-tube rate noise). The realistic arm's read depth +reproduces the local check exactly (same seed and code). The Poisson arm +is evener (median double 925 reads, 614 dropped), since it has no founder +or PCR noise. + +| | realistic | Poisson | +|---|---|---| +| X: Pearson r (all), r above 1e3 reads | 0.36, 0.94 | 0.94, 0.99 | +| X: 95% coverage (all), above 1e3 reads | 0.66, 0.50 | 0.29, 0.14 | +| log K: r, 95% coverage | 0.27, 0.81 | 0.59, 0.82 | +| log n: r, bias, 95% coverage | 0.23, +0.23, 0.96 | 0.61, +0.00, 0.92 | +| dk_geno: r, bias, 95% coverage | 0.25, -0.010, 0.67 | 0.72, -0.004, 0.39 | +| growth k bias per minute | +0.009 | +0.003 | +| growth m kanR+kan, pheS+4CP (truth -0.0138, +0.0141) | -0.0203, +0.0202 | -0.0182, +0.0164 | +| k and m 95% coverage | 0 | 0 (m 0.25) | + +X by total reads per genotype (realistic / Poisson): + +| reads | genotypes | r | RMSE after affine map | slope | bias | 95% coverage | +|---|---|---|---|---|---|---| +| <=100 | 37,707 / 17,526 | 0.26 / 0.72 | 0.42 / 0.30 | 0.05 / 0.60 | -0.07 / -0.12 | 0.97 / 0.87 | +| 1e2-1e3 | 89,022 / 97,548 | 0.62 / 0.96 | 0.34 / 0.12 | 0.49 / 1.10 | -0.09 / -0.12 | 0.82 / 0.51 | +| 1e3-1e4 | 64,362 / 89,313 | 0.93 / 0.99 | 0.16 / 0.06 | 1.19 / 1.16 | -0.10 / -0.12 | 0.56 / 0.16 | +| 1e4-1e5 | 15,325 / 17,129 | 0.99 / 0.99 | 0.07 / 0.06 | 1.36 / 1.17 | -0.10 / -0.12 | 0.29 / 0.02 | +| >1e5 | 2,135 / 1,869 | 0.99 / 0.99 | 0.06 / 0.06 | 1.37 / 1.17 | -0.09 / -0.12 | 0.08 / 0.01 | + +What this says, at realistic depth: + +- **Ranking above 1e3 reads is good in both arms** (r 0.93-0.99). Their + error is shared, not per genotype: X is compressed (slope 1.2-1.4; + |m| 30-45% too large) and shifted (-0.09 to -0.12), and k slid up + against dk_geno. The arrowhead Laplace holds exactly those directions, + so k and m have no interval and coverage falls with depth to near 0. + This is the step 8 item "anchor k and m, give the held directions an + interval"; fixing it should bring deep coverage up without changing r. +- **Realistic noise, not depth alone, is what costs the shallow + genotypes.** At 1e2-1e3 reads (89k-98k genotypes in both arms, the + bulk of the real doubles) Poisson counts give r 0.96 and the realistic + arm 0.62; at <=100 reads 0.72 against 0.26. The realistic arm's extra + variance is founder sampling and the PCR template bottleneck (calibrated + to the real counts' 5-18x overdispersion). For the next experiment's + design that points at the noise sources, not only at depth: with + templates at a tenth of the reads, the PCR bottleneck alone multiplies + the count variance about 13-fold, so more template DNA into PCR is + likely the larger lever than more reads. Not yet tested: a sweep of + `pcr_template_molecules` and `cfu0` on this simulation would say how + much each buys. +- log n is biased up (+0.23) under realistic noise only; dk_geno is + poorly recovered at depth-limited genotypes and undercovers in both. + diff --git a/planning/studies/full-size-sim/growth_priors_loose.csv b/planning/studies/full-size-sim/growth_priors_loose.csv new file mode 100644 index 00000000..4cfb4e48 --- /dev/null +++ b/planning/studies/full-size-sim/growth_priors_loose.csv @@ -0,0 +1,5 @@ +condition_rep,k_loc,k_scale,m_loc,m_scale +kanR+kan,0.015,0.01,0.0,0.01 +kanR-kan,0.015,0.01,0.0,0.01 +pheS+4CP,0.015,0.01,0.0,0.01 +pheS-4CP,0.015,0.01,0.0,0.01 diff --git a/planning/studies/full-size-sim/make_sim_config.py b/planning/studies/full-size-sim/make_sim_config.py new file mode 100644 index 00000000..04ab7e37 --- /dev/null +++ b/planning/studies/full-size-sim/make_sim_config.py @@ -0,0 +1,359 @@ +""" +Build a full-size simulate config that follows the dev-data screen's design. + + python make_sim_config.py [--out_dir full_size_sim] [--seed 1] + [--noise realistic|poisson] + [--phenotype_model ] + [--transition instant|memory] + [--calibrate_from ] + +Run from planning/dev-data (gitignored; the lab's data never enter the +repository, C10). Reads processed/ (written by prep_dev_data.py) and writes +/design.csv, /simulate_config.yaml and a copy of the OD600 +calibration, all inside planning/dev-data. + +What comes from the real experiment, and how: + +- design.csv: the real tube table (processed/sample_df.csv): every + sequenced tube with its real name, replicate, conditions, IPTG and + selection time. tfs-simulate follows it exactly (the ``design`` key), so + the simulated tube table has the real layout. +- Library: the genetics keys, transform_sizes and library_mixture of + processed/library_config.yaml (the nine-spike config). +- Growth (linear, rate = b + m * theta): for each selection condition the + wt monoculture rates (processed/wt_control_rate_summary.csv) at 0 and + 1 mM, with wt taken as theta 1 at 0 mM and 0 at 1 mM (the monokan rule): + b = rate at 1 mM, m = rate at 0 minus b. Control conditions: b = their + mean rate over IPTG, m = 0. +- cfu0 (cells per tube at -t_pre): first guess from the pre-split culture's + OD600 (0.35, diluted 0.2 into 15.2 mL; prep_dev_data.py) through the real + calibration, times the 5 mL tube. That guess left the simulated tubes about + half as dense as the real ones (pheS 0.13 against 0.26 median OD600; + 2026-10-05), so the cfu0 is calibrated per library from a pilot run + (--calibrate_from): with the phenotypes fixed by the seed, a tube's total + scales with cfu0, so each library's cfu0 is multiplied by the geometric + mean of real / simulated tube totals over its tubes. The script prints + what is left by condition; a per-library level cannot fix a spread that + comes from the growth rates. +- Reads: total_num_reads = the sum of the real tubes' called reads, spread + evenly over the tubes (the simulator does not vary depth per tube). + unassigned_read_fraction, per library, is the share of the real tubes' + reads in the __unknown__ row (kanR about 0.66, pheS 0.20: the kanR + library's structural variants), so a simulated genotype gets the reads a + real one does. +- Pool composition: library_mixture is the realized one, estimated from the + real pre-split (initial) composition with estimate_library_mixture and + averaged over the two libraries (they agree within ~0.05), not the + design's (which gives doubles 83% of the pool against about 54% + realized). wt is about 11% of the real pool, ~90 times what the spike + design gives it, for reasons not yet known (being worked on at the bench, + 2026-10-08). The simulation reproduces it by repeating the wt sequence in + the simulated spiked_seqs (a repeated sequence is degeneracy within the + spiked origin) and scaling the spiked origin's mixture and transformants + so the other spikes keep their share. This is simulation-only: the fit is + configured with the real library config (copied to library_config.yaml), + as the real data's fit is. +- OD600: the real calibration, totals estimated from OD600 as the lab does + (sample_cfu_from_od600). +- Congression: transformation_poisson_lambda 0.357, the measured value. +- Pool evenness (2026-10-08), fitted to the real initial composition and + the screen's dropout: doubles are very uneven before any selection (CV 5 + to 7 in the initial sample, 23-30% at zero reads out of a mean of 20-50), + yet only 2.7% of real doubles end below 10 reads over the screen, and + their abundance is shared between the separately transformed libraries + (log-count correlation 0.69). That is a wide assembly skew with many + clones per genotype: doubles skew sigma 2.0 and ~300 clones per double + (transform_sizes double-1-2 = 300 x its genotypes, about 6.7e7); singles + are even (CV 0.8-1.2), skew 0.8. The simulator merges identical clones + into weighted rows, so this costs one row per genotype, and holds the + congressed cells to max_congressed_cells (2e6). +- dk_geno: the real fit's well-measured genotypes (singles, doubles above + 1e4 reads) spread over about 0.016 per minute (1st-99th percentile), with + none below -0.03; the old hyperparameters (scale 1.0) put 31% of + genotypes below -0.03 and killed them. Scale 0.15 keeps the median + (-0.010) and gives a 1st-99th percentile range of about -0.023 to -0.001. + +Chosen, not measured: the noise model (``realistic``: founder sampling, +demographic growth, one shared transformation, PCR templates at assigned +reads per tube / 10 with CV 0.5, the setting that matched the real counts' 5-18x +overdispersion, planning/studies/noise-anatomy/; or ``poisson``), the +phenotypes (hill_geno prior draws, or resampled from a tfs-build-empirical +model with --phenotype_model), dk_geno's location and shift (the example +config's; its scale is fitted, above), tube_noise_sigma (0.002 per minute) and the growth transition +(instant; ``memory`` with the example config's parameters as a test arm). +""" + +import argparse +import glob +import os +import shutil + +import numpy as np +import pandas as pd +import yaml + +from tfscreen.genetics import UNKNOWN_GENOTYPE +from tfscreen.genetics.library_design import ( + estimate_library_mixture, + library_composition_table, +) +from tfscreen.genetics.library_manager import LibraryManager +from tfscreen.process_raw.od600 import od600_to_cfu_per_mL +from tfscreen.simulate.build_sample_dataframes import read_design + +PROCESSED = "processed" +TUBE_VOLUME_ML = 5.0 +PRESPLIT_OD600 = 0.35 +PRESPLIT_DILUTION = 0.2 / 15.2 +LAMBDA = 0.357 +ASSEMBLY_SKEW = {"double-1-2": 2.0, "single-1": 0.8, "single-2": 0.8, + "spiked": 0.5} +CLONES_PER_DOUBLE = 300 +MAX_CONGRESSED_CELLS = 2_000_000 +DK_HYPER_SCALE = 0.15 + +GENETICS_KEYS = ("reading_frame", "first_amplicon_residue", "wt_seq", + "degen_sites", "tiles", "expected_5p", "expected_3p", + "tile_combos", "spiked_seqs", "transform_sizes", + "library_mixture") + + +def growth_block(rates): + """Linear growth (b, m) per condition from the wt monoculture rates.""" + out = {} + for cond, g in rates.groupby("condition_sel"): + r = g.set_index("titrant_conc")["rate_mean"] + if "+" in cond: + b = float(r.loc[1.0]) + m = float(r.loc[0.0]) - b + else: + b, m = float(r.mean()), 0.0 + out[cond] = {"b": round(b, 6), "m": round(m, 6)} + return out + + +def unassigned_fractions(tubes): + """Share of each library's reads in the real count files' __unknown__ row.""" + out = {} + for lib, g in tubes.groupby("library"): + unknown = total = 0 + for name in g["sample"]: + path = os.path.join(PROCESSED, "counts", f"counts__{name}__.csv") + c = pd.read_csv(path) + unknown += c.loc[c["genotype"] == UNKNOWN_GENOTYPE, "counts"].sum() + total += c["counts"].sum() + out[lib] = round(float(unknown / total), 4) + return out + + +def realized_pool(lib): + """ + The realized library_mixture (mean over libraries of + estimate_library_mixture on the initial composition) and wt's share of + the pool. + """ + library_df = LibraryManager(lib).build_library_df() + ic = pd.read_csv(os.path.join(PROCESSED, "initial_composition.csv")) + ic = ic[ic["genotype"] != UNKNOWN_GENOTYPE] + mixes, wt_shares = [], [] + for _, g in ic.groupby("library"): + mix, _, _ = estimate_library_mixture( + library_df, g.set_index("genotype")["counts"]) + mixes.append(mix) + wt_shares.append(g.loc[g["genotype"] == "wt", "counts"].sum() + / g["counts"].sum()) + mixture = {k: float(np.mean([m[k] for m in mixes])) for k in mixes[0]} + return mixture, float(np.mean(wt_shares)) + + +def with_wt_excess(lib, mixture, wt_share): + """ + Genetics keys, mixture and transform sizes with wt repeated in the + spiked origin so wt makes up wt_share of the pool, the other spikes + keeping theirs (simulation only). + + The spiked origin's n sequences share its mixture by degeneracy. Giving + wt d copies and scaling the origin's mixture and transformants by + (n - 1 + d) / n leaves every other spike's mass unchanged; d is the + smallest integer that brings wt's pool fraction (spiked plus bulk, from + library_composition_table) up to wt_share. + """ + wt_seq = lib["wt_seq"] + spiked = list(lib["spiked_seqs"]) + library_df = LibraryManager(lib).build_library_df() + n_doubles = int((library_df["library_origin"] == "double-1-2").sum()) + n = len(spiked) + if spiked.count(wt_seq) != 1: + raise ValueError("expected wt exactly once in spiked_seqs") + + def build(d): + cf = dict(lib) + cf["spiked_seqs"] = spiked + [wt_seq] * (d - 1) + scale = (n - 1 + d) / n + cf["library_mixture"] = dict(mixture, spiked=mixture["spiked"] * scale) + sizes = dict(lib["transform_sizes"]) + sizes["double-1-2"] = CLONES_PER_DOUBLE * n_doubles + sizes["spiked"] = int(round(sizes["spiked"] * scale)) + cf["transform_sizes"] = sizes + comp = library_composition_table(cf) + wt = float(comp.loc[comp["genotype"] == "wt", "pool_fraction"].iloc[0]) + return cf, wt + + lo, hi = 1, 2 + while build(hi)[1] < wt_share: + lo, hi = hi, hi * 2 + while hi - lo > 1: + mid = (lo + hi) // 2 + if build(mid)[1] < wt_share: + lo = mid + else: + hi = mid + cf, wt = build(hi) + return cf, hi, wt + + +TUBE_KEY = ["replicate", "library", "condition_pre", "t_pre", "condition_sel", + "t_sel", "titrant_name", "titrant_conc"] + + +def calibrated_cfu0(pilot_dir, real_tubes): + """ + Per-library cfu0 that brings a pilot run's tube totals to the real ones. + + Matches each real tube to the pilot's simulated tube by design, takes + log(real total / simulated true total), and multiplies each library's + pilot cfu0 by exp(mean log ratio). Prints the residual log ratio by + library and selection condition after the correction. + """ + sim = pd.read_csv(os.path.join(pilot_dir, "tfs_sim_od600.csv")) + with open(os.path.join(pilot_dir, "tfs_sim_input-config.yaml")) as fh: + pilot_cfu0 = yaml.safe_load(fh)["cfu0"] + real = real_tubes.rename(columns={"rep": "replicate"}) + m = real[TUBE_KEY + ["sample_cfu"]].merge( + sim[TUBE_KEY + ["sample_cfu_true"]], on=TUBE_KEY, how="inner") + if len(m) < len(real): + print(f"warning: {len(real) - len(m)} real tubes have no pilot tube") + m["log_ratio"] = np.log(m["sample_cfu"] / m["sample_cfu_true"]) + out = {} + for lib, g in m.groupby("library"): + base = (float(str(pilot_cfu0[lib]).replace("_", "")) + if isinstance(pilot_cfu0, dict) + else float(str(pilot_cfu0).replace("_", ""))) + out[lib] = base * float(np.exp(g["log_ratio"].mean())) + m["residual"] = m["log_ratio"] - m.groupby("library")["log_ratio"].transform("mean") + print("cfu0 calibration: log(real / pilot) total by library, then the " + "residual by selection condition (mean, SD over tubes):") + print(m.groupby("library")["log_ratio"].agg(["mean", "std", "count"]).round(3) + .to_string()) + print(m.groupby(["library", "condition_sel"])["residual"] + .agg(["mean", "std", "count"]).round(3).to_string()) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out_dir", default="full_size_sim") + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--noise", choices=("realistic", "poisson"), + default="realistic") + ap.add_argument("--phenotype_model", default=None) + ap.add_argument("--transition", choices=("instant", "memory"), + default="instant") + ap.add_argument("--calibrate_from", default=None, + help="a pilot run's directory (tfs_sim_od600.csv and " + "tfs_sim_input-config.yaml) to calibrate cfu0 from") + args = ap.parse_args() + os.makedirs(args.out_dir, exist_ok=True) + + tubes = pd.read_csv(os.path.join(PROCESSED, "sample_df.csv")) + design = read_design(tubes) + design.to_csv(os.path.join(args.out_dir, "design.csv"), index=False) + + with open(os.path.join(PROCESSED, "library_config.yaml")) as fh: + lib = yaml.safe_load(fh) + # the fit is configured with the real library config, as the real data's + shutil.copyfile(os.path.join(PROCESSED, "library_config.yaml"), + os.path.join(args.out_dir, "library_config.yaml")) + mixture, wt_share = realized_pool(lib) + sim_lib, wt_copies, wt_pool = with_wt_excess(lib, mixture, wt_share) + unassigned = unassigned_fractions(tubes) + + cal_src = os.path.join(PROCESSED, "od600_calibration.yaml") + cal_dst = os.path.join(args.out_dir, "od600_calibration.yaml") + shutil.copyfile(cal_src, cal_dst) + + rates = pd.read_csv(os.path.join(PROCESSED, + "wt_control_rate_summary.csv")) + presplit = float(od600_to_cfu_per_mL(np.array([PRESPLIT_OD600]), + cal_src)[0][0]) + cfu0 = presplit * PRESPLIT_DILUTION * TUBE_VOLUME_ML + if args.calibrate_from: + cfu0 = calibrated_cfu0(args.calibrate_from, tubes) + total_reads = int(tubes["called_reads"].sum()) + reads_per_tube = total_reads / len(design) + + cf = {k: sim_lib[k] for k in GENETICS_KEYS if k in sim_lib} + cf.update({ + "design": "design.csv", + "growth": growth_block(rates), + "dk_geno_hyper_loc": -3.5, + "dk_geno_hyper_scale": DK_HYPER_SCALE, + "dk_geno_hyper_shift": 0.02, + "activity_wt": 1.0, + "activity_mut_scale": 0.0, + "lib_assembly_skew_sigma": dict(ASSEMBLY_SKEW), + "max_congressed_cells": MAX_CONGRESSED_CELLS, + "transformation_poisson_lambda": LAMBDA, + "congression_theta_rule": "homodimer", + "congression_dk_rule": "dilution", + "cfu0": ({k: float(round(v, -3)) for k, v in cfu0.items()} + if isinstance(cfu0, dict) else float(round(cfu0, -3))), + "tube_noise_sigma": 0.002, + "total_num_reads": total_reads, + "unassigned_read_fraction": unassigned, + "prob_index_hop": 0.0, + "seed": args.seed, + # bare file names: tfs-simulate runs from inside out_dir + "od600": {"calibration": os.path.basename(cal_dst), + "tube_volume_mL": TUBE_VOLUME_ML, + "sample_cfu_from_od600": True}, + }) + if args.phenotype_model: + cf["phenotype_source"] = "empirical" + cf["empirical"] = {"phenotype_model": os.path.abspath(args.phenotype_model)} + else: + cf["theta_component"] = "hill_geno" + if args.noise == "realistic": + cf.update({"founder_sampling": True, "demographic_growth": True, + "shared_transformation": True, + # templates = assigned reads per tube / 10, the ratio + # that matched the real counts' overdispersion + # (noise-anatomy, measured on called reads) + "pcr_template_molecules": int(round( + reads_per_tube * (1 - np.mean(list(unassigned.values()))) + / 10)), + "pcr_amplification_cv": 0.5}) + if args.transition == "memory": + cf["growth_transition"] = [ + {"condition_pre": "pheS-4CP", "model": "memory", + "tau0": 90.0, "k1": 5.0, "k2": 0.2}, + {"condition_pre": "kanR-kan", "model": "memory", + "tau0": 60.0, "k1": 2.0, "k2": 0.2}] + + path = os.path.join(args.out_dir, "simulate_config.yaml") + with open(path, "w") as fh: + fh.write("# Written by planning/studies/full-size-sim/make_sim_config.py\n" + f"# from {os.path.abspath(PROCESSED)}; noise {args.noise}, " + f"transition {args.transition}, seed {args.seed}.\n") + yaml.safe_dump(cf, fh, sort_keys=False) + print(f"{path}: {len(design)} tubes in {design['replicate'].nunique()} " + f"replicates; cfu0 {cf['cfu0']}; {total_reads:.3g} reads " + f"({reads_per_tube:.3g} per tube); growth {cf['growth']}") + print(f"realized library_mixture {cf['library_mixture']} (design " + f"{lib['library_mixture']}); wt {wt_share:.3f} of the real pool, " + f"{wt_copies} wt copies in spiked_seqs give {wt_pool:.3f}; " + f"unassigned_read_fraction {unassigned}") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/full-size-sim/run_full_size.srun b/planning/studies/full-size-sim/run_full_size.srun new file mode 100644 index 00000000..e0be704b --- /dev/null +++ b/planning/studies/full-size-sim/run_full_size.srun @@ -0,0 +1,86 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_fullsim +#SBATCH --output=run.out +#SBATCH --error=run.err +#SBATCH --partition=gpulong +#SBATCH --time=02-00:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# The full-size simulation of the dev-data design, through the same chain as +# real data (planning/studies/full-size-sim). Run from a directory made by +# make_sim_config.py (it holds design.csv, simulate_config.yaml and the +# calibration), inside planning/dev-data: +# +# python ../make_sim_config.py --out_dir sim_realistic_s1 --seed 1 +# cd sim_realistic_s1 && sbatch ../run_full_size.srun +# +# Each step is the documented command; nothing between them is edited by hand. +# The fit is configured with the real library config (library_config.yaml, +# copied by make_sim_config.py), as the real data's fit is; the simulated +# pool's realized composition and excess wt stay in the simulation. +# The fit is the growth-only relative model of the real fit +# (planning/studies/staged-map/run_staged.srun), so the simulation tests the +# recipe the real data use. + +module load cuda/12.4.1 + +set -e + +SEED="${SEED:-1}" + +# A rerun skips the simulation and the processing when their outputs exist +# (the simulation alone takes about 85 minutes). +if [[ -f tfs_sim_tubes.csv && -f tfs_sim_input-config.yaml ]]; then + echo ">>> Simulate: tfs_sim_tubes.csv exists, skipped" +else + echo ">>> Simulate $(date)" + tfs-simulate simulate_config.yaml --out_prefix tfs_sim --seed "${SEED}" +fi + +if [[ -f tfs_growth.csv ]]; then + echo ">>> Process counts: tfs_growth.csv exists, skipped" +else + echo ">>> Process counts $(date)" + tfs-process-counts tfs_sim_tubes.csv tfs_sim_counts \ + --od600_file tfs_sim_tube_od600.csv \ + --od600_calibration_file tfs_sim_od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix tfs_growth --no_verbose +fi + +# --allow_missing_spikes: a spike's simulated phenotype can kill it (seed 1 +# left H74A/K84L with no growth data, 2026-10-06). +echo ">>> Configure $(date)" +tfs-configure-model \ + --allow_missing_spikes \ + --growth_df tfs_growth.csv \ + --library_config library_config.yaml \ + --theta_model hill_relative \ + --growth_shares_replicates \ + --batch_size 4096 \ + --growth_priors ../growth_priors_loose.csv \ + --set_priors sigma_fixed=0.17 theta_log_hill_n_hyper_scale_fixed=0.5 + +echo ">>> Fit (staged MAP) $(date)" +tfs-fit-model tfs_configure_config.yaml --seed "${SEED}" --analysis_method map + +echo ">>> Arrowhead Laplace $(date)" +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --laplace arrowhead --num_posterior_samples 500 --skip_growth_observations + +echo ">>> Extract and predict $(date)" +tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 +tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Score (exact, full batch) $(date)" +python ../score_map.py . + +echo ">>> Summarize against the truth $(date)" +tfs-summarize-fit . + +echo ">>> Done $(date)" diff --git a/planning/studies/full-size-sim/run_pilot.srun b/planning/studies/full-size-sim/run_pilot.srun new file mode 100644 index 00000000..6337de69 --- /dev/null +++ b/planning/studies/full-size-sim/run_pilot.srun @@ -0,0 +1,29 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_pilot +#SBATCH --output=pilot.out +#SBATCH --error=pilot.err +#SBATCH --partition=gpulong +#SBATCH --time=0-06:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# The pilot simulation that calibrates cfu0 (planning/studies/full-size-sim): +# the first-guess cfu0, with the raw count files (for the depth check +# against the real screen) but no growth table. About 85 minutes and a 68 GB peak; numpy only, so no GPU is +# needed. Submit from the study's working directory (processed/ beside it): +# +# sbatch run_pilot.srun +# +# Then make_sim_config.py --calibrate_from pilot_s1 for each arm. + +set -e + +SEED="${SEED:-1}" + +python make_sim_config.py --out_dir "pilot_s${SEED}" --seed "${SEED}" +cd "pilot_s${SEED}" +tfs-simulate simulate_config.yaml --out_prefix tfs_sim --seed "${SEED}" \ + --no_write_growth diff --git a/planning/studies/growth-binding-map/README.md b/planning/studies/growth-binding-map/README.md new file mode 100644 index 00000000..075a9d67 --- /dev/null +++ b/planning/studies/growth-binding-map/README.md @@ -0,0 +1,123 @@ +# Study 0a: growth vs in vitro binding, model-free + +**Status: provisional (2026-09-26).** Run on the 2026-07-23 snapshot: +counts from a FASTQ processor that assigned every unknown read to wt (about +two thirds of kan reads), a stringent Q cutoff, and binding data for five +genotypes at the original (high) protein concentration. The wt-relative +pass is unusable in kan and wt's own points in the absolute pass are +suspect; the mutants' absolute slopes are unaffected. The user has since +collected binding at two lower repressor concentrations (curves much closer +to the in vivo response, wt no longer capped at theta = 1) for 10 genotypes +with diverse behavior, which confirms the concentration offset and should +give this study real leverage. The 4CP null result conflicts with an +earlier per-genotype MLE analysis that showed reasonable pheS response +curves; re-check it on the new data before drawing conclusions. + +## Question + +For the genotypes with in vitro binding data, is growth rate a single +linear function of the binding occupancy theta (the joint fit's +assumptions 1 and 2), a monotone but nonlinear one, or neither? Decides the +base case for Track G (`planning/growth-only-relative-fit.md`, step 0; +`planning/analysis-roadmap.md`, steps 5, 8, 9). + +## How to run + +```bash +python growth_binding_map.py ~/work/programming/git-clones/tfscreen/dev/real-2026-07-23 out/ \ + ~/Desktop/2026-07-23_onward/combined_fit_sample_df.csv +``` + +Inputs from `planning/studies/step0-data/`. The third argument enables the +second pass (absolute slopes and the concentration-scale scan). + +## Method + +Five genotypes have binding data: wt, M42I, H74A, K84L, M42I/H74A (all +spiked; 22,000-56,000 reads per tube). Binding and growth use the same +eight IPTG concentrations (0 to 1 mM). + +1. **wt-relative, total-free.** Slope over `t_sel` of + `ln(reads_g / reads_wt)` per replicate x condition x concentration: + `dk_g + m_c (X_g - X_wt)`, with no tube total and no tube noise. +2. **Absolute.** Slope of `ln(reads_g / D) + fit_lncfu` (the global + smoothed total): `k_c + dk_g + m_c X_g`. Uses theta itself, which for wt + spans 1.00 to 0.33, but trusts the total of each IPTG column: an error in + a column's total shifts every genotype at that concentration together. +3. **Concentration scale.** Hill fits to the binding data, then theta_g + evaluated at `s x` (in vivo effective [IPTG] = s times external) for + s = 0.3-300. + +Slope noise is taken from replicate differences (counting SEs understate it +about 2.7-fold; study 0b). + +## Commit + +Scripts as in this directory on top of `b949095`. + +## Results (2026-09-26) + +**1. wt-relative: no leverage.** Slope noise 0.0032-0.0037 per min +(replicate differences; counting alone predicts 0.0016). The four mutants' +binding curves differ from wt's by at most a few tenths across IPTG, so the +common-m fits are consistent with the linear map and measure nothing (kan +m = +0.005 +/- 0.006, 4CP m = +0.002 +/- 0.009 per min per unit theta). + +**2. Absolute, s = 1: the common linear map fails.** + +| condition | m (per min) | reduced chi2 | genotype-specific m | curvature | +|---|---|---|---|---| +| kan+ | -0.0124 +/- 0.0019 | 6.7 | p = 2e-5 | p = 4e-4 | +| 4CP+ | +0.0006 +/- 0.0019 | 2.0 | p = 6e-4 | p = 0.08 | + +The curves (mean over replicates, per min): + +- kan+: every genotype's growth rises from about 0 to 0.015-0.027 between + 0.001 and 0.01 mM IPTG, then flattens or falls slightly. In vitro, wt, + M42I and M42I/H74A are still fully bound (theta 1.00) at 0.01 mM, and wt + falls only to 0.33 at 1 mM. H74A (theta 0.57 at 0 mM) grows like wt at + 0 mM. +- 4CP+: growth falls with IPTG to a minimum near 0.01-0.03 mM and partly + recovers, for every genotype; no ordering by binding. +- Total-free check of the same point: at 0.01 mM in kan, all four mutants + outgrow wt (replicate 1: +0.006 to +0.015 per min) although M42I and + M42I/H74A bind like wt there in vitro. + +**3. An in vivo concentration scale helps in kan.** Reduced chi2 of the +common linear map by s: + +| s | 0.3 | 1 | 3 | 10 | 30 | 100 | 300 | +|---|---|---|---|---|---|---|---| +| kan+ | 7.1 | 6.7 | 6.0 | 4.8 | 3.4 | 3.0 | 3.7 | +| 4CP+ | 2.0 | 2.0 | 2.0 | 1.9 | 1.8 | 1.8 | 1.8 | + +At s = 30-100, kan m = -0.024 to -0.026 per min. 4CP stays weak +(m = +0.005 to +0.006). + +The no-drug conditions (only 0 and 1 mM) differ between the two +concentrations for every genotype (kanR library faster at 1 mM, pheS +library slower), in line with the OD600 population slopes (study 0c): +marker expression changes growth even without the drug, so "no drug" is not +a pure `dk_geno` control. + +## What it means + +- **The base case (one linear map at s = 1) is not supported.** In kan the + in vivo response sits at roughly 30-100x lower external IPTG than the in + vitro assay, with genotype-specific structure left over; in 4CP growth + does not track in vitro binding for these five genotypes. The joint fit's + assumption 1 (in vitro binding = in vivo occupancy at the same [IPTG]) + is likely violated, which is what Track G was built to find out. The map + in step 8 must include the concentration scale s. +- Plausible sources of s: active IPTG uptake (LacY) concentrating IPTG in + cells; the in vitro assay's conditions (22 uM protein; not all curves + taken with the final protocol, per the snapshot README). +- **Caveats.** Five genotypes, all similar to wt in vitro; the absolute + pass depends on per-column totals (roadmap C4), and the non-monotone + shapes shared by every genotype (the kan peak at 0.01 mM, the 4CP dip) + are what a column-total error would look like. +- **The direct test is a bench measurement:** monoculture growth rates of + these five genotypes across the eight IPTG concentrations in kan and 4CP + (`planning/absolute-abundance-measurements.md`, experiment 1, extended + from the gauge concentrations to the full titration). It measures the + growth-binding map with no library and no totals problem. diff --git a/planning/studies/growth-binding-map/growth_binding_map.py b/planning/studies/growth-binding-map/growth_binding_map.py new file mode 100644 index 00000000..44ec9fce --- /dev/null +++ b/planning/studies/growth-binding-map/growth_binding_map.py @@ -0,0 +1,301 @@ +""" +Step 0a of planning/analysis-roadmap.md: a model-free look at how growth +relates to in vitro binding, for the genotypes with binding data. + +For genotype g and wt in the same tube, ln(reads_g / reads_wt) does not +depend on the tube's total population, so its slope over t_sel is + + s_g(c, x) = dk_g + m_c * (X_g(x) - X_wt(x)) + +for condition c and IPTG concentration x, with no OD600 and no tube +noise. If growth is linear in the in vitro occupancy theta, then for each +selective condition the points (theta_g(x) - theta_wt(x), s_g(c, x)) +from every genotype lie on one line per genotype with a common slope m_c, +offset by dk_g. Genotype-specific slopes or curvature mean the linear map +does not hold. + +Second pass (optional SAMPLE_DF): absolute slopes. The wt-relative +design removes the tube total but also most of the signal, because the +binding genotypes' curves differ from wt's by only a few tenths of theta. +With the tube totals from the global smoothed fit (SAMPLE_DF's fit_lncfu), +ln_cfu_g = ln(reads_g / D) + fit_lncfu, and each genotype's absolute slope + + s_g(c, x) = k_c + dk_g + m_c * X_g(x) + +can be tested against theta_g(x) itself, which for wt spans nearly 0-1. + +Usage: + python growth_binding_map.py DATA_DIR OUT_DIR [SAMPLE_DF] + +DATA_DIR holds counts.parquet, samples.parquet and binding.csv from +planning/studies/step0-data/extract_snapshot.py. +""" + +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import pyarrow.parquet as pq +import statsmodels.formula.api as smf +from matplotlib import pyplot as plt + + +def wt_relative_slopes(counts, samples, genotypes): + """Weighted slope of ln(reads_g / reads_wt) on t_sel per genotype x + replicate x condition_sel x titrant_conc (Poisson weights).""" + c = counts.pivot_table(index="sample", columns="genotype", + values="counts", aggfunc="sum", observed=True) + rows = [] + s = samples.set_index("sample") + for g in genotypes: + if g == "wt": + continue + d = pd.DataFrame({"cg": c[g], "cw": c["wt"]}).join(s) + d = d[(d["cg"] > 0) & (d["cw"] > 0)] + d["y"] = np.log(d["cg"] / d["cw"]) + d["w"] = 1.0 / (1.0 / d["cg"] + 1.0 / d["cw"]) + for key, grp in d.groupby(["replicate", "condition_sel", + "titrant_conc"]): + if grp["t_sel"].nunique() < 2: + continue + f = smf.wls("y ~ t_sel", data=grp, weights=grp["w"]).fit() + rows.append({"genotype": g, "replicate": key[0], + "condition_sel": key[1], "titrant_conc": key[2], + "slope": f.params["t_sel"], + "slope_se": f.bse["t_sel"] if len(grp) > 2 else np.nan, + "n_tubes": len(grp), + "min_reads": int(grp["cg"].min())}) + return pd.DataFrame(rows) + + +def absolute_slopes(counts, samples, sample_df, genotypes): + """Slope of ln(reads_g / D) + fit_lncfu on t_sel per genotype x + replicate x condition_sel x titrant_conc.""" + tot = pd.read_csv(sample_df, index_col=0) + tot["condition_sel"] = tot["condition_sel"].str.strip() + tot = tot.rename(columns={"rep": "replicate"}) + keys = ["replicate", "condition_sel", "titrant_conc", "t_sel"] + smp = samples.merge(tot[keys + ["fit_lncfu"]], on=keys, how="inner") + print(f"\nsecond pass: {len(smp)} of {len(samples)} tubes matched to a " + f"smoothed total") + c = counts.pivot_table(index="sample", columns="genotype", + values="counts", aggfunc="sum", observed=True) + smp = smp.set_index("sample") + rows = [] + for g in genotypes: + d = smp.join(c[g].rename("cg"), how="inner") + d = d[d["cg"] > 0] + d["y"] = np.log(d["cg"] / d["freq_denominator"]) + d["fit_lncfu"] + for key, grp in d.groupby(["replicate", "condition_sel", + "titrant_conc"]): + if grp["t_sel"].nunique() < 2: + continue + f = smf.ols("y ~ t_sel", data=grp).fit() + rows.append({"genotype": g, "replicate": key[0], + "condition_sel": key[1], "titrant_conc": key[2], + "slope": f.params["t_sel"], "n_tubes": len(grp)}) + return pd.DataFrame(rows) + + +def test_map(sl, theta_col, label): + """Per condition: common m vs genotype-specific m vs curvature, with the + slope noise estimated from replicate differences.""" + w = sl.pivot_table(index=["genotype", "condition_sel", "titrant_conc"], + columns="replicate", values="slope") + diff = (w.iloc[:, 1] - w.iloc[:, 0]).dropna() + sd = np.sqrt(diff.var() / 2) + print(f"{label}: slope SD from replicate differences {sd:.5f} per min") + out = [] + for cond, d in sl.groupby("condition_sel"): + common = smf.ols(f"slope ~ C(genotype) + {theta_col}", d).fit() + per_g = smf.ols(f"slope ~ C(genotype) + C(genotype):{theta_col}", d).fit() + curved = smf.ols(f"slope ~ C(genotype) + {theta_col} + " + f"I({theta_col}**2)", d).fit() + # F-tests with the replicate-based noise as the known variance. + def chi_test(big, small): + from scipy import stats + dchi = (small.ssr - big.ssr) / sd ** 2 + ddf = small.df_resid - big.df_resid + return stats.chi2.sf(dchi, ddf) + m, m_se = common.params[theta_col], sd * np.sqrt( + common.normalized_cov_params.loc[theta_col, theta_col]) + red = common.ssr / sd ** 2 / common.df_resid + out.append({"condition_sel": cond, "m": m, "m_se": m_se, + "red_chi2": red, + "p_genotype_slopes": chi_test(per_g, common), + "p_curvature": chi_test(curved, common), "n": len(d)}) + print(f" {cond:10s} m = {m:+.5f} +/- {m_se:.5f}; reduced chi2 " + f"{red:.1f}; genotype-specific m p={out[-1]['p_genotype_slopes']:.3g}; " + f"curvature p={out[-1]['p_curvature']:.3g}") + return pd.DataFrame(out) + + +def hill_fits(binding): + """Four-parameter Hill fit (in log concentration) per genotype, so + theta can be evaluated off the measured grid.""" + from scipy.optimize import least_squares + x0 = binding["titrant_conc"][binding["titrant_conc"] > 0].min() / 10 + out = {} + for g, d in binding.groupby("genotype"): + lx = np.log(np.maximum(d["titrant_conc"].to_numpy(), x0)) + y = d["theta_obs"].to_numpy() + + def f(p, lx=lx): + lo, hi, lk, n = p + return hi + (lo - hi) / (1 + np.exp(-np.exp(n) * (lx - lk))) + r = least_squares(lambda p: f(p) - y, + [y.min(), y.max(), np.log(0.1), 0.0], + bounds=([-0.2, 0, -20, -3], [1, 1.2, 10, 3])) + out[g] = (f, r.x) + return out, x0 + + +def scan_concentration_scale(ab, binding, scales): + """For each s, theta_g evaluated at s * x (in vivo effective + concentration), then the common linear map's residual sum of squares + per selective condition.""" + fits, x0 = hill_fits(binding) + rows = [] + for sc in scales: + d = ab.copy() + d["theta_s"] = [fits[g][0](fits[g][1], np.log(max(x * sc, x0))) + for g, x in zip(d["genotype"], d["titrant_conc"])] + for cond, dd in d.groupby("condition_sel"): + f = smf.ols("slope ~ C(genotype) + theta_s", dd).fit() + rows.append({"s": sc, "condition_sel": cond, "ssr": f.ssr, + "m": f.params["theta_s"], "df": f.df_resid}) + return pd.DataFrame(rows) + + +def main(data_dir, out_dir, sample_df=None): + data_dir, out_dir = Path(data_dir), Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + binding = pd.read_csv(data_dir / "binding.csv") + genotypes = sorted(binding["genotype"].unique()) + counts = pq.read_table(data_dir / "counts.parquet", + filters=[("genotype", "in", genotypes)]).to_pandas() + samples = pd.read_parquet(data_dir / "samples.parquet") + print(f"binding genotypes: {genotypes}") + print(f"binding concentrations: {sorted(binding['titrant_conc'].unique())}") + print(f"growth concentrations: {sorted(samples['titrant_conc'].unique())}") + + sl = wt_relative_slopes(counts, samples, genotypes) + + th = binding.pivot_table(index="titrant_conc", columns="genotype", + values="theta_obs") + th_sd = binding.pivot_table(index="titrant_conc", columns="genotype", + values="theta_std") + sl["dtheta"] = [th.loc[x, g] - th.loc[x, "wt"] + for g, x in zip(sl["genotype"], sl["titrant_conc"])] + sl["dtheta_sd"] = [np.hypot(th_sd.loc[x, g], th_sd.loc[x, "wt"]) + for g, x in zip(sl["genotype"], sl["titrant_conc"])] + sl.to_csv(out_dir / "wt_relative_slopes.csv", index=False) + + print(f"\nslope SE (per min): median {sl['slope_se'].median():.5f}; " + f"min reads in a tube: median {sl['min_reads'].median():.0f}") + + # Replicate agreement. + w = sl.pivot_table(index=["genotype", "condition_sel", "titrant_conc"], + columns="replicate", values="slope") + diff = w.iloc[:, 1] - w.iloc[:, 0] + print(f"replicate difference in slope: SD {diff.std():.5f} " + f"(expected from SEs ~ " + f"{np.sqrt(2) * sl['slope_se'].median():.5f})") + + # Counting SEs miss most of the noise (study 0b: per-tube variance is + # 5-10x Poisson). Add an excess slope variance tau^2, estimated from + # the replicate differences, to every slope's SE. + tau2 = max(diff.var() / 2 - (sl["slope_se"] ** 2).mean(), 0.0) + sl["slope_se_counting"] = sl["slope_se"] + sl["slope_se"] = np.sqrt(sl["slope_se"] ** 2 + tau2) + print(f"excess slope SD tau = {np.sqrt(tau2):.5f} per min, added in " + f"quadrature to every slope SE") + + # Per selective condition: common m vs genotype-specific m vs curvature. + print("\nper condition (both replicates pooled; genotype = dk offset):") + results = [] + for cond, d in sl.groupby("condition_sel"): + d = d.copy() + d["w"] = 1.0 / d["slope_se"] ** 2 + common = smf.wls("slope ~ C(genotype) + dtheta", d, weights=d["w"]).fit() + per_g = smf.wls("slope ~ C(genotype) + C(genotype):dtheta", d, + weights=d["w"]).fit() + curved = smf.wls("slope ~ C(genotype) + dtheta + I(dtheta**2)", d, + weights=d["w"]).fit() + f_g = per_g.compare_f_test(common) + f_c = curved.compare_f_test(common) + red_chi2 = common.ssr / common.df_resid + results.append({"condition_sel": cond, "m_common": common.params["dtheta"], + "m_se": common.bse["dtheta"], "red_chi2_common": red_chi2, + "p_genotype_slopes": f_g[1], "p_curvature": f_c[1], + "n": len(d)}) + print(f" {cond:10s} m = {common.params['dtheta']:+.5f} " + f"+/- {common.bse['dtheta']:.5f}; reduced chi2 {red_chi2:.1f}; " + f"genotype-specific m p={f_g[1]:.3g}; curvature p={f_c[1]:.3g}") + for g in sorted(d["genotype"].unique()): + dg = d[d["genotype"] == g] + if dg["dtheta"].std() > 0.05: + fg = smf.wls("slope ~ dtheta", dg, weights=dg["w"]).fit() + print(f" {g:10s} m_g = {fg.params['dtheta']:+.5f} " + f"+/- {fg.bse['dtheta']:.5f} " + f"(dtheta range {dg['dtheta'].min():+.2f}.." + f"{dg['dtheta'].max():+.2f})") + pd.DataFrame(results).to_csv(out_dir / "map_tests.csv", index=False) + + # Figure: slope vs dtheta per condition, coloured by genotype. + conds = sorted(sl["condition_sel"].unique()) + fig, axes = plt.subplots(1, len(conds), figsize=(4 * len(conds), 3.6)) + for ax, cond in zip(np.atleast_1d(axes), conds): + d = sl[sl["condition_sel"] == cond] + for g, dg in d.groupby("genotype"): + ax.errorbar(dg["dtheta"], dg["slope"], xerr=dg["dtheta_sd"], + yerr=dg["slope_se"], fmt="o", ms=3, label=g) + ax.axhline(0, color="gray", lw=0.5) + ax.set_title(cond) + ax.set_xlabel("theta_g - theta_wt (binding)") + np.atleast_1d(axes)[0].set_ylabel("d ln(reads_g/reads_wt) / dt (per min)") + np.atleast_1d(axes)[0].legend(fontsize=7) + fig.tight_layout() + fig.savefig(out_dir / "growth_vs_binding.pdf") + + if sample_df is None: + return + ab = absolute_slopes(counts, samples, sample_df, genotypes) + ab["theta"] = [th.loc[x, g] for g, x in zip(ab["genotype"], ab["titrant_conc"])] + ab.to_csv(out_dir / "absolute_slopes.csv", index=False) + test_map(ab, "theta", "absolute slopes vs binding theta").to_csv( + out_dir / "map_tests_absolute.csv", index=False) + + scales = [0.3, 1, 3, 10, 30, 100, 300] + sc = scan_concentration_scale(ab, binding, scales) + sc.to_csv(out_dir / "concentration_scale_scan.csv", index=False) + w = ab.pivot_table(index=["genotype", "condition_sel", "titrant_conc"], + columns="replicate", values="slope") + sd2 = (w.iloc[:, 1] - w.iloc[:, 0]).dropna().var() / 2 + print("\nconcentration scale s (in vivo [IPTG] = s * in vitro): " + "reduced chi2 of the common linear map") + tab = sc.assign(red=sc["ssr"] / sd2 / sc["df"]).pivot( + index="s", columns="condition_sel", values="red") + print(tab.round(2).to_string()) + print("m at each s:") + print(sc.pivot(index="s", columns="condition_sel", values="m") + .round(4).to_string()) + + conds = sorted(ab["condition_sel"].unique()) + fig, axes = plt.subplots(1, len(conds), figsize=(4 * len(conds), 3.6)) + for ax, cond in zip(np.atleast_1d(axes), conds): + d = ab[ab["condition_sel"] == cond] + for g, dg in d.groupby("genotype"): + ax.plot(dg["theta"], dg["slope"], "o", ms=3, label=g) + ax.set_title(cond) + ax.set_xlabel("theta_g (binding)") + np.atleast_1d(axes)[0].set_ylabel("d ln_cfu_g / dt (per min)") + np.atleast_1d(axes)[0].legend(fontsize=7) + fig.tight_layout() + fig.savefig(out_dir / "absolute_growth_vs_binding.pdf") + + +if __name__ == "__main__": + main(*sys.argv[1:4]) diff --git a/planning/studies/joint-fit-real/README.md b/planning/studies/joint-fit-real/README.md new file mode 100644 index 00000000..c1587dc1 --- /dev/null +++ b/planning/studies/joint-fit-real/README.md @@ -0,0 +1,128 @@ +# Joint stage 1 on the real screen + +**Question.** With the recipe from `planning/studies/joint-fit-sim/` +(absolute theta from `hill_geno`, binding at weight 1, the plain model +first and the full model started from it, 75 co-resident sets), what do +the real data say: is kanR's growth a switch-like function of occupancy +and pheS's graded, what is the in vivo titrant scale s, which binding +genotypes disagree with their growth (M42I), and does the full model +explain growth and binding better than the plain one? + +**Decision it feeds.** `planning/joint-fit-return.md` step 6, and through +it step 7 (stage 2 with the full library) and which model pieces stay. + +## Inputs + +- Growth: the recipe's growth table (`dev-data/real_fit/recipe/tfs_growth.csv` + on the cluster), wt, spikes and singles (`two_stage.stage1_genotypes`), + with the probably swapped pheS+4CP replicate 2, t_sel 82 row left out + (`planning/studies/relative-m/`, tube QC; the lab judged the swap + probable, and the 1 uM tube is ambiguous, so the row is dropped rather + than relabeled). +- Library: `dev-data/processed/library_config.yaml`. +- Binding: the 2026-10-15 table through `tfs-process-binding` at 2 uM: + 110 points over seven genotypes (wt, M42I, H74A, K84L, M42I/H74A, + H74A/K84L, G65H), theta_std from the pooled replicate SD (0.13, wt's + day-to-day scatter; the mutants have one replicate each). D88A, V95N and + the other spikes have no 2 uM data. The lab's data stay out of the + repository: the table is made locally in `planning/dev-data` (gitignored) + and copied to the cluster. + +## Model + +`stage1_real.py`: `hill_geno`; the mixture with the homodimer rule, lambda +prior 0.38 +- 0.15, co-resident sets 50 20 5 (`--congression_sets`, +joint-fit-sim run 9); binding at weight 1; counts with level tube offsets +(SD 0.17); log(n) population SD 0.5; one k and m per condition with loose +priors. `plain`: linear growth, s = 1, no offsets. `full`: warps on both +selection conditions, s, and one offset per binding genotype; started from +the plain fit. The Laplace uses a Hessian chunk of 8 (75 sets ran the +default 64 out of GPU memory on the simulation). + +## What to expect from the simulations + +The real data have fewer binding anchors than the simulations: wt and +five spikes at 2 uM (the simulations had wt and eight) plus one in-library +genotype (G65H). At 0 to 5 extra binding singles the congression +simulation put stage 1's m 17-21% steep and the growth-only singles' +theta RMSE at 0.046-0.056, intervals 3-6 times too narrow +(`planning/studies/joint-fit-sim/`, runs 8 and 10). Treat the real +fit's theta intervals as provisional; m, the warps, s and the offsets are +the results here. + +## How to run + +Locally, make the binding table (already done on 2026-10-18): + +```bash +cd planning/dev-data && tfs-process-binding binding/2026-10-15_binding.csv --out_prefix binding/processed/2026-10-15_binding_2uM +``` + +Copy it to the cluster: + +```bash +ssh harms@login2.talapas.uoregon.edu 'mkdir -p /gpfs/projects/harmslab/harms/studies/dev-data/binding' +``` + +```bash +scp planning/dev-data/binding/processed/2026-10-15_binding_2uM.csv harms@login2.talapas.uoregon.edu:/gpfs/projects/harmslab/harms/studies/dev-data/binding/ +``` + +Verify: + +```bash +ssh harms@login2.talapas.uoregon.edu 'wc -l /gpfs/projects/harmslab/harms/studies/dev-data/binding/2026-10-15_binding_2uM.csv' +``` + +(111 lines.) Then on the cluster, the plain fit and the full fit started +from it: + +```bash +mkdir -p /gpfs/projects/harmslab/harms/studies/joint-fit-real && cd /gpfs/projects/harmslab/harms/studies/joint-fit-real && R=/gpfs/home/harms/tfscreen/planning/studies/joint-fit-real/stage1_real.srun && j=$(sbatch --parsable --export=ALL,MODEL=plain,OUT=plain --output=plain.out $R) && sbatch --dependency=afterok:$j --export=ALL,MODEL=full,OUT=full,INIT=plain --output=full.out $R +``` + +Then locally, after downloading each fit's `tfs_params_*.csv`, +`tfs_pred_theta*.csv` and convergence tables into `runs/`: + +```bash +python report.py runs/plain runs/full --binding ../../dev-data/binding/processed/2026-10-15_binding_2uM.csv --out out +``` + +## Results + +### Run 1 (2026-10-18): 2 uM binding + +Both fits stopped at the epoch cap still descending slowly (about 20 nats +over the last five windows at step 1e-5); the Laplace ran (chunk 8). +Final MAP losses: plain 813,484, full 804,973. + +| | plain | full | +|---|---|---| +| m kanR+kan (within-tube -0.0046) | -0.0280 | -0.063 [-0.089, -0.035] | +| m pheS+4CP (within-tube +0.0125) | +0.0171 | +0.0156 | +| kanR warp a / x0 | - | 8.9 / 0.81 | +| pheS warp a / x0 | - | 6.6 / 0.52 | +| s | (1) | 0.53 [0.23, 1.26] | +| lambda (prior 0.38 +- 0.15) | 2.10 | 0.82 | +| offset scale tau | - | 0.34 | + +Binding offsets (full): G65H -0.18, H74A -0.23, H74A/K84L -0.27, K84L +-0.28, M42I -0.07, M42I/H74A -0.20, wt -0.17: every one negative, and in +the plain fit every genotype's binding sits below its fitted curve (mean +residual -0.03 to -0.26). + +wt shows why. Its 2 uM binding falls from 0.75 at 0 IPTG to about 0 by +0.01 mM (midpoint near 0.003 mM); the curve growth wants has its midpoint +near 0.03 mM and a floor of 0.15-0.24. An IPTG scale of about 0.1 would +reconcile the two for wt; the fit keeps s near 0.5 and leaves wt's binding +misfit (RMS residual 2.9-4.0 measurement SDs), since growth's rows +outweigh eighteen binding points. The offsets then absorb a common level +difference (a hierarchical offset with mean zero is the wrong shape for +it), and lambda runs to 0.8-2.1 against the measured 0.36 to absorb the +rest. So m, the warps and lambda here are not results. + +The 2 uM binding and growth disagree by more than s plus per-genotype +offsets can express. This matches the open question of +planning/joint-fit-return.md (wt's IPTG midpoint matched 20 uM binding, +the mutants' levels 2 uM): the in vivo occupancy may correspond to more +than 2 uM of repressor. diff --git a/planning/studies/joint-fit-real/report.py b/planning/studies/joint-fit-real/report.py new file mode 100644 index 00000000..88adad61 --- /dev/null +++ b/planning/studies/joint-fit-real/report.py @@ -0,0 +1,105 @@ +""" +Report joint stage 1 fits on the real screen (README). + + python report.py runs/plain runs/full --binding BINDING_CSV --out out + +For each fit: its final MAP loss; k and m per condition (with the +within-tube m of planning/studies/within-tube-m for reference); the warps, +log s and the offset scale tau when present; and per binding genotype its +offset and binding residuals (theta_obs minus the fitted curve at the +binding concentrations plus the offset), as RMS and RMS in units of +theta_std. Writes out/params.csv and out/binding_residuals.csv. +""" + +import argparse +import os + +import numpy as np +import pandas as pd + +Q = ["q0.025", "q0.5", "q0.975"] +# kanR+kan and pheS+4CP, change of the wt/flat-reference slope from 0 to 1 mM +# (planning/studies/within-tube-m, run 3, swapped row left out). +WITHIN_TUBE_M = {"kanR+kan": -0.0046, "pheS+4CP": 0.0125} + + +def _read(path): + return pd.read_csv(path) if os.path.exists(path) else None + + +def params(run, name): + rows = [] + for p in ("growth_k", "growth_m", "warp_log_a", "warp_x0", + "growth_log_conc_shift", "offset_scale"): + d = _read(os.path.join(run, f"tfs_params_{p}.csv")) + if d is None: + continue + key = ("condition_rep" if "condition_rep" in d.columns + else "parameter" if "parameter" in d.columns else d.columns[0]) + for _, r in d.iterrows(): + rows.append(dict(run=name, parameter=p, key=r[key], + **{q: r[q] for q in Q})) + out = pd.DataFrame(rows) + out["within_tube_m"] = np.where(out["parameter"] == "growth_m", + out["key"].map(WITHIN_TUBE_M), np.nan) + return out + + +def residuals(run, name, binding): + pred = pd.read_csv(os.path.join(run, "tfs_pred_theta_binding.csv")) + pred["genotype"] = pred["genotype"].astype(str) + b = binding.merge(pred[["genotype", "titrant_conc", "q0.5"]], + on=["genotype", "titrant_conc"], how="inner") + off = _read(os.path.join(run, "tfs_params_offset.csv")) + offset = ({} if off is None + else dict(zip(off["genotype"].astype(str), off["q0.5"]))) + b["offset"] = b["genotype"].map(offset).fillna(0.0) + b["resid"] = b["theta_obs"] - (b["q0.5"] + b["offset"]) + b["z"] = b["resid"] / b["theta_std"] + out = (b.groupby("genotype") + .agg(n=("resid", "size"), offset=("offset", "first"), + mean_resid=("resid", "mean"), + rms_resid=("resid", lambda x: float(np.sqrt(np.mean(x ** 2)))), + rms_z=("z", lambda x: float(np.sqrt(np.mean(x ** 2))))) + .reset_index()) + out.insert(0, "run", name) + return out + + +def final_loss(run): + c = _read(os.path.join(run, "tfs_fit_model_convergence.csv")) + if c is None or c.empty: + return np.nan, "no convergence windows" + r = c.iloc[-1] + return float(r["loss_exact"]), str(r["decision"]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("runs", nargs="+") + ap.add_argument("--binding", required=True) + ap.add_argument("--out", default="out") + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + binding = pd.read_csv(args.binding) + binding["genotype"] = binding["genotype"].astype(str) + ps, rs = [], [] + for run in args.runs: + name = os.path.basename(os.path.normpath(run)) + loss, decision = final_loss(run) + print(f"{name}: final MAP loss {loss:.1f} ({decision})") + ps.append(params(run, name)) + rs.append(residuals(run, name, binding)) + p = pd.concat(ps, ignore_index=True) + r = pd.concat(rs, ignore_index=True) + p.to_csv(os.path.join(args.out, "params.csv"), index=False) + r.to_csv(os.path.join(args.out, "binding_residuals.csv"), index=False) + pd.set_option("display.width", 200) + print() + print(p.to_string(index=False, float_format="%+.4f")) + print() + print(r.to_string(index=False, float_format="%+.3f")) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/joint-fit-real/stage1_real.py b/planning/studies/joint-fit-real/stage1_real.py new file mode 100644 index 00000000..34399f4d --- /dev/null +++ b/planning/studies/joint-fit-real/stage1_real.py @@ -0,0 +1,164 @@ +""" +Joint stage 1 on the real screen (README; planning/joint-fit-return.md +step 6): wt, spikes and singles, growth and binding together. + + python stage1_real.py GROWTH_CSV LIBRARY_CONFIG BINDING_CSV OUT_DIR --model plain|full + [--init_from OTHER/tfs_fit_model_params.npz] + [--drop_tubes COND:REP:T_SEL ...] [--congression_sets 50 20 5] + [--seed 1] [--hessian_chunk_size 8] + +The recipe from planning/studies/joint-fit-sim: hill_geno (absolute theta, +shared by both markers), the congression mixture with the homodimer rule +(lambda prior 0.38 +- 0.15, the lab's measurement), binding at weight 1, +the count likelihood with level tube offsets (SD 0.17), log(n) population +SD 0.5, one k and m per condition with loose priors. + +- plain: linear growth in theta, s = 1, binding as measured. +- full: plus a logistic warp of occupancy on both selection conditions, + the growth titrant scale s (log s ~ Normal(0, 1)) and one binding offset + per binding genotype (genotype_offset). Start it from the plain fit + (--init_from): from scratch the full model can stop in a wrong mode + (joint-fit-sim, runs 1-3). + +Each step is skipped when its output exists, so a rerun resumes. theta is +predicted at the growth concentrations (tfs_pred_theta.csv) and at the +binding concentrations (tfs_pred_theta_binding.csv), both in binding units. +""" + +import argparse +import os + +import pandas as pd + +from tfscreen.genetics.library_design import library_composition_table +from tfscreen.tfmodel import two_stage as ts +from tfscreen.tfmodel.scripts.configure_model_cli import configure_model +from tfscreen.tfmodel.scripts.extract_params_cli import extract_params +from tfscreen.tfmodel.scripts.fit_model_cli import fit_model +from tfscreen.tfmodel.scripts.predict_theta_cli import predict_theta +from tfscreen.tfmodel.scripts.sample_posterior_cli import sample_posterior + +CONFIG = "tfs_configure_config.yaml" +CHECKPOINT = "tfs_fit_model_checkpoint.pkl" +LOOSE = dict(k_loc=0.015, k_scale=0.01, m_loc=0.0, m_scale=0.01) + + +def growth_priors(growth_csv, warp): + """Loose k/m priors for every condition; m_warp on selection conditions.""" + conds = set() + for chunk in pd.read_csv(growth_csv, usecols=["condition_pre", "condition_sel"], + chunksize=1_000_000): + conds |= set(chunk["condition_pre"]) | set(chunk["condition_sel"]) + t = pd.DataFrame({"condition_rep": sorted(conds)}).assign(**LOOSE) + if warp: + t["m_warp"] = t["condition_rep"].str.contains(r"\+").astype(int) + return t + + +def drop_rows(df, specs): + """Drop the tubes named condition_sel:replicate:t_sel.""" + keep = pd.Series(True, index=df.index) + for spec in specs: + parts = spec.split(":") + if len(parts) != 3: + raise ValueError(f"--drop_tubes {spec!r}: expected condition_sel:replicate:t_sel") + cond, rep, t_sel = parts + hit = ((df["condition_sel"] == cond) + & (df["replicate"].astype(float) == float(rep)) + & (df["t_sel"].astype(float) == float(t_sel))) + if not hit.any(): + raise ValueError(f"--drop_tubes {spec!r} matches no rows") + keep &= ~hit + return df[keep] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("growth_csv") + ap.add_argument("library_config") + ap.add_argument("binding_csv") + ap.add_argument("out_dir") + ap.add_argument("--model", choices=["full", "plain"], required=True) + ap.add_argument("--init_from", default=None) + ap.add_argument("--drop_tubes", nargs="*", default=[]) + ap.add_argument("--congression_sets", type=int, nargs="+", default=None) + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--hessian_chunk_size", type=int, default=8) + ap.add_argument("--max_num_epochs", type=int, default=None) + args = ap.parse_args() + growth_csv = os.path.abspath(args.growth_csv) + library_config = os.path.abspath(args.library_config) + binding_csv = os.path.abspath(args.binding_csv) + init_from = None if args.init_from is None else os.path.abspath(args.init_from) + full = args.model == "full" + os.makedirs(args.out_dir, exist_ok=True) + os.chdir(args.out_dir) + + if not os.path.exists("tfs_growth.csv"): + lib = library_composition_table(library_config) + keep = ts.stage1_genotypes(lib, ts.growth_genotypes(growth_csv)) + ts.filter_growth_csv(growth_csv, "tfs_growth.csv.partial", genotypes=keep) + if args.drop_tubes: + g = pd.read_csv("tfs_growth.csv.partial") + n = len(g) + g = drop_rows(g, args.drop_tubes) + print(f"dropped {n - len(g)} rows ({', '.join(args.drop_tubes)})", flush=True) + g.to_csv("tfs_growth.csv.partial", index=False) + os.replace("tfs_growth.csv.partial", "tfs_growth.csv") + print(f"stage 1: {len(keep)} genotypes (wt, spikes, singles)", flush=True) + + if not os.path.exists(CONFIG): + growth_priors("tfs_growth.csv", warp=full).to_csv("growth_priors.csv", index=False) + set_priors = ["sigma_fixed=0.17", "theta_log_hill_n_hyper_scale_fixed=0.5"] + if full: + set_priors.append("theta_growth_log_conc_shift_scale=1.0") + extra = {} if args.congression_sets is None else { + "congression_sets": list(args.congression_sets)} + configure_model( + binding_df=binding_csv, + growth_df=os.path.abspath("tfs_growth.csv"), + library_config=library_config, + theta_model="hill_geno", + transformation_model="mixture", + transformation_lambda=[0.38, 0.15], + congression_theta_rule="homodimer", + theta_binding_noise_model="genotype_offset" if full else "zero", + growth_shares_replicates=True, + batch_size=4096, + allow_missing_spikes=True, + growth_priors=os.path.abspath("growth_priors.csv"), + set_priors=set_priors, + skip_model_stats=True, + out_prefix="tfs_configure", + **extra) + + fit_kw = {} if args.max_num_epochs is None else {"max_num_epochs": args.max_num_epochs} + if init_from is not None: + fit_kw["init_from"] = init_from + if not os.path.exists("tfs_fit_model_params.npz"): + fit_model(CONFIG, seed=args.seed, analysis_method="map", **fit_kw) + if not os.path.exists("tfs_posterior.h5"): + sample_posterior(CONFIG, CHECKPOINT, out_prefix="tfs_posterior", + num_posterior_samples=500, skip_growth_observations=True, + hessian_chunk_size=args.hessian_chunk_size) + if not os.path.exists("tfs_params_provenance.json"): + extract_params(CONFIG, "tfs_posterior.h5", out_prefix="tfs_params") + if not os.path.exists("tfs_pred_theta.csv"): + predict_theta(CONFIG, "tfs_posterior.h5", out_prefix="tfs_pred_theta") + if not os.path.exists("tfs_pred_theta_binding.csv"): + b = pd.read_csv(binding_csv) + with open("binding_genotypes.txt", "w") as fh: + fh.write("\n".join(sorted(b["genotype"].astype(str).unique())) + "\n") + with open("binding_concs.txt", "w") as fh: + fh.write("\n".join(str(c) for c in sorted(b["titrant_conc"].unique())) + "\n") + with open("binding_titrants.txt", "w") as fh: + fh.write("\n".join(sorted(b["titrant_name"].astype(str).unique())) + "\n") + predict_theta(CONFIG, "tfs_posterior.h5", out_prefix="tfs_pred_theta_binding", + genotypes_file="binding_genotypes.txt", + titrant_names_file="binding_titrants.txt", + titrant_concs_file="binding_concs.txt", only_files=True) + print("stage 1 done", flush=True) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/joint-fit-real/stage1_real.srun b/planning/studies/joint-fit-real/stage1_real.srun new file mode 100644 index 00000000..435e06b2 --- /dev/null +++ b/planning/studies/joint-fit-real/stage1_real.srun @@ -0,0 +1,46 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=jfreal_fit +#SBATCH --partition=gpu +#SBATCH --time=06:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# Joint stage 1 on the real screen (planning/studies/joint-fit-real). Run +# from the study's cluster directory: +# +# sbatch --export=ALL,MODEL=plain,OUT=plain --output=plain.out stage1_real.srun +# +# MODEL plain | full +# OUT output directory +# INIT optional: a fit directory to start from (its MAP params) +# BINDING binding_df (default: dev-data/binding/2026-10-15_binding_2uM.csv) +# CSETS co-resident sets, ':'-separated (default 50:20:5) +# DROP tubes to leave out, ' '-separated condition_sel:replicate:t_sel +# (default the probably swapped pheS+4CP replicate 2, t_sel 82 row) +# +# A rerun resumes. + +module load cuda/12.4.1 +set -e +MODEL="${MODEL:?set MODEL}" +OUT="${OUT:?set OUT}" +DEV=/gpfs/projects/harmslab/harms/studies/dev-data +GROWTH="${GROWTH:-${DEV}/real_fit/recipe/tfs_growth.csv}" +LIB="${LIB:-${DEV}/processed/library_config.yaml}" +BINDING="${BINDING:-${DEV}/binding/2026-10-15_binding_2uM.csv}" +CSETS="${CSETS:-50:20:5}" +DROP="${DROP:-pheS+4CP:2:82}" +STUDY=/gpfs/home/harms/tfscreen/planning/studies/joint-fit-real +IFS=':' read -r -a SETS <<< "${CSETS}" +read -r -a DROPS <<< "${DROP}" +EXTRA=(--congression_sets "${SETS[@]}" --drop_tubes "${DROPS[@]}") +if [[ -n "${INIT:-}" ]]; then + EXTRA+=(--init_from "${INIT}/tfs_fit_model_params.npz") +fi +echo ">>> stage 1 real ${MODEL} -> ${OUT} $(date)" +python "${STUDY}/stage1_real.py" "${GROWTH}" "${LIB}" "${BINDING}" "${OUT}" --model "${MODEL}" "${EXTRA[@]}" +echo ">>> done $(date)" diff --git a/planning/studies/joint-fit-sim/README.md b/planning/studies/joint-fit-sim/README.md new file mode 100644 index 00000000..a132e8f3 --- /dev/null +++ b/planning/studies/joint-fit-sim/README.md @@ -0,0 +1,610 @@ +# Joint stage 1 on a simulation + +**Question.** With binding back at weight 1 and the joint model's pieces +(absolute theta from `hill_geno`, a per-marker warp of occupancy, the +binding-to-growth titrant scale s, one binding offset per binding +genotype), does stage 1 (wt, spikes and singles, with the congression +mixture) recover m, the maps, s and theta on a simulation where we know +them? Does binding at weight 1 leave growth undistorted, and does the full +model return the null when there is nothing to find? + +**Decision it feeds.** `planning/joint-fit-return.md` step 5: whether to +run step 6 (the real stage 1) with this model, and which pieces it needs. + +## Simulations (`make_sim.py`) + +Both start from the full-size realistic configuration +(`planning/studies/full-size-sim/make_sim_config.py`; on the cluster +`full-sized-sims-3/sim_realistic_s2`), seed 1, so the library, the +phenotypes and the spikes' curves are the same. Every spike gets binding +at the lab's 2 uM IPTG grid (16 concentrations, 0 to 5 mM) with noise 0.04, +the scatter within one real curve. Its curve comes from `spike_hill.csv`: +wt at the simulator's reference, the other eight drawn by the script. + +| arm | kanR+kan | pheS+4CP | s | binding offsets | +|---|---|---|---|---| +| `joint` | switch-like, warp a = 8, x0 = 0.5 | graded, a = 1.5, x0 = 0.4 | 0.5 | SD 0.1 per spike | +| `null` | linear in theta | linear | 1 | none | + +New simulator keys (2026-10-16): a linear growth condition's `warp_a` and +`warp_x0`, the top-level `growth_titrant_scale` and +`binding_data.genotype_offset_sd`. Their truths are in +`tfs_sim_growth_parameters.csv` (`warp_log_a`, `warp_x0`), +`tfs_sim_growth_titrant_scale.csv` (log s) and +`tfs_sim_binding_offsets.csv`. + +## Fits (`stage1.py`) + +Stage 1 of the two-stage fit, joint: wt, spikes and singles; `hill_geno`; +the mixture with the homodimer rule (lambda prior 0.357 +- 0.15); binding +at weight 1; counts with level tube offsets (SD 0.17); log(n) population +SD 0.5; one k and m per condition with loose priors. MAP (staged), Laplace +with 500 draws, extraction, theta predictions. + +- `full`: plus the warp on both selection conditions, s (log s ~ + Normal(0, 1)) and the binding offsets (`genotype_offset`). +- `plain`: none of those. + +| run | simulation | model | asks | +|---|---|---|---| +| `fit_joint_s1_full` | joint | full | recovery of m, warps, s, offsets, theta | +| `fit_joint_s1_plain` | joint | plain | what the pieces buy (the misspecified model) | +| `fit_null_s1_full` | null | full | the null comes back (a small, s near 1, tau near 0) without costing theta | + +**Pass.** On `fit_joint_s1_full`: m within its 95% interval and within +about 10% of the truth in both selection conditions; warp log a and x0 +covered; log s covered; the offsets covered and tau's interval holding +0.1; theta coverage near 0.95 for wt, spikes and singles. On +`fit_null_s1_full`: kanR's and pheS's warps compatible with near-linear, +log s covering 0, tau's interval reaching toward 0, and theta no worse than +`fit_joint_s1_full`'s. `fit_joint_s1_plain` should miss somewhere; where +it does says which piece matters. + +## How to run + +On the cluster, after pulling the commit that adds this study: + +```bash +mkdir -p /gpfs/projects/harmslab/harms/studies/joint-fit-sim && cd /gpfs/projects/harmslab/harms/studies/joint-fit-sim +``` + +```bash +for a in joint null; do sbatch --export=ALL,ARM=$a,SEED=1 --output=sim_$a.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/simulate.srun; done +``` + +Once both finish (about 90 minutes each; check `ls sim_*_s1/tfs_sim_binding.csv`): + +```bash +for r in joint_s1:full joint_s1:plain null_s1:full; do sbatch --export=ALL,SIM=sim_${r%%:*},MODEL=${r##*:} --output=fit_${r%%:*}_${r##*:}.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun; done +``` + +Then, locally after downloading the fits' outputs and each simulation's +truth files (`simulate_config.yaml`, `tfs_sim_parameters.csv`, +`tfs_sim_growth_parameters.csv`, `tfs_sim_growth_titrant_scale.csv`, +`tfs_sim_binding_offsets.csv`): + +```bash +python analyze.py runs/fit_joint_s1_full:runs/sim_joint_s1 runs/fit_joint_s1_plain:runs/sim_joint_s1 runs/fit_null_s1_full:runs/sim_null_s1 --out out +``` + +`analyze.py` writes `out/params.csv` (each fitted parameter against its +truth, with its 95% interval) and `out/theta.csv` (theta RMSE, bias and +95% coverage by group: wt, spikes, singles). Theta is compared in binding +units, the fitted curve against the truth curve at the growth +concentrations. + +## Results + +### Run 1 (2026-10-17, commit of 2026-10-16) + +Simulations from `sim_realistic_s2`'s config, seed 1; 955 stage-1 +genotypes (wt, 8 spikes with binding, 946 singles); M42I/H74A survived +this time. Final MAP losses: joint/full 780,800 (converged), joint/plain +798,807 (converged), null/full 784,304 (still descending at step 1e-5). + +Shared parameters, median [95%] against the truth: + +| | truth | joint/full | joint/plain | null/full (truth) | +|---|---|---|---|---| +| m kanR+kan | -0.0141 | -0.0144 [-0.0145, -0.0143] | -0.0187 | -0.0167 (-0.0141) | +| m pheS+4CP | +0.0144 | +0.0164 [0.0160, 0.0167] | +0.0155 | +0.0169 (+0.0144) | +| kanR warp log a / x0 | 2.08 / 0.50 | 2.20 / 0.57 | - | 2.78 / 0.70 (linear) | +| pheS warp log a / x0 | 0.41 / 0.40 | -0.14 [-0.68, 0.44] / 1.25 [0.28, 2.16] | - | 2.71 / 0.70 (linear) | +| s | 0.50 | 0.42 [0.37, 0.48] | (1) | 0.25 [0.17, 0.35] (1) | +| offset scale tau | 0.10 | 0.25 [0.11, 0.51] | - | 0.34 (0) | +| k, every condition | | 0.002-0.005 high | 0.003-0.005 high | 0.000-0.003 high | + +Theta RMSE of the median (95% coverage), binding units, and the RMSE as +growth sees it (each curve at its own s times nominal): + +| | joint/full | joint/plain | null/full | +|---|---|---|---| +| singles (946) | 0.072 (0.18); in vivo 0.074 | 0.130 (0.04); in vivo 0.091 | 0.293 (0.04); in vivo 0.302 | +| spikes (8) | 0.059 (0.31) | 0.047 (0.31) | 0.226 (0.09) | +| wt | 0.055 (0.25) | 0.137 (0.00) | 0.151 (0.13) | + +What it says: + +- **On the joint simulation the pieces work and are needed.** The full + model gets kanR's m within 2% and its warp close (a 9.0 against 8, + x0 0.57 against 0.50), s within 16%, and halves the singles' theta + error of the plain model, which steepens kanR's m by 33% to bend a + linear map round a switch. pheS's graded warp is weakly identified + (wide, covers) and its m is 14% steep. Most offsets are near their + truth; tau is high (0.25 against 0.1). +- **The null fit finds a wrong mode.** With linear truth, both markers get + the same steep warp (a about 16, x0 0.70), s drops to 0.25 and the + spikes take large offsets (K84L -0.32, M42I/K84L -0.38); theta is off by + 0.3 whether read in binding or growth units. A warp shared by both + markers trades exactly with every single's curve (growth sees f(theta), + and singles have no binding), so only the binding genotypes pin it, and + the offsets let those same genotypes disagree with their binding. On + the joint arm the two markers' warps differ, so the singles pin their + difference and binding only sets the common scale. +- **Intervals are far too narrow everywhere** (theta 18-31% coverage + where the median error is small; m's interval excludes the truth), + the MAP plus Laplace holding the shared parameters' error out of each + genotype, as before. +- k sits high in every condition, the k/dk_geno slide, as in every stage + 1 so far. + +### Run 2 (2026-10-17): trap or degeneracy? + +`fit_null_s1_plain` (the null simulation's own model) converged at a MAP +loss of 783,286, about 1,000 nats below `fit_null_s1_full` (784,304, still +descending at step 1e-5 when it hit the epoch cap). The full model nests +the plain one at a cost of tens of nats in prior terms, so run 1's null +fit was trapped, not unidentified: the data do separate the linear truth +from the steep shared warp. It reached the steep warps early, while the +step size was large. + +The plain fit's theta is good (singles RMSE 0.056, bias -0.002; spikes +0.036) but its intervals still undercover (singles 0.10), and m is about +20% steep in both markers with the true model (kanR -0.0173 against +-0.0141, pheS +0.0173 against +0.0144) while k sits 0.005 high: an open +question for stage 1 itself, separate from the warp. + +Next (run 3): the full model started from the plain fit's MAP +(`stage1.py --init_from`, `stage1.srun` `INIT=`), on both simulations. Pass: +the null fit returns to the plain fit's loss or below with near-linear +warps, s near 1 and small offsets; the joint fit reaches run 1's 780,800 +or below. + +```bash +for s in null joint; do sbatch --export=ALL,SIM=sim_${s}_s1,MODEL=full,INIT=fit_${s}_s1_plain --output=fit_${s}_s1_full_from_plain.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun; done +``` + +### Run 3 (2026-10-17): the full model started from the plain fit + +Both converged below the fits they had to beat: null 783,021 (plain +783,286; run 1's full 784,304), joint 780,782 (run 1's full 780,800). +Starting the full model from the plain one is the recipe. + +| | truth | null, full from plain | truth | joint, full from plain | +|---|---|---|---|---| +| m kanR+kan | -0.0141 | -0.0176 | -0.0141 | -0.0144 | +| m pheS+4CP | +0.0144 | +0.0180 | +0.0144 | +0.0166 | +| kanR warp a / x0 | linear | 2.6 / 0.59 | 8 / 0.50 | 9.0 [8.7, 9.5] / 0.53 [0.50, 0.56] | +| pheS warp a / x0 | linear | 1.7 / 0.57 | 1.5 / 0.40 | 0.9 [0.2, 4.3] / 0.59 [-0.24, 1.32] | +| log s | 0 | -0.05 [-0.22, 0.11] | -0.69 | -0.80 [-0.94, -0.66] | +| offsets covered | 0 | 9 of 9 | | 8 of 9 | +| tau | 0 | 0.17 [0.05, 0.52] | 0.10 | 0.22 [0.09, 0.50] | +| singles theta RMSE (95% cov.) | | 0.077 (0.14) | | 0.070 (0.17) | +| spikes theta RMSE (95% cov.) | | 0.056 (0.64) | | 0.039 (0.48) | + +- **The null comes back.** The warps sit at about the prior's median + steepness (a about 2.7, close to linear on [0, 1]), s covers 1, every + offset covers 0. The extra freedom costs theta a little against the + plain fit (singles 0.077 against 0.056). +- **The joint truth comes back.** kanR's switch, s and the offsets are + recovered; pheS's graded warp is weakly identified (wide, covers). +- **Still open, both independent of the warp:** + 1. m is steep: 22-25% in both markers on the null simulation, with the + plain (true) model as well; pheS 15% on the joint one (kanR right). + The simulator and the fit use the same congression rules (homodimer, + dilution). k is high by 0.002-0.005, the k/dk_geno slide. + 2. Theta intervals undercover badly (singles 14-17%, wt 13-25%) while + the median error is small: the MAP plus Laplace leaves the shared + parameters' error (m above all) out of each genotype's interval. + +### Run 4 (2026-10-17): the loss at the true k and m + +The null simulation, plain (true) model, k and m pinned at their truth: +from `fit_null_s1_plain`'s MAP, converged at 784,031; from scratch, +784,068 (still descending at step 1e-5). The free fit is 783,286, so the +truth costs about 745 nats: stage 1 prefers the steep m, by far more +than four parameters' noise. At the truth, theta is much better +(singles RMSE 0.020 against 0.056, coverage 0.70 against 0.10; spikes +0.012, wt 0.011): the steep m is most of stage 1's theta error. + +Attribution (`attribute.py`, pinned minus free at each MAP; total -720): + +| | nats | +|---|---| +| growth, pheS+4CP | -396 | +| growth, pheS-4CP | -106 | +| growth, kanR+kan / kanR-kan | -64 / -31 | +| log prior (theta population scales and offsets, mostly) | -161 | +| binding | +37 | + +- pheS+4CP's loss is in the deep singles (> 1e5 reads), at 0.1 and 1 mM + IPTG (-261 together), growing with selection time, and in rows whose + true theta is below 0.1 (-366 of -396): the cells that grow slowest + under 4CP, whose counts their co-transformed partners come to + dominate. Fitted lambda is 0.335 free and 0.301 pinned, against the + simulated 0.357. +- The prior's share: at the true m the singles' curves must spread wider, + which the theta population scales charge for; shrinkage of the theta + range is part of what a steeper m buys. +- Binding agrees with the truth better (+37). + +So the steep m most likely compensates for how the fit's congression +mixture (fixed co-resident sets, `congression_sets` [12, 3, 1]) describes +slow growers, plus some shrinkage. The simulator and the fit share the +congression rules (homodimer, dilution); what differs is that the fit +approximates each cell's co-residents by a few fixed draws. + +### Run 5 (2026-10-17): without congression + +The control: the null simulation without congression +(`make_sim.py --arm nocong`, `transformation_poisson_lambda` 0, same seed +and library), fitted with the plain model without the mixture +(`stage1.py --no_mixture`, `NOMIX=1`), free and with k and m pinned at the +truth. If m comes back unbiased and the truth costs a few nats, the +mixture's approximation is the cause. + +```bash +cd /gpfs/projects/harmslab/harms/studies/joint-fit-sim && sim=$(sbatch --parsable --export=ALL,ARM=nocong,SEED=1 --output=sim_nocong.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/simulate.srun) && echo "sim $sim" +``` + +```bash +sbatch --dependency=afterok:$sim --export=ALL,SIM=sim_nocong_s1,MODEL=plain,NOMIX=1 --output=fit_nocong_s1_plain_single.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun && sbatch --dependency=afterok:$sim --export=ALL,SIM=sim_nocong_s1,MODEL=plain,NOMIX=1,PIN=1 --output=fit_nocong_s1_plain_single_pinkm.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun +``` + +Results. Free 776,488, pinned at the truth 776,871 (both converged): the +truth still costs 383 nats, half of run 4's 745. m is 14% steep in both +markers (kanR -0.0161, pheS +0.0164), against 22-25% with congression. At +the truth theta is nearly exact (singles RMSE 0.010, 95% coverage 0.91; +spikes 0.006, wt 0.002); free, singles 0.046 (coverage 0.11). + +Attribution (pinned minus free, total -357): growth -235, spread over all +four conditions and the deep singles with no hot spot; log prior -144, of +which the theta population scales and offsets take about -130 (logit low, +logit delta and log K, about -21 to -23 each for scale and offsets); +binding +22. + +So the bias has two parts of about equal size: + +1. **The MAP over-shrinks the theta population** (no congression needed). + The population scales of the singles' Hill parameters run small, which + compresses their theta range, and a steeper m restores the growth swing; + at the true m the curves need their real spread, which the population + prior charges for. The doubles' pull on m (the m-bias study) works the + same way. A hierarchical MAP has no reason to get those scales right + (the funnel); `hill_relative` holds them for the same reason + (`theta_*_hyper_scale_fixed`), `hill_geno` only holds log(n)'s. +2. **The congression mixture's approximation** adds the rest (run 4's + pheS slow growers). + +Correction to the reading above (checked before run 6): the MAP's +population SDs are not small. Free, logit theta_low's is 6.8, logit +delta's 8.4 and log K's 1.22, against the truth's spread among singles +(SD 2.6, 3.9 and 0.50; robust SD 0.56, 0.63, 0.49: a bulk with outliers, +the simulator's stuck-bound, never-binds and inverted genotypes); pinned +at the true k and m they are larger still (9.5, 10.8, 1.38). What +compresses theta is the population location: the free fit's logit +theta_low location is 2.04, the default hyperprior's location (2.0 +- 2), +so its typical single runs from theta 0.88 to 0.085 where the truth runs +from 0.99 to 0.01, about 20% compression. Growth alone cannot tell those +ranges apart while m is free, and eight binding genotypes set the scale, +so the location rides its prior; the SDs' tight HalfNormal priors (log +K's at 0.1) add their cost at the truth. + +### Run 6 (2026-10-17): held SDs, moved locations + +`hill_geno` now holds any population SD (`theta_*_hyper_scale_fixed`). +The no-congression simulation, plain model without the mixture, free k +and m, three arms (`stage1.py --set_priors`, `stage1.srun` `PRIORS`/`TAG`): + +| arm | priors | +|---|---| +| `heldsd` | SDs held at the truth's spread: logit low 2.6, logit delta 3.9, log K 0.5, log n 0.3 | +| `loc` | location priors at the truth's medians: logit low loc 4.5, logit delta loc -9.0 (scales 2 as before) | +| `both` | both | + +Pass: m within a few percent of the truth and theta near run 5's pinned +fit (singles RMSE 0.010). Compare each arm's MAP loss with run 5's free +fit (776,488). + +```bash +cd /gpfs/projects/harmslab/harms/studies/joint-fit-sim +``` + +```bash +SD=theta_logit_low_hyper_scale_fixed=2.6:theta_logit_delta_hyper_scale_fixed=3.9:theta_log_hill_K_hyper_scale_fixed=0.5:theta_log_hill_n_hyper_scale_fixed=0.3; LOC=theta_logit_low_hyper_loc_loc=4.5:theta_logit_delta_hyper_loc_loc=-9.0; for a in heldsd:$SD loc:$LOC both:$SD:$LOC; do sbatch --export=ALL,SIM=sim_nocong_s1,MODEL=plain,NOMIX=1,TAG=${a%%:*},PRIORS=${a#*:} --output=fit_nocong_s1_plain_single_${a%%:*}.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun; done +``` + +Results (priors checked in each `tfs_configure_priors.csv`; `heldsd` hit +the epoch cap still descending at step 1e-5, the others converged; MAP +losses are not comparable across arms, whose priors differ): + +| arm | m kanR (truth -0.0141) | m pheS (truth +0.0144) | singles theta RMSE (cov.) | +|---|---|---|---| +| free (run 5) | -0.0161 | +0.0164 | 0.046 (0.11) | +| `loc` | -0.0160 | +0.0164 | 0.046 (0.11) | +| `heldsd` | -0.0184 | +0.0187 | 0.085 (0.04) | +| `both` | -0.0184 | +0.0188 | 0.088 (0.04) | + +Neither fixes it; holding the SDs at the truth's spread makes it worse. +What the free fit does, by regression on the truth: the singles' theta is +0.048 + 0.90 theta_true (the fit pinned at the true k and m: 0.000 + +0.997), and m is stretched by about 1/0.90; dk_geno is right in both +(slope 0.99). For a single without binding this is close to a symmetry of +the growth model: an affine squeeze of a Hill curve is a Hill curve, k +absorbs the constant, m the slope. Only the binding genotypes (wt and +eight spikes) resist it. What pushes along it is the theta population +prior in logit space: plateaus near 0.99 and 0.01 (logit +-4.6, outliers +further) cost prior density, a squeezed curve less. That is why holding +the SDs at 2.6 / 3.9 / 0.5 (smaller than the free fit's 6.8 / 8.4 / 1.2) +squeezes harder, and why moving the location does nothing at SDs near 7. +The congression mixture's approximation (run 4) adds to the same +direction. + +So the remedies are along that direction, not at the priors' values: +pooling weak enough not to penalize saturated plateaus (held SDs larger +than the free fit's), more genotypes whose binding pins theta's scale +(in-library binding genotypes, like the real data's G65H and V95N), or a +parameterization whose population prior does not prefer the middle of +[0, 1]. + +### Run 7 (2026-10-17): weaker pooling, more binding + +The no-congression simulation, plain model without the mixture, free k +and m, against run 5's free fit: + +| arm | change | +|---|---| +| `weakpool` | theta population SDs held large: logit low 10, logit delta 10, log K 2 (log n 0.5, as before) | +| `lib2` | 2 extra binding singles (the real data have two in-library binding genotypes, G65H and V95N) | +| `lib20` | 20 extra binding singles | +| `weakpool_lib20` | both | + +The extra binding singles are random singles with growth data, measured on +the lab's 2 uM grid from their true Hill curves with the spikes' noise +(`stage1.py --add_binding_singles`, `stage1.srun` `ADDB=`; the table is +`binding.csv` in the fit's folder, and `analyze.py` scores them as their +own group). The growth data are run 5's, so the comparison is clean. +Pass: m within a few percent of the truth and the singles' theta slope on +the truth near 1 (run 5 free: 0.90). + +```bash +cd /gpfs/projects/harmslab/harms/studies/joint-fit-sim +``` + +```bash +W=theta_logit_low_hyper_scale_fixed=10:theta_logit_delta_hyper_scale_fixed=10:theta_log_hill_K_hyper_scale_fixed=2; for a in weakpool:$W:0 lib2::2 lib20::20 weakpool_lib20:$W:20; do t=${a%%:*}; r=${a#*:}; n=${r##*:}; p=${r%:*}; sbatch --export=ALL,SIM=sim_nocong_s1,MODEL=plain,NOMIX=1,TAG=$t,PRIORS=$p,ADDB=$([ "$n" = 0 ] || echo $n) --output=fit_nocong_s1_plain_single_$t.out /gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun; done +``` + +Results (all converged; settings checked in each fit's priors and binding +table): + +| arm | m steep, kanR / pheS | singles' theta slope on truth | singles theta RMSE (95% cov.) | spikes | wt | +|---|---|---|---|---|---| +| free (run 5) | 14% / 14% | 0.900 | 0.046 (0.11) | 0.023 | 0.041 | +| `weakpool` | 11% / 11% | 0.914 | 0.039 (0.13) | 0.020 | 0.035 | +| `lib2` | 11% / 10% | 0.924 | 0.036 (0.15) | 0.018 | 0.031 | +| `lib20` | 6% / 6% | 0.961 | 0.021 (0.36) | 0.012 | 0.014 | +| `weakpool_lib20` | 5% / 5% | 0.964 | 0.019 (0.38) | 0.011 | 0.013 | +| pinned at truth (run 5) | 0 | 0.997 | 0.010 (0.91) | 0.006 | 0.002 | + +- **Binding genotypes are the lever.** Twenty binding singles (spanning + the library's theta range at random) cut the m bias from 14% to 6% and + halve the singles' theta error; two do about as much as weak pooling. + Each binding curve pins one genotype's theta on an absolute scale, and + the squeeze of run 6 has to fight all of them. +- **Weak pooling helps a little** (14% to 11% alone, 6% to 5% with 20 + binding singles): the logit population prior is part of the push, not + most of it. +- **Not yet enough:** 5% left and theta coverage 0.36-0.38, against 0.91 + at the true k and m. The remaining m error is shared by every genotype + and missing from their intervals (the Laplace at the MAP), as in every + run so far. + +For the real data, this argues for measuring binding on more singles, +about twenty spread over the theta range, beyond the nine spikes and the +two in-library genotypes. The congression part (run 4) is untested with +binding singles. + +### Run 8 (2026-10-17): binding singles with congression, more of them, and the whole recipe + +No separate interval study: at the true k and m the singles' intervals +cover 0.91 (run 5), so the per-genotype uncertainty is right and the +undercoverage is the shared parameters' bias (the truth sits hundreds of +nats from the MAP, outside any honest posterior). Fixing the bias is the +interval fix. Config-only runs on the existing simulations: + +| run | asks | +|---|---| +| `fit_null_s1_plain_lib20` | 20 binding singles with congression (null; plain model with the mixture; run 4's bias 22-25%) | +| `fit_null_s1_plain_weakpool_lib20` | the same with weak pooling | +| `fit_nocong_s1_plain_single_lib50` | 50 binding singles: does more binding keep closing the last 5%? | +| `fit_joint_s1_plain_lib20` then `fit_joint_s1_full_from_plain_lib20` | the recipe on the joint truth (warps, s, offsets) with 20 binding singles | + +The chained full fit builds its own `binding.csv` with the same seed from +the same stage-1 genotypes, so it measures the same twenty singles. + +```bash +cd /gpfs/projects/harmslab/harms/studies/joint-fit-sim && R=/gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun && W=theta_logit_low_hyper_scale_fixed=10:theta_logit_delta_hyper_scale_fixed=10:theta_log_hill_K_hyper_scale_fixed=2 +``` + +```bash +sbatch --export=ALL,SIM=sim_null_s1,MODEL=plain,TAG=lib20,ADDB=20 --output=fit_null_s1_plain_lib20.out $R && sbatch --export=ALL,SIM=sim_null_s1,MODEL=plain,TAG=weakpool_lib20,PRIORS=$W,ADDB=20 --output=fit_null_s1_plain_weakpool_lib20.out $R && sbatch --export=ALL,SIM=sim_nocong_s1,MODEL=plain,NOMIX=1,TAG=lib50,ADDB=50 --output=fit_nocong_s1_plain_single_lib50.out $R && j=$(sbatch --parsable --export=ALL,SIM=sim_joint_s1,MODEL=plain,TAG=lib20,ADDB=20 --output=fit_joint_s1_plain_lib20.out $R) && sbatch --dependency=afterok:$j --export=ALL,SIM=sim_joint_s1,MODEL=full,INIT=fit_joint_s1_plain_lib20,ADDB=20 --output=fit_joint_s1_full_from_plain_lib20.out $R +``` + +Results (all converged except `fit_joint_s1_full_from_plain_lib20`, at the +floor step size 1e-6 when it hit the epoch cap, still creeping): + +| run | m steep, kanR / pheS | singles theta RMSE (95% cov.) | wt | +|---|---|---|---| +| null, plain (run 2) | 23% / 20% | 0.056 (0.10) | 0.040 | +| null, plain, 20 binding singles | 13% / 11% | 0.037 (0.31) | 0.018 | +| null, plain, weak pooling, 20 binding singles | 13% / 11% | 0.036 (0.33) | 0.017 | +| no congression, 20 binding singles (run 7) | 6% / 6% | 0.021 (0.35) | 0.014 | +| no congression, 50 binding singles | 5% / 5% | 0.017 (0.51) | 0.010 | +| joint, full from plain (run 3) | 2% / 15% | 0.070 (0.17) | 0.057 | +| joint, plain, 20 binding singles | 19% / -3% | 0.123 (0.13) | 0.129 | +| joint, full from plain, 20 binding singles | 1% / 7% | 0.048 (0.37) | 0.029 | + +On the joint truth with 20 binding singles: log s -0.74 [-0.78, -0.69] +against -0.69 (covered, much tighter than without); kanR's warp a 8.3 +[8.2, 8.4] against 8 and x0 0.535 [0.52, 0.55] against 0.50 (tight, near, +not covered); pheS's graded warp flips to log a -0.83 [-1.8, 0.05] +against 0.41 (not covered; it was wide and covered without); tau 0.23 +against 0.10. + +- **With congression, binding singles halve the bias** (21% to 12%) but + leave about twice the no-congression bias: the congression mixture's + approximation is now the largest piece. Weak pooling adds nothing here. +- **More binding has diminishing returns:** 50 singles take the + no-congression bias from 6% to 5%, a floor of about 5% from the + population prior and the squeeze. +- **The recipe on the joint truth works best with binding singles:** + kanR's m within 1%, s covered, theta error down a third; pheS's m still + 7% steep and its weak warp the least identified piece. +- Coverage rises with binding (0.10 to 0.37, 0.51 with 50) but stays far + from 0.95, as the bias it tracks. + +Next: the congression mixture's approximation (`congression_sets`, the +fixed co-resident draws), on the null simulation with 20 binding singles. + +### Run 9 (2026-10-18): more co-resident draws + +`tfs-configure-model --congression_sets` (new) sets the mixture's fixed +co-resident draws; `stage1.py --congression_sets`, `stage1.srun` +`CSETS=` (':'-separated). The congression null simulation, plain model +with the mixture and 20 binding singles, against +`fit_null_s1_plain_lib20` (default 12 3 1, 16 sets: m 13% / 11% steep): + +| arm | sets (1, 2, 3 co-residents) | total | +|---|---|---| +| `cs75` | 50 20 5 | 75 | +| `cs140` | 100 30 10 | 140 | +| `cs430` | 300 100 30 | 430 | + +On a subsampled stage 1, 140 sets took 2.1x the default's wall time +(fixed overheads included); the default fits took 20-35 minutes here. +Pass: the bias falls toward the no-congression fit's 6% with 20 binding +singles (run 7). If it does not move, the error is in the mixture's form +(the fixed-set quadrature, the homodimer rule's equal shares), not in the +number of draws. + +```bash +cd /gpfs/projects/harmslab/harms/studies/joint-fit-sim && R=/gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun && for a in cs75:50:20:5 cs140:100:30:10 cs430:300:100:30; do sbatch --export=ALL,SIM=sim_null_s1,MODEL=plain,ADDB=20,TAG=${a%%:*},CSETS=${a#*:} --output=fit_null_s1_plain_${a%%:*}.out $R; done +``` + +To check each arm's sets once it has configured: + +```bash +for t in cs75 cs140 cs430; do echo "$t: $(python -c "import yaml; print(yaml.safe_load(open('fit_null_s1_plain_$t/tfs_configure_config.yaml'))['components']['congression_sets'])")"; done +``` + +Results. All three MAPs finished (75 and 140 sets at the epoch cap, still +creeping; 430 converged), and all three then ran the GPU out of memory in +the Laplace Hessian (`tfs-sample-posterior`, `_chunked_hessian`: every +cell class is evaluated per Hessian probe), so there are no posteriors or +theta predictions; m comes from each MAP (`two_stage.km_from_map`). + +| sets | total | m steep, kanR / pheS | +|---|---|---| +| 12 3 1 (run 8, `fit_null_s1_plain_lib20`) | 16 | 13.4% / 11.0% | +| 50 20 5 | 75 | 9.7% / 8.8% | +| 100 30 10 | 140 | 9.5% / 8.8% | +| 300 100 30 | 430 | 8.8% / 8.4% | +| no congression, same 20 binding singles (run 7) | - | 5.7% / 5.9% | + +- **The number of draws is about a third of the congression excess,** and + nearly all of that third comes by 75 sets. +- **The rest (about 3 points) does not shrink with more draws:** it is in + the mixture's form, not its sample size. Candidates: the simulator's + cells carry their co-residents' real abundances, while the fit's sets + are drawn from the pool composition and weighted by the design; the + homodimer rule's equal plasmid shares; the classes beyond three + co-residents the fit leaves out. +- **Memory:** many sets need the Laplace Hessian chunked smaller (or a + posterior that does not take one) before they can be the default. + +### Theta so far (2026-10-18) + +Growth-only singles (no binding of their own), from the fits with +predictions; SD of z is the SD of (error / posterior SD), 1 for honest +intervals: + +| simulation | binding singles | theta RMSE | slope on truth | 95% coverage | median 95% width | SD of z | +|---|---|---|---|---|---|---| +| no congression | 0 | 0.046 | 0.90 | 0.11 | 0.039 | 4.3 | +| | 2 | 0.036 | 0.92 | 0.15 | 0.033 | 4.2 | +| | 20 | 0.021 | 0.96 | 0.36 | 0.031 | 2.7 | +| | 50 | 0.017 | 0.97 | 0.51 | 0.032 | 2.2 | +| | 0, true k and m | 0.010 | 1.00 | 0.91 | 0.047 | 0.9 | +| congression (null) | 0 | 0.056 | 0.89 | 0.10 | 0.033 | 6.4 | +| | 20 | 0.037 | 0.94 | 0.31 | 0.032 | 4.3 | +| | 0, true k and m | 0.020 | 0.98 | 0.70 | 0.042 | 1.8 | +| joint truth | 0 | 0.070 | 0.85 | 0.17 | 0.053 | 5.2 | +| | 20 | 0.048 | 0.92 | 0.37 | 0.044 | 3.8 | + +Theta's median is good to a few hundredths and its error is mostly the +shared affine squeeze (slope below 1); its intervals are 2-6 times too +narrow. With congression the intervals stay about 1.8 times too narrow +even at the true k and m: the mixture's approximation also adds +per-genotype error the model does not carry. + +### Run 10 (2026-10-18): how theta scales with binding singles + +`analyze.py` now also reports each group's median 95% width, SD of z and +slope on the truth; `stage1.srun` takes `BSEED` (which random singles). +Arms: the no-congression simulation with 2, 5, 10 and 20 binding singles, +three random choices each (`lib2` and `lib20` from run 7 are choice 1), +and the congression null with 5 and 10 (choice 1; 20 is run 8's). Twelve +fits of about 30 minutes. + +```bash +cd /gpfs/projects/harmslab/harms/studies/joint-fit-sim && R=/gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim/stage1.srun +``` + +```bash +for n in 2 5 10 20; do for b in 1 2 3; do if [[ ($n == 2 || $n == 20) && $b == 1 ]]; then continue; fi; sbatch --export=ALL,SIM=sim_nocong_s1,MODEL=plain,NOMIX=1,ADDB=$n,BSEED=$b,TAG=lib${n}_b${b} --output=fit_nocong_s1_plain_single_lib${n}_b${b}.out $R; done; done; for n in 5 10; do sbatch --export=ALL,SIM=sim_null_s1,MODEL=plain,ADDB=$n,TAG=lib$n --output=fit_null_s1_plain_lib$n.out $R; done +``` + +Results (`binding_scaling.py runs --out out`: `out/binding_scaling.csv` +and `out/YYYY-MM-DD_binding-scaling.pdf`). Growth-only singles; mean over +the three random choices (range in brackets), one choice on the +congression simulation: + +| simulation | binding singles | m steep | theta RMSE | slope | 95% coverage | SD of z | +|---|---|---|---|---|---|---| +| no congression | 0 | 14% | 0.046 | 0.90 | 0.11 | 4.3 | +| | 2 | 11% | 0.036 [0.036, 0.037] | 0.92 | 0.15 | 4.1 | +| | 5 | 9% | 0.030 [0.029, 0.032] | 0.94 | 0.20 | 3.7 | +| | 10 | 8% | 0.027 [0.025, 0.029] | 0.95 | 0.23 | 3.6 | +| | 20 | 6% | 0.021 [0.021, 0.022] | 0.96 | 0.34 | 2.9 | +| | 50 | 5% | 0.017 | 0.97 | 0.51 | 2.2 | +| | true k and m | 0 | 0.010 | 1.00 | 0.91 | 0.9 | +| congression | 0 | 21% | 0.056 | 0.89 | 0.10 | 6.4 | +| | 5 | 17% | 0.046 | 0.91 | 0.18 | 5.4 | +| | 10 | 17% | 0.046 | 0.92 | 0.20 | 5.2 | +| | 20 | 12% | 0.037 | 0.94 | 0.31 | 4.3 | +| | true k and m | 0 | 0.020 | 0.98 | 0.70 | 1.8 | + +- **Which singles are chosen barely matters** (RMSE within 0.003 across + choices at every count). +- **The error falls as one over the square root:** without congression, + theta RMSE = 0.010 + 0.053 / sqrt(n + 2) fits every count from 0 to 50 + (the 0.010 is the fit at the true k and m; the "+2" is what wt and the + eight spikes are worth in binding singles). The binding curves average + down the shared scale error. Halving the excess at 20 would take about + 90 binding singles. +- **Coverage climbs slowly** (0.11 to 0.51 at 50) and SD of z falls only + to 2.2: binding alone does not give honest intervals at any realistic + count. With congression everything is worse and the intervals stay about + 1.8 times too narrow even at the true k and m. + diff --git a/planning/studies/joint-fit-sim/analyze.py b/planning/studies/joint-fit-sim/analyze.py new file mode 100644 index 00000000..c39ba93b --- /dev/null +++ b/planning/studies/joint-fit-sim/analyze.py @@ -0,0 +1,138 @@ +""" +Score joint stage 1 fits against the simulation's truth (README). + + python analyze.py RUN_DIR:SIM_DIR [RUN_DIR:SIM_DIR ...] --out out + +For each fit: k, m, the warps, log s, the binding offsets and their scale +against the truth files, and theta (in binding units, from the fitted +curves at the growth concentrations) against the truth Hill curves for +wt, the spikes and the singles: RMSE of the median and 95% coverage. +Writes out/params.csv and out/theta.csv and prints both. +""" + +import argparse +import os + +import numpy as np +import pandas as pd + +from tfscreen.tfmodel import two_stage as ts + +Q_LO, Q_MID, Q_HI = "q0.025", "q0.5", "q0.975" + + +def _read(path): + return pd.read_csv(path) if os.path.exists(path) else None + + +def hill(p, conc): + c = np.where(np.asarray(conc) == 0, 1e-20, conc) + occ = 1 / (1 + np.exp(-p["hill_n"] * (np.log(c) - p["log_hill_K"]))) + return p["theta_low"] + (p["theta_high"] - p["theta_low"]) * occ + + +def param_rows(run, sim, name): + rows = [] + + def add(what, key, est, truth): + if est is None: + return + for _, r in est.iterrows(): + t = truth(r) if callable(truth) else truth + rows.append(dict(run=name, parameter=what, key=key(r), truth=t, + q0_5=r[Q_MID], lo=r[Q_LO], hi=r[Q_HI], + covered=bool(r[Q_LO] <= t <= r[Q_HI]))) + + gp = pd.read_csv(os.path.join(sim, "tfs_sim_growth_parameters.csv")).set_index("condition_rep") + for p, col in (("growth_k", "growth_k"), ("growth_m", "growth_m"), + ("warp_log_a", "warp_log_a"), ("warp_x0", "warp_x0")): + if col not in gp.columns: + continue + add(p, lambda r: r["condition_rep"], + _read(os.path.join(run, f"tfs_params_{p}.csv")), + lambda r, col=col: gp.loc[r["condition_rep"], col]) + scale = _read(os.path.join(sim, "tfs_sim_growth_titrant_scale.csv")) + log_s = 0.0 if scale is None else float(scale["ref"].iloc[0]) + add("log_s", lambda r: "global", + _read(os.path.join(run, "tfs_params_growth_log_conc_shift.csv")), log_s) + offs = _read(os.path.join(sim, "tfs_sim_binding_offsets.csv")) + off_truth = {} if offs is None else dict(zip(offs["genotype"], offs["offset"])) + add("binding_offset", lambda r: r["genotype"], + _read(os.path.join(run, "tfs_params_offset.csv")), + lambda r: off_truth.get(r["genotype"], 0.0)) + import yaml + with open(os.path.join(sim, "simulate_config.yaml")) as fh: + tau = float((yaml.safe_load(fh).get("binding_data") or {}) + .get("genotype_offset_sd", 0.0)) + add("binding_offset_scale", lambda r: "global", + _read(os.path.join(run, "tfs_params_offset_scale.csv")), tau) + return rows + + +def theta_rows(run, sim, name): + pred = pd.read_csv(os.path.join(run, "tfs_pred_theta.csv")) + truth = pd.read_csv(os.path.join(sim, "tfs_sim_parameters.csv")) + truth = truth.assign(genotype=truth["genotype"].astype(str)).set_index("genotype") + pred["genotype"] = pred["genotype"].astype(str) + pred = pred[pred["genotype"].isin(truth.index)].copy() + t = truth.loc[pred["genotype"]] + pred["truth"] = hill(t[["theta_low", "theta_high", "log_hill_K", "hill_n"]] + .reset_index(drop=True), + pred["titrant_conc"].to_numpy()).to_numpy() + lib = pd.read_csv(os.path.join(run, "tfs_configure_library.csv")) + spiked = set(lib.loc[lib["in_spiked_origin"].astype(bool), "genotype"].astype(str)) + bfile = os.path.join(run, "binding.csv") + bound = (set(pd.read_csv(bfile)["genotype"].astype(str)) - spiked + if os.path.exists(bfile) else set()) + pred["group"] = np.where(pred["genotype"] == "wt", "wt", + np.where(pred["genotype"].isin(spiked), "spike", + np.where(pred["genotype"].isin(bound), + "single, binding", + np.where(pred["genotype"].map(ts.is_single), + "single", "other")))) + pred["err"] = pred[Q_MID] - pred["truth"] + pred["covered"] = (pred[Q_LO] <= pred["truth"]) & (pred["truth"] <= pred[Q_HI]) + pred["width95"] = pred[Q_HI] - pred[Q_LO] + pred["z"] = pred["err"] / ((pred["q0.841"] - pred["q0.159"]) / 2) + + def slope(d): + return float(np.polyfit(d["truth"], d[Q_MID], 1)[0]) if len(d) > 2 else np.nan + + out = (pred.groupby("group") + .agg(n_genotypes=("genotype", "nunique"), + rmse=("err", lambda e: float(np.sqrt(np.mean(e ** 2)))), + bias=("err", "mean"), + coverage95=("covered", "mean"), + median_width95=("width95", "median"), + sd_z=("z", "std")) + .reset_index()) + out["slope"] = out["group"].map( + {g: slope(d) for g, d in pred.groupby("group")}) + out.insert(0, "run", name) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("pairs", nargs="+", help="RUN_DIR:SIM_DIR") + ap.add_argument("--out", default="out") + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + params, thetas = [], [] + for pair in args.pairs: + run, sim = pair.split(":") + name = os.path.basename(os.path.normpath(run)) + params.extend(param_rows(run, sim, name)) + thetas.append(theta_rows(run, sim, name)) + p = pd.DataFrame(params) + th = pd.concat(thetas, ignore_index=True) + p.to_csv(os.path.join(args.out, "params.csv"), index=False) + th.to_csv(os.path.join(args.out, "theta.csv"), index=False) + pd.set_option("display.width", 200) + print(p.to_string(index=False, float_format="%+.4f")) + print() + print(th.to_string(index=False, float_format="%.3f")) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/joint-fit-sim/attribute.py b/planning/studies/joint-fit-sim/attribute.py new file mode 100644 index 00000000..8e9c04b5 --- /dev/null +++ b/planning/studies/joint-fit-sim/attribute.py @@ -0,0 +1,140 @@ +""" +Where two stage 1 fits differ in log density (README, run 4): the count +log likelihood of every growth row, the binding log likelihood of every +binding genotype and the log prior of every latent site, at each fit's +MAP; their differences summed by marker, genotype class, IPTG and tube. + + python attribute.py runs/fit_null_s1_plain runs/fit_null_s1_plain_from_plain_pinkm --out out/attr + +Each folder is a stage1.py fit downloaded under runs/ (its config, +priors, guesses, library, tfs_growth.csv and MAP params), next to the +simulation folders its config names. The difference is the second minus +the first: negative where the second fit explains the data worse. +""" + +import argparse +import os +import warnings + +warnings.filterwarnings("ignore") + +import jax # noqa: E402 +import jax.numpy as jnp # noqa: E402 +import numpy as np # noqa: E402 +import pandas as pd # noqa: E402 +from numpyro import handlers # noqa: E402 + +from tfscreen.tfmodel.configuration_io import read_configuration # noqa: E402 +from tfscreen.tfmodel.inference.run_inference import RunInference # noqa: E402 + +CLUSTER = "/gpfs/projects/harmslab/harms/studies/joint-fit-sim/" +IDX = ["replicate_idx", "time_idx", "condition_pre_idx", "condition_sel_idx", + "titrant_name_idx", "titrant_conc_idx", "genotype_idx"] +KEY = ["genotype", "replicate", "condition_sel", "titrant_conc", "t_sel"] + + +def local_config(fit): + """The fit's config with cluster paths pointed at the local runs/ copy.""" + runs = os.path.dirname(os.path.abspath(fit)) + text = open(os.path.join(fit, "tfs_configure_config.yaml")).read() + out = os.path.join(fit, "config_local.yaml") + open(out, "w").write(text.replace(CLUSTER, runs + "/")) + return out + + +def score(fit): + """Per-row growth and per-genotype binding log likelihood, per-site prior.""" + cwd = os.getcwd() + cfg = local_config(fit) + os.chdir(fit) + try: + orch, _ = read_configuration(os.path.basename(cfg)) + ri = RunInference(orch, 0) + with np.load("tfs_fit_model_params.npz") as p: + values = {k[:-len("_auto_loc")]: jnp.asarray(p[k]) for k in p.files + if k.endswith("_auto_loc")} + G = orch.data.num_genotype + batch = ri._unscaled_batch(jax.device_put(orch.data), jnp.arange(G)) + model = handlers.substitute(orch.jax_model, data=values) + tr = handlers.trace(handlers.seed(model, 0)).get_trace( + data=batch, priors=orch.priors) + site = tr["growth_obs"] + lp = np.asarray(site["fn"].log_prob(site["value"])) + df = orch.growth_df + rows = df[KEY].copy() + rows["ll"] = lp[tuple(df[c].to_numpy() for c in IDX)] + good = np.asarray(orch.data.growth.good_mask) + growth_total = float(lp[good].sum()) + + b = tr["binding_obs"] + blp = np.where(np.asarray(batch.binding.good_mask), + np.asarray(b["fn"].log_prob(b["value"])), 0.0) + names = list(orch.binding_tm.tensor_dim_names) + labels = orch.binding_tm.tensor_dim_labels[names.index("genotype")] + binding = pd.Series(blp.sum(axis=(0, 1)), index=[str(g) for g in labels]) + + priors = {n: float(s["fn"].log_prob(s["value"]).sum()) + for n, s in tr.items() + if s["type"] == "sample" and not s["is_observed"]} + return rows, binding, pd.Series(priors), growth_total + finally: + os.chdir(cwd) + + +def genotype_class(g, spikes, reads): + if g == "wt": + return "wt" + if g in spikes: + return "spike" + r = reads.get(g, 0) + for edge, name in ((1e4, "single <1e4"), (1e5, "single 1e4-1e5"), + (1e6, "single 1e5-1e6")): + if r < edge: + return name + return "single >=1e6" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("first") + ap.add_argument("second") + ap.add_argument("--out", default="out/attr") + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + a, ba, pa, ga = score(args.first) + b, bb, pb, gb = score(args.second) + m = a.merge(b, on=KEY, suffixes=("_1", "_2")) + m["d"] = m["ll_2"] - m["ll_1"] + lib = pd.read_csv(os.path.join(args.first, "tfs_configure_library.csv")) + spikes = set(lib.loc[lib["in_spiked_origin"].astype(bool), "genotype"].astype(str)) + reads = pd.read_csv(os.path.join(args.first, "tfs_growth.csv"), + usecols=["genotype", "counts"]).groupby("genotype")["counts"].sum() + m["class"] = m["genotype"].astype(str).map(lambda g: genotype_class(g, spikes, reads)) + m.to_csv(os.path.join(args.out, "per_row.csv"), index=False) + + db = (bb - ba) + dp = (pb - pa).dropna() + print(f"growth log likelihood: second - first {gb - ga:+.1f} ({len(m)} rows)") + print(f"binding log likelihood: second - first {db.sum():+.1f}") + print(f"log prior: second - first {dp.sum():+.1f}") + print(f"total: {gb - ga + db.sum() + dp.sum():+.1f}") + print("\n== binding, second - first, by genotype") + print(db.sort_values().round(1).to_string()) + print("\n== log prior, largest sites") + print(dp[dp.abs().sort_values(ascending=False).index].head(10).round(1).to_string()) + for by in (["condition_sel"], ["condition_sel", "class"], + ["condition_sel", "titrant_conc"], + ["condition_sel", "replicate", "t_sel"]): + t = m.groupby(by, observed=True)["d"].agg(["sum", "size"]).round(1) + t = t[t["size"] > 0] + print(f"\n== growth, second - first, by {', '.join(by)}") + print(t.to_string()) + g = m.groupby("genotype")["d"].sum().sort_values() + print("\n== genotypes losing most under the second fit") + print(g.head(10).round(1).to_string()) + print("\n== genotypes gaining most under the second fit") + print(g.tail(5).round(1).to_string()) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/joint-fit-sim/binding_scaling.py b/planning/studies/joint-fit-sim/binding_scaling.py new file mode 100644 index 00000000..2be1a0d2 --- /dev/null +++ b/planning/studies/joint-fit-sim/binding_scaling.py @@ -0,0 +1,119 @@ +""" +Theta of growth-only singles against the number of binding singles +(README, run 10). + + python binding_scaling.py runs --out out + +Scores every stage 1 fit in RUNS_DIR named fit_nocong_s1_plain_single[_libN[_bK]] +or fit_null_s1_plain[_libN[_bK]] (analyze.theta_rows; growth-only singles), +writes out/binding_scaling.csv (one row per fit: sim, n, pick, m_steep, +rmse, slope, cov95, width, sd_z) and draws RMSE, 95% coverage and SD of z +against n, each pick a point and the mean a line, with the fit at the true +k and m (run 5) as the floor and the curve floor + a / sqrt(n + 2). +""" + +import argparse +import datetime +import os + +import numpy as np +import pandas as pd + +import analyze +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + +plt.rc("pdf", fonttype=42) +plt.rc("font", size=14) +plt.rc("axes", titlesize=14, labelsize=16) +plt.rc("legend", fontsize=12) + +FLOOR = {"nocong": dict(rmse=0.010, cov95=0.910, sd_z=0.909), + "null": dict(rmse=0.020, cov95=0.699, sd_z=1.845)} +COLOR = {"nocong": "#0072B2", "null": "#D55E00"} +LABEL = {"nocong": "no congression", "null": "congression"} + + +BASES = {"nocong": "fit_nocong_s1_plain_single", "null": "fit_null_s1_plain"} + + +def parse(name, base): + """(n, pick) for base, base_libN or base_libN_bK; None otherwise.""" + rest = name[len(base):] + if rest == "": + return 0, 1 + if not rest.startswith("_lib"): + return None + parts = rest[4:].split("_b") + if not parts[0].isdigit() or (len(parts) > 1 and not parts[1].isdigit()): + return None + return int(parts[0]), (int(parts[1]) if len(parts) > 1 else 1) + + +def build_table(runs_dir): + rows = [] + for sim, base in BASES.items(): + sim_dir = os.path.join(runs_dir, f"sim_{sim}_s1") + truth = pd.read_csv(os.path.join(sim_dir, "tfs_sim_growth_parameters.csv")) + truth = truth.set_index("condition_rep")["growth_m"] + for d in sorted(os.listdir(runs_dir)): + run = os.path.join(runs_dir, d) + got = parse(d, base) if d.startswith(base) else None + if got is None or not os.path.exists(os.path.join(run, "tfs_pred_theta.csv")): + continue + th = analyze.theta_rows(run, sim_dir, d) + s = th[th["group"] == "single"].iloc[0] + m = pd.read_csv(os.path.join(run, "tfs_params_growth_m.csv")) + m = m.set_index("condition_rep")["q0.5"] + steep = np.mean([abs(m[c] / truth[c] - 1) for c in ("kanR+kan", "pheS+4CP")]) + rows.append(dict(sim=sim, n=got[0], pick=got[1], m_steep=100 * steep, + rmse=s["rmse"], slope=s["slope"], cov95=s["coverage95"], + width=s["median_width95"], sd_z=s["sd_z"])) + return pd.DataFrame(rows).sort_values(["sim", "n", "pick"]) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("runs_dir") + ap.add_argument("--out", default="out") + args = ap.parse_args() + os.makedirs(args.out, exist_ok=True) + r = build_table(args.runs_dir) + r.to_csv(os.path.join(args.out, "binding_scaling.csv"), index=False) + print(r.round(3).to_string(index=False)) + nocong = r[r["sim"] == "nocong"].groupby("n")["rmse"].mean() + a = float(np.mean((nocong - FLOOR["nocong"]["rmse"]) * np.sqrt(nocong.index + 2))) + print(f"no congression: excess RMSE = {a:.4f} / sqrt(n + 2)") + fig, axes = plt.subplots(1, 3, figsize=(15, 4.5)) + for ax, col, name in zip(axes, ["rmse", "cov95", "sd_z"], + ["theta RMSE", "95% coverage", "SD of z"]): + for sim, d in r.groupby("sim"): + x = d["n"] + 1 + ax.plot(x, d[col], "o", color=COLOR[sim], alpha=0.5, ms=5) + mean = d.groupby("n")[col].mean() + ax.plot(mean.index + 1, mean.values, "-", color=COLOR[sim], label=LABEL[sim]) + ax.axhline(FLOOR[sim][col], color=COLOR[sim], ls=":", lw=1) + if col == "rmse": + n = np.linspace(0, 60, 200) + ax.plot(n + 1, FLOOR["nocong"]["rmse"] + a / np.sqrt(n + 2), "--", + color="0.4", lw=1, label=f"{FLOOR['nocong']['rmse']} + {a:.3f}/sqrt(n+2)") + if col == "cov95": + ax.axhline(0.95, color="0.6", lw=0.8) + if col == "sd_z": + ax.axhline(1.0, color="0.6", lw=0.8) + ax.set_xscale("log") + ax.set_xticks([1, 3, 6, 11, 21, 51]) + ax.set_xticklabels(["0", "2", "5", "10", "20", "50"]) + ax.set_xlabel("binding singles") + ax.set_ylabel(name) + axes[0].legend(fontsize=10) + fig.suptitle("growth-only singles; dotted: fit at the true k and m", fontsize=13) + fig.tight_layout() + out = os.path.join(args.out, f"{datetime.date.today().isoformat()}_binding-scaling.pdf") + fig.savefig(out) + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/joint-fit-sim/make_sim.py b/planning/studies/joint-fit-sim/make_sim.py new file mode 100644 index 00000000..216b1e1f --- /dev/null +++ b/planning/studies/joint-fit-sim/make_sim.py @@ -0,0 +1,129 @@ +""" +Simulate configs for the joint-fit simulation test (README). + + python make_sim.py BASE_CONFIG OUT_DIR --arm joint|null|nocong [--seed 1] + +BASE_CONFIG is a full-size realistic simulate config +(planning/studies/full-size-sim/make_sim_config.py; on the cluster +full-sized-sims-3/sim_realistic_s2/simulate_config.yaml). Writes +OUT_DIR/simulate_config.yaml and OUT_DIR/spike_hill.csv: the base config +with its relative paths made absolute, plus + +- binding for every spike at the lab's 2 uM IPTG grid, from Hill curves + in spike_hill.csv (wt at the simulator's reference; the others drawn + here, seeded), noise 0.04 (the scatter within one real curve); +- arm joint: kanR+kan's growth a switch-like function of occupancy + (warp a = 8, x0 = 0.5), pheS+4CP's graded (a = 1.5, x0 = 0.4), growth + seeing IPTG at s = 0.5 times nominal, and one binding offset per spike + (SD 0.1); +- arm null: none of those (linear growth, s = 1, no offsets); +- arm nocong: null without congression (transformation_poisson_lambda 0), + the control for run 4's attribution, which put the steep m in pheS+4CP's + slow-growing, congression-dominated rows. + +The spikes' curves come from a params file so that their Hill truth is +the curve simulated (the stratified path pastes pool curves over the +spikes without patching tfs_sim_parameters.csv). +""" + +import argparse +import os + +import numpy as np +import pandas as pd +import yaml + +from tfscreen.genetics import library_composition_table + +# The lab's 2 uM binding grid (mM), from the 2026-10-15 binding table. +BINDING_IPTG = [0.0, 1e-6, 6e-6, 1e-5, 6e-5, 1e-4, 6e-4, 1e-3, 6e-3, 0.01, + 0.06, 0.1, 0.6, 1.0, 3.0, 5.0] +BINDING_NOISE = 0.04 + +ARMS = { + "joint": dict(warp={"kanR+kan": (8.0, 0.5), "pheS+4CP": (1.5, 0.4)}, + growth_titrant_scale=0.5, genotype_offset_sd=0.1), + "null": dict(warp={}, growth_titrant_scale=None, genotype_offset_sd=0.0), + "nocong": dict(warp={}, growth_titrant_scale=None, genotype_offset_sd=0.0, + transformation_poisson_lambda=0.0), +} + +# The simulator's hill_geno wt reference (get_sim_hyperparameters). +WT = dict(theta_low=0.99, theta_high=0.01, log_hill_K=-4.1, hill_n=2.0) + + +def absolute_paths(cf, base_dir): + for keys in (("design",), ("od600", "calibration"), ("library_genotypes",), + ("thermo_data",)): + d = cf + for k in keys[:-1]: + d = d.get(k) or {} + v = d.get(keys[-1]) if isinstance(d, dict) else None + if isinstance(v, str) and not os.path.isabs(v): + d[keys[-1]] = os.path.abspath(os.path.join(base_dir, v)) + return cf + + +def spike_curves(spikes, seed): + """wt at the reference; each other spike a random repressor-like curve.""" + rng = np.random.default_rng([seed, 2026]) + rows = [] + for g in spikes: + if g == "wt": + rows.append(dict(genotype=g, **WT)) + continue + low = rng.uniform(0.3, 0.99) + rows.append(dict(genotype=g, theta_low=low, + theta_high=rng.uniform(0.01, low), + log_hill_K=rng.normal(-4.1, 1.0), + hill_n=rng.uniform(1.0, 3.0))) + return pd.DataFrame(rows) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("base_config") + ap.add_argument("out_dir") + ap.add_argument("--arm", choices=sorted(ARMS), required=True) + ap.add_argument("--seed", type=int, default=1) + args = ap.parse_args() + + with open(args.base_config) as fh: + cf = yaml.safe_load(fh) + cf = absolute_paths(cf, os.path.dirname(os.path.abspath(args.base_config))) + cf["seed"] = args.seed + os.makedirs(args.out_dir, exist_ok=True) + + lib = library_composition_table(args.base_config) + spikes = sorted(set(lib.loc[lib["in_spiked_origin"], "genotype"].astype(str))) + hill_path = os.path.abspath(os.path.join(args.out_dir, "spike_hill.csv")) + spike_curves(spikes, args.seed).to_csv(hill_path, index=False) + + arm = ARMS[args.arm] + for cond, (a, x0) in arm["warp"].items(): + if cond not in cf["growth"]: + raise ValueError(f"{cond} is not a growth condition of the base config") + cf["growth"][cond]["warp_a"] = a + cf["growth"][cond]["warp_x0"] = x0 + if "transformation_poisson_lambda" in arm: + cf["transformation_poisson_lambda"] = arm["transformation_poisson_lambda"] + if arm["growth_titrant_scale"] is not None: + cf["growth_titrant_scale"] = arm["growth_titrant_scale"] + cf["binding_data"] = { + "titrant_name": "iptg", + "titrant_conc": BINDING_IPTG, + "noise": BINDING_NOISE, + "genotype_offset_sd": arm["genotype_offset_sd"], + "spiked_binding": {"choose_by": hill_path}, + } + + out = os.path.join(args.out_dir, "simulate_config.yaml") + with open(out, "w") as fh: + fh.write(f"# joint-fit simulation test, arm {args.arm}, seed {args.seed} " + f"(planning/studies/joint-fit-sim)\n") + yaml.safe_dump(cf, fh, sort_keys=False) + print(f"wrote {out} and {hill_path} ({len(spikes)} spikes: {', '.join(spikes)})") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/joint-fit-sim/out_null/params.csv b/planning/studies/joint-fit-sim/out_null/params.csv new file mode 100644 index 00000000..69ef6779 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_null/params.csv @@ -0,0 +1,28 @@ +run,parameter,key,truth,q0_5,lo,hi,covered +fit_null_s1_plain,growth_k,kanR+kan,0.015042,0.019639313,0.01776531,0.021214087,False +fit_null_s1_plain,growth_k,kanR-kan,0.023923,0.029058484,0.026518818,0.03145352,False +fit_null_s1_plain,growth_k,pheS+4CP,0.003107,0.0037996653,0.0013952442,0.0064904196,True +fit_null_s1_plain,growth_k,pheS-4CP,0.022042,0.02379319,0.020922774,0.027036328,True +fit_null_s1_plain,growth_m,kanR+kan,-0.014122,-0.017343588,-0.01765266,-0.016980322,False +fit_null_s1_plain,growth_m,kanR-kan,0.0,2.2137636e-05,-2.0148449e-05,6.657838e-05,True +fit_null_s1_plain,growth_m,pheS+4CP,0.014379,0.017263267,0.016902821,0.017571792,False +fit_null_s1_plain,growth_m,pheS-4CP,0.0,2.6166083e-05,-5.8999135e-06,5.760233e-05,True +fit_null_s1_full,growth_k,kanR+kan,0.015042,0.018222786,0.016564487,0.019878788,False +fit_null_s1_full,growth_k,kanR-kan,0.023923,0.02736361,0.024703149,0.0301056,False +fit_null_s1_full,growth_k,pheS+4CP,0.003107,0.003581851,0.0014208781,0.005962828,True +fit_null_s1_full,growth_k,pheS-4CP,0.022042,0.023629753,0.021096645,0.026247969,True +fit_null_s1_full,growth_m,kanR+kan,-0.014122,-0.016731359,-0.016920654,-0.016567638,False +fit_null_s1_full,growth_m,kanR-kan,0.0,1.9714233e-05,-8.185913e-05,0.000114059236,True +fit_null_s1_full,growth_m,pheS+4CP,0.014379,0.016879555,0.016714346,0.01707239,False +fit_null_s1_full,growth_m,pheS-4CP,0.0,0.00011692887,2.695735e-05,0.0002005057,False +fit_null_s1_full,log_s,global,0.0,-1.3931419,-1.7649902,-1.0404953,False +fit_null_s1_full,binding_offset,wt,0.0,0.024139058,-0.0062584644,0.053019974,True +fit_null_s1_full,binding_offset,M42I,0.0,-0.0752502,-0.12573017,-0.029268635,False +fit_null_s1_full,binding_offset,H74A,0.0,-0.060534805,-0.10915214,-0.016090112,False +fit_null_s1_full,binding_offset,K84L,0.0,-0.3232484,-0.37575576,-0.2427785,False +fit_null_s1_full,binding_offset,D88A,0.0,-0.1492683,-0.19367711,-0.09986605,False +fit_null_s1_full,binding_offset,M42I/H74A,0.0,-0.20263562,-0.25535667,-0.14406282,False +fit_null_s1_full,binding_offset,M42I/K84L,0.0,-0.38100737,-0.44027537,-0.303622,False +fit_null_s1_full,binding_offset,H74A/K84L,0.0,-0.049540114,-0.11304207,0.023334747,True +fit_null_s1_full,binding_offset,M42I/H74A/K84L,0.0,-0.05620514,-0.10274805,-0.011132386,False +fit_null_s1_full,binding_offset_scale,global,0.0,0.34289342,0.19601426,0.59310955,False diff --git a/planning/studies/joint-fit-sim/out_null/theta.csv b/planning/studies/joint-fit-sim/out_null/theta.csv new file mode 100644 index 00000000..09efc92a --- /dev/null +++ b/planning/studies/joint-fit-sim/out_null/theta.csv @@ -0,0 +1,7 @@ +run,group,n_genotypes,rmse,bias,coverage95 +fit_null_s1_plain,single,946,0.056310164351737704,-0.002109800618809592,0.102669133192389 +fit_null_s1_plain,spike,8,0.03564173580072257,0.005594124974061381,0.421875 +fit_null_s1_plain,wt,1,0.039885856632109154,-0.008674691594757433,0.0 +fit_null_s1_full,single,946,0.29316910442072874,0.09677065014736853,0.038319238900634246 +fit_null_s1_full,spike,8,0.225658662648604,0.15255901831150648,0.09375 +fit_null_s1_full,wt,1,0.15129934306821724,-0.03213419625712716,0.125 diff --git a/planning/studies/joint-fit-sim/out_run3/params.csv b/planning/studies/joint-fit-sim/out_run3/params.csv new file mode 100644 index 00000000..a25aafa1 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run3/params.csv @@ -0,0 +1,43 @@ +run,parameter,key,truth,q0_5,lo,hi,covered +fit_null_s1_full_from_plain,growth_k,kanR+kan,0.015042,0.019716062,0.017876662,0.021552911,False +fit_null_s1_full_from_plain,growth_k,kanR-kan,0.023923,0.028928714,0.026172755,0.031623535,False +fit_null_s1_full_from_plain,growth_k,pheS+4CP,0.003107,0.003445853,0.0020440621,0.0048898384,True +fit_null_s1_full_from_plain,growth_k,pheS-4CP,0.022042,0.02417104,0.021403147,0.026987113,True +fit_null_s1_full_from_plain,growth_m,kanR+kan,-0.014122,-0.01760393,-0.01799401,-0.017248493,False +fit_null_s1_full_from_plain,growth_m,kanR-kan,0.0,3.3065917e-05,-1.9288647e-05,7.71631e-05,True +fit_null_s1_full_from_plain,growth_m,pheS+4CP,0.014379,0.018049337,0.017655252,0.018451113,False +fit_null_s1_full_from_plain,growth_m,pheS-4CP,0.0,3.2067503e-05,-2.4998012e-06,6.6250024e-05,True +fit_null_s1_full_from_plain,log_s,global,0.0,-0.051260404,-0.21699598,0.109757304,True +fit_null_s1_full_from_plain,binding_offset,wt,0.0,-0.00874895,-0.049160227,0.030390788,True +fit_null_s1_full_from_plain,binding_offset,M42I,0.0,0.0037606945,-0.08931366,0.07740348,True +fit_null_s1_full_from_plain,binding_offset,H74A,0.0,0.0050335433,-0.07902607,0.061427925,True +fit_null_s1_full_from_plain,binding_offset,K84L,0.0,-0.07332194,-0.13243912,0.053684596,True +fit_null_s1_full_from_plain,binding_offset,D88A,0.0,-0.01866046,-0.107220404,0.06887053,True +fit_null_s1_full_from_plain,binding_offset,M42I/H74A,0.0,-0.027172983,-0.115051314,0.05755074,True +fit_null_s1_full_from_plain,binding_offset,M42I/K84L,0.0,-0.08818379,-0.13815251,0.029590687,True +fit_null_s1_full_from_plain,binding_offset,H74A/K84L,0.0,-0.0089068655,-0.07606615,0.060055368,True +fit_null_s1_full_from_plain,binding_offset,M42I/H74A/K84L,0.0,0.0125195645,-0.061470274,0.059541415,True +fit_null_s1_full_from_plain,binding_offset_scale,global,0.0,0.16763236,0.052113824,0.5166569,False +fit_joint_s1_full_from_plain,growth_k,kanR+kan,0.015042,0.017957725,0.016482297,0.019431103,False +fit_joint_s1_full_from_plain,growth_k,kanR-kan,0.023923,0.02928865,0.027100608,0.031431463,False +fit_joint_s1_full_from_plain,growth_k,pheS+4CP,0.003107,0.0051531717,0.0032590851,0.0072666015,False +fit_joint_s1_full_from_plain,growth_k,pheS-4CP,0.022042,0.027519124,0.025027083,0.03014029,False +fit_joint_s1_full_from_plain,growth_m,kanR+kan,-0.014122,-0.014448609,-0.014514818,-0.01439054,False +fit_joint_s1_full_from_plain,growth_m,kanR-kan,0.0,-1.0012214e-05,-5.552533e-05,2.6604968e-05,True +fit_joint_s1_full_from_plain,growth_m,pheS+4CP,0.014379,0.01656733,0.016284943,0.016876321,False +fit_joint_s1_full_from_plain,growth_m,pheS-4CP,0.0,1.2326492e-05,-1.8744171e-05,4.392883e-05,True +fit_joint_s1_full_from_plain,warp_log_a,kanR+kan,2.0794415416798357,2.2021394,2.1576385,2.248794,False +fit_joint_s1_full_from_plain,warp_log_a,pheS+4CP,0.4054651081081644,-0.08407947,-1.5147213,1.4601364,True +fit_joint_s1_full_from_plain,warp_x0,kanR+kan,0.5,0.53383434,0.49808156,0.5636751,True +fit_joint_s1_full_from_plain,warp_x0,pheS+4CP,0.4,0.58642846,-0.23549522,1.322586,True +fit_joint_s1_full_from_plain,log_s,global,-0.6931471805599453,-0.79566884,-0.9376141,-0.6642408,True +fit_joint_s1_full_from_plain,binding_offset,wt,-0.1056078950314186,-0.10702185,-0.1374449,-0.042688668,True +fit_joint_s1_full_from_plain,binding_offset,M42I,0.0282415842244401,0.009926669,-0.043777168,0.05783087,True +fit_joint_s1_full_from_plain,binding_offset,H74A,-0.0420857002548871,-0.043082345,-0.081066534,0.011327274,True +fit_joint_s1_full_from_plain,binding_offset,K84L,0.0425148735331908,-0.017117418,-0.0507347,0.02963588,False +fit_joint_s1_full_from_plain,binding_offset,D88A,0.0511536419917619,0.037221782,-0.022255406,0.081511445,True +fit_joint_s1_full_from_plain,binding_offset,M42I/H74A,-0.1159296726932429,-0.13883291,-0.18324412,-0.043825623,True +fit_joint_s1_full_from_plain,binding_offset,M42I/K84L,-0.0590434943289043,-0.11167629,-0.14532866,-0.03547845,True +fit_joint_s1_full_from_plain,binding_offset,H74A/K84L,-0.0228535367478158,-0.023133399,-0.06173983,0.037326477,True +fit_joint_s1_full_from_plain,binding_offset,M42I/H74A/K84L,0.0833342596669661,0.07477276,0.014866954,0.10796515,True +fit_joint_s1_full_from_plain,binding_offset_scale,global,0.1,0.2238241,0.0949375,0.49544805,True diff --git a/planning/studies/joint-fit-sim/out_run3/theta.csv b/planning/studies/joint-fit-sim/out_run3/theta.csv new file mode 100644 index 00000000..cc6bd50c --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run3/theta.csv @@ -0,0 +1,7 @@ +run,group,n_genotypes,rmse,bias,coverage95 +fit_null_s1_full_from_plain,single,946,0.07655127606143827,0.007179007491373085,0.13860993657505286 +fit_null_s1_full_from_plain,spike,8,0.05580013936709474,0.02425916185497325,0.640625 +fit_null_s1_full_from_plain,wt,1,0.05699302949382643,-0.0024569782542474654,0.125 +fit_joint_s1_full_from_plain,single,946,0.07012804425293707,0.004091526192144923,0.16596194503171247 +fit_joint_s1_full_from_plain,spike,8,0.038620122460039444,0.020425483904325994,0.484375 +fit_joint_s1_full_from_plain,wt,1,0.05676126023676298,-0.0053830364664834325,0.25 diff --git a/planning/studies/joint-fit-sim/out_run4/params.csv b/planning/studies/joint-fit-sim/out_run4/params.csv new file mode 100644 index 00000000..03318771 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run4/params.csv @@ -0,0 +1,17 @@ +run,parameter,key,truth,q0_5,lo,hi,covered +fit_null_s1_plain_from_plain_pinkm,growth_k,kanR+kan,0.015042,0.015042,0.015042,0.015042,True +fit_null_s1_plain_from_plain_pinkm,growth_k,kanR-kan,0.023923,0.023923,0.023923,0.023923,True +fit_null_s1_plain_from_plain_pinkm,growth_k,pheS+4CP,0.003107,0.003107,0.003107,0.003107,True +fit_null_s1_plain_from_plain_pinkm,growth_k,pheS-4CP,0.022042,0.022042,0.022042,0.022042,True +fit_null_s1_plain_from_plain_pinkm,growth_m,kanR+kan,-0.014122,-0.014122,-0.014122,-0.014122,True +fit_null_s1_plain_from_plain_pinkm,growth_m,kanR-kan,0.0,0.0,0.0,0.0,True +fit_null_s1_plain_from_plain_pinkm,growth_m,pheS+4CP,0.014379,0.014379,0.014379,0.014379,True +fit_null_s1_plain_from_plain_pinkm,growth_m,pheS-4CP,0.0,0.0,0.0,0.0,True +fit_null_s1_plain,growth_k,kanR+kan,0.015042,0.019639313,0.01776531,0.021214087,False +fit_null_s1_plain,growth_k,kanR-kan,0.023923,0.029058484,0.026518818,0.03145352,False +fit_null_s1_plain,growth_k,pheS+4CP,0.003107,0.0037996653,0.0013952442,0.0064904196,True +fit_null_s1_plain,growth_k,pheS-4CP,0.022042,0.02379319,0.020922774,0.027036328,True +fit_null_s1_plain,growth_m,kanR+kan,-0.014122,-0.017343588,-0.01765266,-0.016980322,False +fit_null_s1_plain,growth_m,kanR-kan,0.0,2.2137636e-05,-2.0148449e-05,6.657838e-05,True +fit_null_s1_plain,growth_m,pheS+4CP,0.014379,0.017263267,0.016902821,0.017571792,False +fit_null_s1_plain,growth_m,pheS-4CP,0.0,2.6166083e-05,-5.8999135e-06,5.760233e-05,True diff --git a/planning/studies/joint-fit-sim/out_run4/theta.csv b/planning/studies/joint-fit-sim/out_run4/theta.csv new file mode 100644 index 00000000..14c0ebc9 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run4/theta.csv @@ -0,0 +1,7 @@ +run,group,n_genotypes,rmse,bias,coverage95 +fit_null_s1_plain_from_plain_pinkm,single,946,0.02023168631524801,0.006409369313332721,0.6993921775898521 +fit_null_s1_plain_from_plain_pinkm,spike,8,0.012441048865219476,0.004809608444669076,0.78125 +fit_null_s1_plain_from_plain_pinkm,wt,1,0.011436729538816675,0.0083255390645486,0.625 +fit_null_s1_plain,single,946,0.056310164351737704,-0.002109800618809592,0.102669133192389 +fit_null_s1_plain,spike,8,0.03564173580072257,0.005594124974061381,0.421875 +fit_null_s1_plain,wt,1,0.039885856632109154,-0.008674691594757433,0.0 diff --git a/planning/studies/joint-fit-sim/out_run5/params.csv b/planning/studies/joint-fit-sim/out_run5/params.csv new file mode 100644 index 00000000..103e8a36 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run5/params.csv @@ -0,0 +1,17 @@ +run,parameter,key,truth,q0_5,lo,hi,covered +fit_nocong_s1_plain_single,growth_k,kanR+kan,0.015042,0.013801275,0.012038811,0.015553157,True +fit_nocong_s1_plain_single,growth_k,kanR-kan,0.023923,0.01879035,0.016206073,0.021419218,False +fit_nocong_s1_plain_single,growth_k,pheS+4CP,0.003107,0.0015561959,-0.0005445291,0.0034262936,True +fit_nocong_s1_plain_single,growth_k,pheS-4CP,0.022042,0.020876065,0.017976936,0.023448763,True +fit_nocong_s1_plain_single,growth_m,kanR+kan,-0.014122,-0.016061418,-0.016527396,-0.015459808,False +fit_nocong_s1_plain_single,growth_m,kanR-kan,0.0,-4.444495e-07,-3.4166256e-05,3.535745e-05,True +fit_nocong_s1_plain_single,growth_m,pheS+4CP,0.014379,0.016373929,0.01574738,0.016852725,False +fit_nocong_s1_plain_single,growth_m,pheS-4CP,0.0,-1.2868813e-06,-2.9207054e-05,2.2709995e-05,True +fit_nocong_s1_plain_single_pinkm,growth_k,kanR+kan,0.015042,0.015042,0.015042,0.015042,True +fit_nocong_s1_plain_single_pinkm,growth_k,kanR-kan,0.023923,0.023923,0.023923,0.023923,True +fit_nocong_s1_plain_single_pinkm,growth_k,pheS+4CP,0.003107,0.003107,0.003107,0.003107,True +fit_nocong_s1_plain_single_pinkm,growth_k,pheS-4CP,0.022042,0.022042,0.022042,0.022042,True +fit_nocong_s1_plain_single_pinkm,growth_m,kanR+kan,-0.014122,-0.014122,-0.014122,-0.014122,True +fit_nocong_s1_plain_single_pinkm,growth_m,kanR-kan,0.0,0.0,0.0,0.0,True +fit_nocong_s1_plain_single_pinkm,growth_m,pheS+4CP,0.014379,0.014379,0.014379,0.014379,True +fit_nocong_s1_plain_single_pinkm,growth_m,pheS-4CP,0.0,0.0,0.0,0.0,True diff --git a/planning/studies/joint-fit-sim/out_run5/theta.csv b/planning/studies/joint-fit-sim/out_run5/theta.csv new file mode 100644 index 00000000..4c2c847c --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run5/theta.csv @@ -0,0 +1,7 @@ +run,group,n_genotypes,rmse,bias,coverage95 +fit_nocong_s1_plain_single,single,946,0.04600651406708678,-0.010776528232766088,0.10782241014799154 +fit_nocong_s1_plain_single,spike,8,0.023051857667078703,-0.003161633681085658,0.46875 +fit_nocong_s1_plain_single,wt,1,0.04132453085322089,-0.013980366024244472,0.0 +fit_nocong_s1_plain_single_pinkm,single,946,0.010026984980318813,-0.0017261095541820563,0.9100158562367865 +fit_nocong_s1_plain_single_pinkm,spike,8,0.006414977708462745,-0.002008960226691008,0.984375 +fit_nocong_s1_plain_single_pinkm,wt,1,0.0019766868091285602,-0.0019484390082925964,1.0 diff --git a/planning/studies/joint-fit-sim/out_run6/params.csv b/planning/studies/joint-fit-sim/out_run6/params.csv new file mode 100644 index 00000000..a0786011 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run6/params.csv @@ -0,0 +1,33 @@ +run,parameter,key,truth,q0_5,lo,hi,covered +fit_nocong_s1_plain_single,growth_k,kanR+kan,0.015042,0.013801275,0.012038811,0.015553157,True +fit_nocong_s1_plain_single,growth_k,kanR-kan,0.023923,0.01879035,0.016206073,0.021419218,False +fit_nocong_s1_plain_single,growth_k,pheS+4CP,0.003107,0.0015561959,-0.0005445291,0.0034262936,True +fit_nocong_s1_plain_single,growth_k,pheS-4CP,0.022042,0.020876065,0.017976936,0.023448763,True +fit_nocong_s1_plain_single,growth_m,kanR+kan,-0.014122,-0.016061418,-0.016527396,-0.015459808,False +fit_nocong_s1_plain_single,growth_m,kanR-kan,0.0,-4.444495e-07,-3.4166256e-05,3.535745e-05,True +fit_nocong_s1_plain_single,growth_m,pheS+4CP,0.014379,0.016373929,0.01574738,0.016852725,False +fit_nocong_s1_plain_single,growth_m,pheS-4CP,0.0,-1.2868813e-06,-2.9207054e-05,2.2709995e-05,True +fit_nocong_s1_plain_single_heldsd,growth_k,kanR+kan,0.015042,0.015109595,0.013546642,0.016911933,True +fit_nocong_s1_plain_single_heldsd,growth_k,kanR-kan,0.023923,0.01908277,0.016755098,0.021597177,False +fit_nocong_s1_plain_single_heldsd,growth_k,pheS+4CP,0.003107,0.00043212692,-0.0009298731,0.0016626724,False +fit_nocong_s1_plain_single_heldsd,growth_k,pheS-4CP,0.022042,0.02102251,0.018773993,0.02310895,True +fit_nocong_s1_plain_single_heldsd,growth_m,kanR+kan,-0.014122,-0.018351812,-0.018829105,-0.017876515,False +fit_nocong_s1_plain_single_heldsd,growth_m,kanR-kan,0.0,-2.695143e-06,-4.473586e-05,4.2335116e-05,True +fit_nocong_s1_plain_single_heldsd,growth_m,pheS+4CP,0.014379,0.018708836,0.018215489,0.01919493,False +fit_nocong_s1_plain_single_heldsd,growth_m,pheS-4CP,0.0,-1.6029747e-06,-3.200255e-05,2.505449e-05,True +fit_nocong_s1_plain_single_loc,growth_k,kanR+kan,0.015042,0.01380008,0.012327066,0.01526425,True +fit_nocong_s1_plain_single_loc,growth_k,kanR-kan,0.023923,0.018815354,0.016599998,0.021009775,False +fit_nocong_s1_plain_single_loc,growth_k,pheS+4CP,0.003107,0.001565194,-6.083904e-05,0.0028944348,False +fit_nocong_s1_plain_single_loc,growth_k,pheS-4CP,0.022042,0.020891104,0.01818344,0.023047762,True +fit_nocong_s1_plain_single_loc,growth_m,kanR+kan,-0.014122,-0.01604757,-0.016499398,-0.015521235,False +fit_nocong_s1_plain_single_loc,growth_m,kanR-kan,0.0,1.8007813e-07,-3.5932873e-05,3.7369908e-05,True +fit_nocong_s1_plain_single_loc,growth_m,pheS+4CP,0.014379,0.016371025,0.015813284,0.016805565,False +fit_nocong_s1_plain_single_loc,growth_m,pheS-4CP,0.0,-3.2650047e-07,-3.1479118e-05,2.145867e-05,True +fit_nocong_s1_plain_single_both,growth_k,kanR+kan,0.015042,0.015178808,0.013595535,0.017004577,True +fit_nocong_s1_plain_single_both,growth_k,kanR-kan,0.023923,0.019120358,0.016526548,0.021932742,False +fit_nocong_s1_plain_single_both,growth_k,pheS+4CP,0.003107,0.00036602712,-0.0018512164,0.0024347708,False +fit_nocong_s1_plain_single_both,growth_k,pheS-4CP,0.022042,0.021021845,0.018108612,0.023728538,True +fit_nocong_s1_plain_single_both,growth_m,kanR+kan,-0.014122,-0.018444527,-0.01890321,-0.01800809,False +fit_nocong_s1_plain_single_both,growth_m,kanR-kan,0.0,-1.8536489e-06,-4.3988308e-05,4.3155676e-05,True +fit_nocong_s1_plain_single_both,growth_m,pheS+4CP,0.014379,0.01880873,0.01834993,0.019277444,False +fit_nocong_s1_plain_single_both,growth_m,pheS-4CP,0.0,-2.406667e-06,-3.215514e-05,2.4122559e-05,True diff --git a/planning/studies/joint-fit-sim/out_run6/theta.csv b/planning/studies/joint-fit-sim/out_run6/theta.csv new file mode 100644 index 00000000..03a80a48 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run6/theta.csv @@ -0,0 +1,13 @@ +run,group,n_genotypes,rmse,bias,coverage95 +fit_nocong_s1_plain_single,single,946,0.04600651406708678,-0.010776528232766088,0.10782241014799154 +fit_nocong_s1_plain_single,spike,8,0.023051857667078703,-0.003161633681085658,0.46875 +fit_nocong_s1_plain_single,wt,1,0.04132453085322089,-0.013980366024244472,0.0 +fit_nocong_s1_plain_single_heldsd,single,946,0.08540881392358195,-0.018294370036810996,0.04373678646934461 +fit_nocong_s1_plain_single_heldsd,spike,8,0.04221698927449781,-0.004739749984244781,0.171875 +fit_nocong_s1_plain_single_heldsd,wt,1,0.07880600956917144,-0.024576634203527545,0.0 +fit_nocong_s1_plain_single_loc,single,946,0.04563101607446293,-0.010643469689389149,0.10689746300211417 +fit_nocong_s1_plain_single_loc,spike,8,0.022944694706117076,-0.0029301735532101456,0.484375 +fit_nocong_s1_plain_single_loc,wt,1,0.04091447279596791,-0.014013389128223145,0.0 +fit_nocong_s1_plain_single_both,single,946,0.08761831526429646,-0.01872845471482825,0.04254756871035941 +fit_nocong_s1_plain_single_both,spike,8,0.04332452652241193,-0.004835552177489268,0.140625 +fit_nocong_s1_plain_single_both,wt,1,0.08108988569030989,-0.025144357819514025,0.0 diff --git a/planning/studies/joint-fit-sim/out_run7/params.csv b/planning/studies/joint-fit-sim/out_run7/params.csv new file mode 100644 index 00000000..2e34169c --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run7/params.csv @@ -0,0 +1,41 @@ +run,parameter,key,truth,q0_5,lo,hi,covered +fit_nocong_s1_plain_single,growth_k,kanR+kan,0.015042,0.013801275,0.012038811,0.015553157,True +fit_nocong_s1_plain_single,growth_k,kanR-kan,0.023923,0.01879035,0.016206073,0.021419218,False +fit_nocong_s1_plain_single,growth_k,pheS+4CP,0.003107,0.0015561959,-0.0005445291,0.0034262936,True +fit_nocong_s1_plain_single,growth_k,pheS-4CP,0.022042,0.020876065,0.017976936,0.023448763,True +fit_nocong_s1_plain_single,growth_m,kanR+kan,-0.014122,-0.016061418,-0.016527396,-0.015459808,False +fit_nocong_s1_plain_single,growth_m,kanR-kan,0.0,-4.444495e-07,-3.4166256e-05,3.535745e-05,True +fit_nocong_s1_plain_single,growth_m,pheS+4CP,0.014379,0.016373929,0.01574738,0.016852725,False +fit_nocong_s1_plain_single,growth_m,pheS-4CP,0.0,-1.2868813e-06,-2.9207054e-05,2.2709995e-05,True +fit_nocong_s1_plain_single_weakpool,growth_k,kanR+kan,0.015042,0.013628845,0.011500317,0.016083382,True +fit_nocong_s1_plain_single_weakpool,growth_k,kanR-kan,0.023923,0.01892056,0.01560307,0.022467064,False +fit_nocong_s1_plain_single_weakpool,growth_k,pheS+4CP,0.003107,0.0017348563,-0.0013139428,0.004589546,True +fit_nocong_s1_plain_single_weakpool,growth_k,pheS-4CP,0.022042,0.020952094,0.016725298,0.02493233,True +fit_nocong_s1_plain_single_weakpool,growth_m,kanR+kan,-0.014122,-0.015718333,-0.016057761,-0.01538869,False +fit_nocong_s1_plain_single_weakpool,growth_m,kanR-kan,0.0,-1.20813e-06,-3.727754e-05,3.714259e-05,True +fit_nocong_s1_plain_single_weakpool,growth_m,pheS+4CP,0.014379,0.016023008,0.015685845,0.016374357,False +fit_nocong_s1_plain_single_weakpool,growth_m,pheS-4CP,0.0,-2.3394914e-06,-2.7781969e-05,1.9868905e-05,True +fit_nocong_s1_plain_single_lib2,growth_k,kanR+kan,0.015042,0.013561364,0.011840986,0.015271412,True +fit_nocong_s1_plain_single_lib2,growth_k,kanR-kan,0.023923,0.0187679,0.016024807,0.021588586,False +fit_nocong_s1_plain_single_lib2,growth_k,pheS+4CP,0.003107,0.0018079232,-0.00046474964,0.0036126454,True +fit_nocong_s1_plain_single_lib2,growth_k,pheS-4CP,0.022042,0.02095231,0.017813662,0.02331125,True +fit_nocong_s1_plain_single_lib2,growth_m,kanR+kan,-0.014122,-0.0156019665,-0.015870579,-0.015286126,False +fit_nocong_s1_plain_single_lib2,growth_m,kanR-kan,0.0,-1.2407652e-06,-3.2953893e-05,3.317217e-05,True +fit_nocong_s1_plain_single_lib2,growth_m,pheS+4CP,0.014379,0.015908673,0.015576741,0.016182851,False +fit_nocong_s1_plain_single_lib2,growth_m,pheS-4CP,0.0,-4.3367936e-07,-2.7493896e-05,2.2053584e-05,True +fit_nocong_s1_plain_single_lib20,growth_k,kanR+kan,0.015042,0.013215949,0.012228213,0.014197754,False +fit_nocong_s1_plain_single_lib20,growth_k,kanR-kan,0.023923,0.018749062,0.016980382,0.020398566,False +fit_nocong_s1_plain_single_lib20,growth_k,pheS+4CP,0.003107,0.0020643813,0.0009044189,0.0030294056,False +fit_nocong_s1_plain_single_lib20,growth_k,pheS-4CP,0.022042,0.0209004,0.018916024,0.02242581,True +fit_nocong_s1_plain_single_lib20,growth_m,kanR+kan,-0.014122,-0.014929175,-0.015063497,-0.01479034,False +fit_nocong_s1_plain_single_lib20,growth_m,kanR-kan,0.0,-4.4176616e-07,-3.27292e-05,3.3555803e-05,True +fit_nocong_s1_plain_single_lib20,growth_m,pheS+4CP,0.014379,0.015220229,0.0150832655,0.015350807,False +fit_nocong_s1_plain_single_lib20,growth_m,pheS-4CP,0.0,-1.3181095e-06,-2.6742477e-05,2.0230633e-05,True +fit_nocong_s1_plain_single_weakpool_lib20,growth_k,kanR+kan,0.015042,0.0131746335,0.01162946,0.014956468,False +fit_nocong_s1_plain_single_weakpool_lib20,growth_k,kanR-kan,0.023923,0.018806567,0.016182827,0.021627234,False +fit_nocong_s1_plain_single_weakpool_lib20,growth_k,pheS+4CP,0.003107,0.0020984784,0.00034642438,0.0036215098,True +fit_nocong_s1_plain_single_weakpool_lib20,growth_k,pheS-4CP,0.022042,0.02089941,0.018101713,0.023452591,True +fit_nocong_s1_plain_single_weakpool_lib20,growth_m,kanR+kan,-0.014122,-0.0148265865,-0.014932872,-0.014723662,False +fit_nocong_s1_plain_single_weakpool_lib20,growth_m,kanR-kan,0.0,-2.0679395e-06,-3.5970006e-05,3.4226574e-05,True +fit_nocong_s1_plain_single_weakpool_lib20,growth_m,pheS+4CP,0.014379,0.015116502,0.015009742,0.015218562,False +fit_nocong_s1_plain_single_weakpool_lib20,growth_m,pheS-4CP,0.0,-1.5685268e-06,-2.5498137e-05,1.9750209e-05,True diff --git a/planning/studies/joint-fit-sim/out_run7/theta.csv b/planning/studies/joint-fit-sim/out_run7/theta.csv new file mode 100644 index 00000000..af22ac2f --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run7/theta.csv @@ -0,0 +1,19 @@ +run,group,n_genotypes,rmse,bias,coverage95 +fit_nocong_s1_plain_single,single,946,0.04600651406708678,-0.010776528232766088,0.10782241014799154 +fit_nocong_s1_plain_single,spike,8,0.023051857667078703,-0.003161633681085658,0.46875 +fit_nocong_s1_plain_single,wt,1,0.04132453085322089,-0.013980366024244472,0.0 +fit_nocong_s1_plain_single_weakpool,single,946,0.0394620077243776,-0.008725156518903036,0.1261892177589852 +fit_nocong_s1_plain_single_weakpool,spike,8,0.01980491592948896,-0.002717789717918243,0.46875 +fit_nocong_s1_plain_single_weakpool,wt,1,0.0350033887773782,-0.01130932117445482,0.0 +fit_nocong_s1_plain_single_lib2,single,944,0.03573471290021584,-0.0080530998358043,0.1481726694915254 +fit_nocong_s1_plain_single_lib2,"single, binding",2,0.022768642812501568,4.6763618178457655e-05,0.25 +fit_nocong_s1_plain_single_lib2,spike,8,0.018451340792210596,-0.0018307648317920575,0.515625 +fit_nocong_s1_plain_single_lib2,wt,1,0.03056125393936438,-0.010622998827147592,0.0 +fit_nocong_s1_plain_single_lib20,single,926,0.020627103634744764,-0.002397537332113211,0.35461663066954646 +fit_nocong_s1_plain_single_lib20,"single, binding",20,0.018520133288800207,8.424415839208539e-05,0.4 +fit_nocong_s1_plain_single_lib20,spike,8,0.012215949671842655,0.0008572381069769292,0.703125 +fit_nocong_s1_plain_single_lib20,wt,1,0.014084600170187629,-0.003690915351246332,0.0 +fit_nocong_s1_plain_single_weakpool_lib20,single,926,0.0193553533381434,-0.0022052652264256027,0.3752699784017279 +fit_nocong_s1_plain_single_weakpool_lib20,"single, binding",20,0.01712178379393477,0.00047100702239295756,0.425 +fit_nocong_s1_plain_single_weakpool_lib20,spike,8,0.010915790643631365,4.878429160192757e-05,0.625 +fit_nocong_s1_plain_single_weakpool_lib20,wt,1,0.013225308865114584,-0.0032218532631413106,0.0 diff --git a/planning/studies/joint-fit-sim/out_run8/params.csv b/planning/studies/joint-fit-sim/out_run8/params.csv new file mode 100644 index 00000000..8310b626 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run8/params.csv @@ -0,0 +1,115 @@ +run,parameter,key,truth,q0_5,lo,hi,covered +fit_null_s1_plain,growth_k,kanR+kan,0.015042,0.019639313,0.01776531,0.021214087,False +fit_null_s1_plain,growth_k,kanR-kan,0.023923,0.029058484,0.026518818,0.03145352,False +fit_null_s1_plain,growth_k,pheS+4CP,0.003107,0.0037996653,0.0013952442,0.0064904196,True +fit_null_s1_plain,growth_k,pheS-4CP,0.022042,0.02379319,0.020922774,0.027036328,True +fit_null_s1_plain,growth_m,kanR+kan,-0.014122,-0.017343588,-0.01765266,-0.016980322,False +fit_null_s1_plain,growth_m,kanR-kan,0.0,2.2137636e-05,-2.0148449e-05,6.657838e-05,True +fit_null_s1_plain,growth_m,pheS+4CP,0.014379,0.017263267,0.016902821,0.017571792,False +fit_null_s1_plain,growth_m,pheS-4CP,0.0,2.6166083e-05,-5.8999135e-06,5.760233e-05,True +fit_null_s1_plain_lib20,growth_k,kanR+kan,0.015042,0.019513223,0.018394718,0.020453129,False +fit_null_s1_plain_lib20,growth_k,kanR-kan,0.023923,0.029919986,0.028066596,0.031689644,False +fit_null_s1_plain_lib20,growth_k,pheS+4CP,0.003107,0.004991972,0.0034726348,0.0065823984,False +fit_null_s1_plain_lib20,growth_k,pheS-4CP,0.022042,0.024593122,0.022723252,0.026306747,False +fit_null_s1_plain_lib20,growth_m,kanR+kan,-0.014122,-0.016021106,-0.016161803,-0.015866857,False +fit_null_s1_plain_lib20,growth_m,kanR-kan,0.0,2.108172e-05,-1.7717397e-05,6.212296e-05,True +fit_null_s1_plain_lib20,growth_m,pheS+4CP,0.014379,0.015963431,0.015811132,0.01611957,False +fit_null_s1_plain_lib20,growth_m,pheS-4CP,0.0,2.3652243e-05,-5.4952243e-06,5.275869e-05,True +fit_null_s1_plain_weakpool_lib20,growth_k,kanR+kan,0.015042,0.019537132,0.017989652,0.021163287,False +fit_null_s1_plain_weakpool_lib20,growth_k,kanR-kan,0.023923,0.029986436,0.027413847,0.033001345,False +fit_null_s1_plain_weakpool_lib20,growth_k,pheS+4CP,0.003107,0.005170502,0.003279138,0.0071089477,False +fit_null_s1_plain_weakpool_lib20,growth_k,pheS-4CP,0.022042,0.024631048,0.022674588,0.026770502,False +fit_null_s1_plain_weakpool_lib20,growth_m,kanR+kan,-0.014122,-0.015952587,-0.016091129,-0.01580104,False +fit_null_s1_plain_weakpool_lib20,growth_m,kanR-kan,0.0,2.0243358e-05,-2.3737131e-05,6.384833e-05,True +fit_null_s1_plain_weakpool_lib20,growth_m,pheS+4CP,0.014379,0.015898272,0.015745746,0.01606892,False +fit_null_s1_plain_weakpool_lib20,growth_m,pheS-4CP,0.0,2.5069487e-05,-7.2455196e-06,5.3015283e-05,True +fit_nocong_s1_plain_single_lib20,growth_k,kanR+kan,0.015042,0.013215949,0.012228213,0.014197754,False +fit_nocong_s1_plain_single_lib20,growth_k,kanR-kan,0.023923,0.018749062,0.016980382,0.020398566,False +fit_nocong_s1_plain_single_lib20,growth_k,pheS+4CP,0.003107,0.0020643813,0.0009044189,0.0030294056,False +fit_nocong_s1_plain_single_lib20,growth_k,pheS-4CP,0.022042,0.0209004,0.018916024,0.02242581,True +fit_nocong_s1_plain_single_lib20,growth_m,kanR+kan,-0.014122,-0.014929175,-0.015063497,-0.01479034,False +fit_nocong_s1_plain_single_lib20,growth_m,kanR-kan,0.0,-4.4176616e-07,-3.27292e-05,3.3555803e-05,True +fit_nocong_s1_plain_single_lib20,growth_m,pheS+4CP,0.014379,0.015220229,0.0150832655,0.015350807,False +fit_nocong_s1_plain_single_lib20,growth_m,pheS-4CP,0.0,-1.3181095e-06,-2.6742477e-05,2.0230633e-05,True +fit_nocong_s1_plain_single_lib50,growth_k,kanR+kan,0.015042,0.013170511,0.011740442,0.014591995,False +fit_nocong_s1_plain_single_lib50,growth_k,kanR-kan,0.023923,0.018710695,0.016430056,0.021022294,False +fit_nocong_s1_plain_single_lib50,growth_k,pheS+4CP,0.003107,0.002142149,-0.00020314584,0.004049731,True +fit_nocong_s1_plain_single_lib50,growth_k,pheS-4CP,0.022042,0.020901732,0.018200245,0.023223246,True +fit_nocong_s1_plain_single_lib50,growth_m,kanR+kan,-0.014122,-0.014778586,-0.014881822,-0.014663564,False +fit_nocong_s1_plain_single_lib50,growth_m,kanR-kan,0.0,-2.481416e-06,-3.1933774e-05,3.0242873e-05,True +fit_nocong_s1_plain_single_lib50,growth_m,pheS+4CP,0.014379,0.0150651205,0.014947455,0.015167577,False +fit_nocong_s1_plain_single_lib50,growth_m,pheS-4CP,0.0,8.895906e-07,-2.4735891e-05,2.2106657e-05,True +fit_joint_s1_full_from_plain,growth_k,kanR+kan,0.015042,0.017957725,0.016482297,0.019431103,False +fit_joint_s1_full_from_plain,growth_k,kanR-kan,0.023923,0.02928865,0.027100608,0.031431463,False +fit_joint_s1_full_from_plain,growth_k,pheS+4CP,0.003107,0.0051531717,0.0032590851,0.0072666015,False +fit_joint_s1_full_from_plain,growth_k,pheS-4CP,0.022042,0.027519124,0.025027083,0.03014029,False +fit_joint_s1_full_from_plain,growth_m,kanR+kan,-0.014122,-0.014448609,-0.014514818,-0.01439054,False +fit_joint_s1_full_from_plain,growth_m,kanR-kan,0.0,-1.0012214e-05,-5.552533e-05,2.6604968e-05,True +fit_joint_s1_full_from_plain,growth_m,pheS+4CP,0.014379,0.01656733,0.016284943,0.016876321,False +fit_joint_s1_full_from_plain,growth_m,pheS-4CP,0.0,1.2326492e-05,-1.8744171e-05,4.392883e-05,True +fit_joint_s1_full_from_plain,warp_log_a,kanR+kan,2.0794415416798357,2.2021394,2.1576385,2.248794,False +fit_joint_s1_full_from_plain,warp_log_a,pheS+4CP,0.4054651081081644,-0.08407947,-1.5147213,1.4601364,True +fit_joint_s1_full_from_plain,warp_x0,kanR+kan,0.5,0.53383434,0.49808156,0.5636751,True +fit_joint_s1_full_from_plain,warp_x0,pheS+4CP,0.4,0.58642846,-0.23549522,1.322586,True +fit_joint_s1_full_from_plain,log_s,global,-0.6931471805599453,-0.79566884,-0.9376141,-0.6642408,True +fit_joint_s1_full_from_plain,binding_offset,wt,-0.1056078950314186,-0.10702185,-0.1374449,-0.042688668,True +fit_joint_s1_full_from_plain,binding_offset,M42I,0.0282415842244401,0.009926669,-0.043777168,0.05783087,True +fit_joint_s1_full_from_plain,binding_offset,H74A,-0.0420857002548871,-0.043082345,-0.081066534,0.011327274,True +fit_joint_s1_full_from_plain,binding_offset,K84L,0.0425148735331908,-0.017117418,-0.0507347,0.02963588,False +fit_joint_s1_full_from_plain,binding_offset,D88A,0.0511536419917619,0.037221782,-0.022255406,0.081511445,True +fit_joint_s1_full_from_plain,binding_offset,M42I/H74A,-0.1159296726932429,-0.13883291,-0.18324412,-0.043825623,True +fit_joint_s1_full_from_plain,binding_offset,M42I/K84L,-0.0590434943289043,-0.11167629,-0.14532866,-0.03547845,True +fit_joint_s1_full_from_plain,binding_offset,H74A/K84L,-0.0228535367478158,-0.023133399,-0.06173983,0.037326477,True +fit_joint_s1_full_from_plain,binding_offset,M42I/H74A/K84L,0.0833342596669661,0.07477276,0.014866954,0.10796515,True +fit_joint_s1_full_from_plain,binding_offset_scale,global,0.1,0.2238241,0.0949375,0.49544805,True +fit_joint_s1_plain_lib20,growth_k,kanR+kan,0.015042,0.018819705,0.016253246,0.020976365,False +fit_joint_s1_plain_lib20,growth_k,kanR-kan,0.023923,0.029214837,0.025166273,0.032829322,False +fit_joint_s1_plain_lib20,growth_k,pheS+4CP,0.003107,0.006786953,0.003960558,0.009375356,False +fit_joint_s1_plain_lib20,growth_k,pheS-4CP,0.022042,0.02691646,0.023299571,0.030369096,False +fit_joint_s1_plain_lib20,growth_m,kanR+kan,-0.014122,-0.016758459,-0.016872358,-0.016635366,False +fit_joint_s1_plain_lib20,growth_m,kanR-kan,0.0,-0.00010611255,-0.00015000328,-5.9442915e-05,False +fit_joint_s1_plain_lib20,growth_m,pheS+4CP,0.014379,0.013973749,0.013877165,0.014074243,False +fit_joint_s1_plain_lib20,growth_m,pheS-4CP,0.0,-0.0001146983,-0.00015081318,-7.984436e-05,False +fit_joint_s1_full_from_plain_lib20,growth_k,kanR+kan,0.015042,0.017779581,0.016860586,0.018680673,False +fit_joint_s1_full_from_plain_lib20,growth_k,kanR-kan,0.023923,0.029037233,0.027302386,0.030687595,False +fit_joint_s1_full_from_plain_lib20,growth_k,pheS+4CP,0.003107,0.0056578703,0.0041585388,0.0071753864,False +fit_joint_s1_full_from_plain_lib20,growth_k,pheS-4CP,0.022042,0.027364645,0.025275411,0.0294674,False +fit_joint_s1_full_from_plain_lib20,growth_m,kanR+kan,-0.014122,-0.014273547,-0.014312269,-0.014231038,False +fit_joint_s1_full_from_plain_lib20,growth_m,kanR-kan,0.0,-1.1004556e-05,-5.270699e-05,2.7652484e-05,True +fit_joint_s1_full_from_plain_lib20,growth_m,pheS+4CP,0.014379,0.015366033,0.015236356,0.015508619,False +fit_joint_s1_full_from_plain_lib20,growth_m,pheS-4CP,0.0,7.2036037e-06,-2.1223015e-05,3.653898e-05,True +fit_joint_s1_full_from_plain_lib20,warp_log_a,kanR+kan,2.0794415416798357,2.1142178,2.102064,2.1257231,False +fit_joint_s1_full_from_plain_lib20,warp_log_a,pheS+4CP,0.4054651081081644,-0.82945144,-1.798458,0.044557422,False +fit_joint_s1_full_from_plain_lib20,warp_x0,kanR+kan,0.5,0.5354536,0.52431077,0.5475014,False +fit_joint_s1_full_from_plain_lib20,warp_x0,pheS+4CP,0.4,0.4527818,-0.44477156,1.380398,True +fit_joint_s1_full_from_plain_lib20,log_s,global,-0.6931471805599453,-0.73761225,-0.77925617,-0.69210213,True +fit_joint_s1_full_from_plain_lib20,binding_offset,wt,-0.1056078950314186,-0.12130986,-0.14442615,-0.03954602,True +fit_joint_s1_full_from_plain_lib20,binding_offset,R35V,0.0,0.0066577913,-0.027562032,0.035075385,True +fit_joint_s1_full_from_plain_lib20,binding_offset,E39G,0.0,-0.014689369,-0.058493417,0.04300828,True +fit_joint_s1_full_from_plain_lib20,binding_offset,M42I,0.0282415842244401,-0.0024007813,-0.033313878,0.02533397,False +fit_joint_s1_full_from_plain_lib20,binding_offset,M42R,0.0,-0.028005581,-0.059273943,0.0022332526,True +fit_joint_s1_full_from_plain_lib20,binding_offset,I48T,0.0,-0.006387547,-0.044981755,0.038059637,True +fit_joint_s1_full_from_plain_lib20,binding_offset,P49R,0.0,0.006776956,-0.029772121,0.035331957,True +fit_joint_s1_full_from_plain_lib20,binding_offset,R51S,0.0,0.0026306093,-0.02836745,0.03084597,True +fit_joint_s1_full_from_plain_lib20,binding_offset,Q54N,0.0,0.0031762463,-0.037408613,0.03518658,True +fit_joint_s1_full_from_plain_lib20,binding_offset,G58P,0.0,0.009631575,-0.024174592,0.042895533,True +fit_joint_s1_full_from_plain_lib20,binding_offset,I64S,0.0,0.004168044,-0.03434479,0.036891572,True +fit_joint_s1_full_from_plain_lib20,binding_offset,T68P,0.0,-0.06675541,-0.09764926,-0.011501151,False +fit_joint_s1_full_from_plain_lib20,binding_offset,S69Y,0.0,-0.010365965,-0.04579498,0.020010848,True +fit_joint_s1_full_from_plain_lib20,binding_offset,A72L,0.0,-0.002680641,-0.034203846,0.024273165,True +fit_joint_s1_full_from_plain_lib20,binding_offset,L73A,0.0,-0.032399192,-0.05919222,0.0023936494,True +fit_joint_s1_full_from_plain_lib20,binding_offset,L73T,0.0,-0.02334304,-0.051212806,0.013600973,True +fit_joint_s1_full_from_plain_lib20,binding_offset,H74A,-0.0420857002548871,-0.05864109,-0.08231991,-0.01892689,True +fit_joint_s1_full_from_plain_lib20,binding_offset,I79R,0.0,-0.0011421873,-0.028837943,0.029050298,True +fit_joint_s1_full_from_plain_lib20,binding_offset,K84L,0.0425148735331908,-0.006878795,-0.033184737,0.019459294,False +fit_joint_s1_full_from_plain_lib20,binding_offset,A87Y,0.0,-0.02838487,-0.055748343,0.010387307,True +fit_joint_s1_full_from_plain_lib20,binding_offset,D88A,0.0511536419917619,0.030758113,-0.0026694841,0.06234709,True +fit_joint_s1_full_from_plain_lib20,binding_offset,L90F,0.0,-0.037861533,-0.07017803,0.014069745,True +fit_joint_s1_full_from_plain_lib20,binding_offset,A92T,0.0,-0.015601618,-0.041187625,0.015074927,True +fit_joint_s1_full_from_plain_lib20,binding_offset,S93D,0.0,-0.032036737,-0.06792169,0.008784562,True +fit_joint_s1_full_from_plain_lib20,binding_offset,M98C,0.0,-0.020271339,-0.04684171,0.006408527,True +fit_joint_s1_full_from_plain_lib20,binding_offset,M42I/H74A,-0.1159296726932429,-0.14236295,-0.16812178,-0.06912915,True +fit_joint_s1_full_from_plain_lib20,binding_offset,M42I/K84L,-0.0590434943289043,-0.09321104,-0.11819824,-0.045555465,True +fit_joint_s1_full_from_plain_lib20,binding_offset,H74A/K84L,-0.0228535367478158,-0.034255706,-0.07295843,0.011829977,True +fit_joint_s1_full_from_plain_lib20,binding_offset,M42I/H74A/K84L,0.0833342596669661,0.062235218,-0.0064693503,0.08571177,True +fit_joint_s1_full_from_plain_lib20,binding_offset_scale,global,0.1,0.23241907,0.104634695,0.5254489,False diff --git a/planning/studies/joint-fit-sim/out_run8/theta.csv b/planning/studies/joint-fit-sim/out_run8/theta.csv new file mode 100644 index 00000000..d1ac7a50 --- /dev/null +++ b/planning/studies/joint-fit-sim/out_run8/theta.csv @@ -0,0 +1,31 @@ +run,group,n_genotypes,rmse,bias,coverage95 +fit_null_s1_plain,single,946,0.056310164351737704,-0.002109800618809592,0.102669133192389 +fit_null_s1_plain,spike,8,0.03564173580072257,0.005594124974061381,0.421875 +fit_null_s1_plain,wt,1,0.039885856632109154,-0.008674691594757433,0.0 +fit_null_s1_plain_lib20,single,926,0.03742019087357716,0.006280591514255646,0.31128509719222464 +fit_null_s1_plain_lib20,"single, binding",20,0.02948787173413293,0.002053183470534506,0.23125 +fit_null_s1_plain_lib20,spike,8,0.028131534122881947,0.009055072049138246,0.484375 +fit_null_s1_plain_lib20,wt,1,0.01777155343423133,0.0017060284501499686,0.125 +fit_null_s1_plain_weakpool_lib20,single,926,0.03641671775598344,0.006851190370131859,0.3334233261339093 +fit_null_s1_plain_weakpool_lib20,"single, binding",20,0.028394803008836775,0.0025508700755496713,0.24375 +fit_null_s1_plain_weakpool_lib20,spike,8,0.026920229825424385,0.008794566164607782,0.390625 +fit_null_s1_plain_weakpool_lib20,wt,1,0.0169375881728929,0.0025135766127508197,0.125 +fit_nocong_s1_plain_single_lib20,single,926,0.020627103634744764,-0.002397537332113211,0.35461663066954646 +fit_nocong_s1_plain_single_lib20,"single, binding",20,0.018520133288800207,8.424415839208539e-05,0.4 +fit_nocong_s1_plain_single_lib20,spike,8,0.012215949671842655,0.0008572381069769292,0.703125 +fit_nocong_s1_plain_single_lib20,wt,1,0.014084600170187629,-0.003690915351246332,0.0 +fit_nocong_s1_plain_single_lib50,single,896,0.017310637101799704,0.0003807505451593771,0.5129743303571429 +fit_nocong_s1_plain_single_lib50,"single, binding",50,0.014311556027633864,1.7586267188102147e-06,0.58 +fit_nocong_s1_plain_single_lib50,spike,8,0.011325124552406352,0.0024836520634945914,0.765625 +fit_nocong_s1_plain_single_lib50,wt,1,0.009545868703611097,-0.0008252714673179138,0.125 +fit_joint_s1_full_from_plain,single,946,0.07012804425293707,0.004091526192144923,0.16596194503171247 +fit_joint_s1_full_from_plain,spike,8,0.038620122460039444,0.020425483904325994,0.484375 +fit_joint_s1_full_from_plain,wt,1,0.05676126023676298,-0.0053830364664834325,0.25 +fit_joint_s1_plain_lib20,single,926,0.1234452092416617,0.03522000583807315,0.1345842332613391 +fit_joint_s1_plain_lib20,"single, binding",20,0.11332281121631436,0.02392954237848126,0.15 +fit_joint_s1_plain_lib20,spike,8,0.0560554303999554,0.009719645536402303,0.25 +fit_joint_s1_plain_lib20,wt,1,0.1294237078331936,0.02906087291808343,0.125 +fit_joint_s1_full_from_plain_lib20,single,926,0.04834778668533071,0.016795755583481432,0.37405507559395246 +fit_joint_s1_full_from_plain_lib20,"single, binding",20,0.03720021546543201,0.01387537335109203,0.4125 +fit_joint_s1_full_from_plain_lib20,spike,8,0.03157813125665683,0.025924187186855078,0.40625 +fit_joint_s1_full_from_plain_lib20,wt,1,0.029264566946553074,0.010577851409671971,0.125 diff --git a/planning/studies/joint-fit-sim/simulate.srun b/planning/studies/joint-fit-sim/simulate.srun new file mode 100644 index 00000000..03152780 --- /dev/null +++ b/planning/studies/joint-fit-sim/simulate.srun @@ -0,0 +1,42 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=jfsim_sim +#SBATCH --partition=gpu +#SBATCH --time=06:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# One simulation of the joint-fit simulation test +# (planning/studies/joint-fit-sim). Run from the study's cluster directory: +# +# sbatch --export=ALL,ARM=joint,SEED=1 --output=sim_joint.out simulate.srun +# +# ARM joint | null (make_sim.py) +# SEED simulation seed (default 1) +# BASE base simulate config (default: full-sized-sims-3/sim_realistic_s2) +# +# Writes sim_${ARM}_s${SEED}/. The direct growth table only (--no_write_raw): +# stage 1 reads tfs_sim_growth.csv, which tfs-process-counts reproduces row +# for row from the raw output. The simulation takes about 90 minutes. + +module load cuda/12.4.1 +set -e +ARM="${ARM:?set ARM}" +SEED="${SEED:-1}" +BASE="${BASE:-/gpfs/projects/harmslab/harms/studies/full-sized-sims-3/sim_realistic_s2/simulate_config.yaml}" +STUDY=/gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim +OUT="sim_${ARM}_s${SEED}" + +if [[ -f "${OUT}/tfs_sim_binding.csv" ]]; then + echo ">>> ${OUT}: simulated, skipped" + exit 0 +fi +echo ">>> make_sim ${ARM} ${SEED} $(date)" +python "${STUDY}/make_sim.py" "${BASE}" "${OUT}" --arm "${ARM}" --seed "${SEED}" +cd "${OUT}" +echo ">>> simulate $(date)" +tfs-simulate simulate_config.yaml --out_prefix tfs_sim --seed "${SEED}" --no_write_raw +echo ">>> done $(date)" diff --git a/planning/studies/joint-fit-sim/stage1.py b/planning/studies/joint-fit-sim/stage1.py new file mode 100644 index 00000000..09e79a94 --- /dev/null +++ b/planning/studies/joint-fit-sim/stage1.py @@ -0,0 +1,218 @@ +""" +Joint stage 1 on a simulation (README): wt, spikes and singles, growth and +binding together. + + python stage1.py SIM_DIR OUT_DIR --model full|plain [--seed 1] [--max_num_epochs N] + [--init_from OTHER_FIT/tfs_fit_model_params.npz] + [--pin_km_truth] [--no_mixture] [--set_priors NAME=VALUE ...] + [--add_binding_singles N] [--binding_seed 1] + [--congression_sets N1 N2 N3] + +SIM_DIR holds a tfs-simulate run of make_sim.py's config (simulate_config.yaml, +tfs_sim_growth.csv, tfs_sim_binding.csv). Each step is skipped when its +output exists, so a rerun resumes. + +Both models: hill_geno (absolute theta, shared by both markers), the +congression mixture with the homodimer rule (lambda prior 0.357 +- 0.15), +binding at weight 1, the count likelihood with level tube offsets +(SD 0.17), log(n) population SD 0.5, one k and m per condition, loose +k/m priors (growth_priors_loose.csv's values). + +- full: plus a logistic warp of occupancy on both selection conditions, + the growth titrant scale s (log s ~ Normal(0, 1)) and one binding offset + per binding genotype (genotype_offset). +- plain: none of those (linear growth in theta, s = 1, binding as + measured). + +--init_from starts the MAP at another fit's MAP (tfs-fit-model --init_from; +not staged): run 1's full fit on the null simulation stopped in a wrong +mode about 1,000 nats above the plain fit, so the full model is started +from the plain one, its extra sites (warps, s, offsets) at their guesses. + +--pin_km_truth pins every condition's k and m at the simulation's truth +(tfs_sim_growth_parameters.csv; growth_priors k_pinned/m_pinned): run 3's +fits put m 15-25% steep even with the true model, and the loss at the +true k and m against the free fit's says whether stage 1 prefers that m +or only drifts along a flat direction. + +--no_mixture fits without the congression mixture (transformation +single), for a simulation without congression (make_sim.py --arm nocong). + +--set_priors adds tfs-configure-model --set_priors items to the study's +own (run 6: the theta population SDs held, or their locations' priors +moved). + +--add_binding_singles N measures N random singles with growth data as +well, on the lab's 2 uM grid from their true Hill curves with the spikes' +noise (make_sim.BINDING_IPTG, BINDING_NOISE), appended to the simulated +binding table in OUT_DIR/binding.csv (run 7: run 6 found the singles' +theta squeezed along a near-symmetry of the growth model that only the +binding genotypes resist). + +--congression_sets sets the mixture's co-resident draws (tfs-configure-model +--congression_sets; default 12 3 1): run 8 left the congression simulation's +m bias at twice the no-congression one with binding singles. +""" + +import argparse +import os + +import numpy as np +import pandas as pd + +from tfscreen.genetics.library_design import library_composition_table +from make_sim import BINDING_IPTG, BINDING_NOISE +from tfscreen.tfmodel import two_stage as ts +from tfscreen.tfmodel.scripts.configure_model_cli import configure_model +from tfscreen.tfmodel.scripts.extract_params_cli import extract_params +from tfscreen.tfmodel.scripts.fit_model_cli import fit_model +from tfscreen.tfmodel.scripts.predict_theta_cli import predict_theta +from tfscreen.tfmodel.scripts.sample_posterior_cli import sample_posterior + +CONFIG = "tfs_configure_config.yaml" +CHECKPOINT = "tfs_fit_model_checkpoint.pkl" +LOOSE = dict(k_loc=0.015, k_scale=0.01, m_loc=0.0, m_scale=0.01) + + +def growth_priors(growth_csv, warp, truth=None): + """ + Loose k/m priors for every condition; m_warp on selection conditions; + with ``truth`` (a growth-parameters table), k and m pinned at it. + """ + conds = set() + for chunk in pd.read_csv(growth_csv, usecols=["condition_pre", "condition_sel"], + chunksize=1_000_000): + conds |= set(chunk["condition_pre"]) | set(chunk["condition_sel"]) + t = pd.DataFrame({"condition_rep": sorted(conds)}).assign(**LOOSE) + if warp: + t["m_warp"] = t["condition_rep"].str.contains(r"\+").astype(int) + if truth is not None: + tr = truth.set_index("condition_rep") + missing = sorted(set(t["condition_rep"]) - set(tr.index)) + if missing: + raise ValueError(f"no growth truth for {missing}") + t["k_loc"] = t["condition_rep"].map(tr["growth_k"]) + t["m_loc"] = t["condition_rep"].map(tr["growth_m"]) + t["k_pinned"] = 1 + t["m_pinned"] = 1 + return t + + +def binding_singles(sim, genotypes, num, seed): + """Binding rows for num random singles, from their true Hill curves.""" + truth = pd.read_csv(os.path.join(sim, "tfs_sim_parameters.csv")) + truth = truth.assign(genotype=truth["genotype"].astype(str)).set_index("genotype") + spiked = pd.read_csv(os.path.join(sim, "tfs_sim_binding.csv"))["genotype"] + pool = sorted(g for g in genotypes + if ts.is_single(g) and g in truth.index and g not in set(spiked)) + if num > len(pool): + raise ValueError(f"only {len(pool)} singles to choose from") + rng = np.random.default_rng([seed, 7]) + chosen = rng.choice(pool, size=num, replace=False) + c = np.asarray(BINDING_IPTG, dtype=float) + lc = np.log(np.where(c == 0, 1e-20, c)) + rows = [] + for g in chosen: + r = truth.loc[g] + th = r["theta_low"] + (r["theta_high"] - r["theta_low"]) / ( + 1 + np.exp(-r["hill_n"] * (lc - r["log_hill_K"]))) + obs = th + rng.normal(0.0, BINDING_NOISE, size=len(c)) + rows.extend(dict(genotype=g, titrant_name="iptg", titrant_conc=float(ci), + theta_obs=float(o), theta_std=BINDING_NOISE) + for ci, o in zip(c, obs)) + return pd.DataFrame(rows) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("sim_dir") + ap.add_argument("out_dir") + ap.add_argument("--model", choices=["full", "plain"], required=True) + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--max_num_epochs", type=int, default=None) + ap.add_argument("--init_from", default=None) + ap.add_argument("--pin_km_truth", action="store_true") + ap.add_argument("--no_mixture", action="store_true") + ap.add_argument("--set_priors", nargs="*", default=[]) + ap.add_argument("--add_binding_singles", type=int, default=0) + ap.add_argument("--binding_seed", type=int, default=1) + ap.add_argument("--congression_sets", type=int, nargs="+", default=None) + args = ap.parse_args() + sim = os.path.abspath(args.sim_dir) + init_from = None if args.init_from is None else os.path.abspath(args.init_from) + full = args.model == "full" + os.makedirs(args.out_dir, exist_ok=True) + os.chdir(args.out_dir) + + library_config = os.path.join(sim, "simulate_config.yaml") + if not os.path.exists("tfs_growth.csv"): + lib = library_composition_table(library_config) + src = os.path.join(sim, "tfs_sim_growth.csv") + keep = ts.stage1_genotypes(lib, ts.growth_genotypes(src)) + ts.filter_growth_csv(src, "tfs_growth.csv.partial", genotypes=keep) + os.replace("tfs_growth.csv.partial", "tfs_growth.csv") + print(f"stage 1: {len(keep)} genotypes (wt, spikes, singles)", flush=True) + + binding_csv = os.path.join(sim, "tfs_sim_binding.csv") + if args.add_binding_singles > 0: + binding_csv = os.path.abspath("binding.csv") + if not os.path.exists(binding_csv): + extra = binding_singles(sim, ts.growth_genotypes("tfs_growth.csv"), + args.add_binding_singles, args.binding_seed) + base = pd.read_csv(os.path.join(sim, "tfs_sim_binding.csv")) + pd.concat([base, extra[base.columns]], ignore_index=True).to_csv( + binding_csv, index=False) + print(f"binding: {base['genotype'].nunique()} spikes plus " + f"{extra['genotype'].nunique()} singles", flush=True) + + if not os.path.exists(CONFIG): + truth = (pd.read_csv(os.path.join(sim, "tfs_sim_growth_parameters.csv")) + if args.pin_km_truth else None) + growth_priors("tfs_growth.csv", warp=full, truth=truth).to_csv( + "growth_priors.csv", index=False) + set_priors = ["sigma_fixed=0.17", "theta_log_hill_n_hyper_scale_fixed=0.5"] + if full: + set_priors.append("theta_growth_log_conc_shift_scale=1.0") + extra = {i.split("=")[0] for i in args.set_priors} + set_priors = [i for i in set_priors if i.split("=")[0] not in extra] + set_priors += list(args.set_priors) + mixture = ({} if args.no_mixture else + dict(transformation_model="mixture", + transformation_lambda=[0.357, 0.15], + congression_theta_rule="homodimer")) + if args.congression_sets is not None: + if args.no_mixture: + raise ValueError("--congression_sets needs the mixture") + mixture["congression_sets"] = list(args.congression_sets) + configure_model( + binding_df=binding_csv, + growth_df=os.path.abspath("tfs_growth.csv"), + library_config=library_config, + theta_model="hill_geno", + **mixture, + theta_binding_noise_model="genotype_offset" if full else "zero", + growth_shares_replicates=True, + batch_size=4096, + allow_missing_spikes=True, + growth_priors=os.path.abspath("growth_priors.csv"), + set_priors=set_priors, + skip_model_stats=True, + out_prefix="tfs_configure") + + fit_kw = {} if args.max_num_epochs is None else {"max_num_epochs": args.max_num_epochs} + if init_from is not None: + fit_kw["init_from"] = init_from + if not os.path.exists("tfs_fit_model_params.npz"): + fit_model(CONFIG, seed=args.seed, analysis_method="map", **fit_kw) + if not os.path.exists("tfs_posterior.h5"): + sample_posterior(CONFIG, CHECKPOINT, out_prefix="tfs_posterior", + num_posterior_samples=500, skip_growth_observations=True) + if not os.path.exists("tfs_params_provenance.json"): + extract_params(CONFIG, "tfs_posterior.h5", out_prefix="tfs_params") + if not os.path.exists("tfs_pred_theta.csv"): + predict_theta(CONFIG, "tfs_posterior.h5", out_prefix="tfs_pred_theta") + print("stage 1 done", flush=True) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/joint-fit-sim/stage1.srun b/planning/studies/joint-fit-sim/stage1.srun new file mode 100644 index 00000000..2b5e972a --- /dev/null +++ b/planning/studies/joint-fit-sim/stage1.srun @@ -0,0 +1,74 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=jfsim_fit +#SBATCH --partition=gpu +#SBATCH --time=06:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# Joint stage 1 on one simulation (planning/studies/joint-fit-sim). Run from +# the study's cluster directory after simulate.srun: +# +# sbatch --export=ALL,SIM=sim_joint_s1,MODEL=full --output=fit_joint_full.out stage1.srun +# +# SIM simulation directory (from simulate.srun) +# MODEL full | plain +# SEED fit seed (default 1) +# INIT optional: start at this fit's MAP (e.g. fit_null_s1_plain); the +# output is then fit_${SIM#sim_}_${MODEL}_from_${INIT#fit_*_s1_} +# PIN optional: 1 pins k and m at the simulation's truth (stage1.py +# --pin_km_truth); output suffix _pinkm +# NOMIX optional: 1 fits without the congression mixture (stage1.py +# --no_mixture); output suffix _single +# PRIORS optional: extra --set_priors items, separated by ':' (sbatch +# --export splits on commas); give TAG with it +# TAG optional: output suffix _${TAG} +# CSETS optional: the mixture's co-resident sets, ':'-separated (e.g. +# 50:20:5; stage1.py --congression_sets); name the arm with TAG +# ADDB optional: N extra binding singles (stage1.py +# --add_binding_singles); name the arm with TAG +# BSEED optional: which random singles (stage1.py --binding_seed, +# default 1) +# +# Writes fit_${SIM#sim_}_${MODEL}/; a rerun resumes. + +module load cuda/12.4.1 +set -e +SIM="${SIM:?set SIM}" +MODEL="${MODEL:?set MODEL}" +SEED="${SEED:-1}" +STUDY=/gpfs/home/harms/tfscreen/planning/studies/joint-fit-sim +OUT="fit_${SIM#sim_}_${MODEL}" +EXTRA=() +if [[ -n "${INIT:-}" ]]; then + OUT="${OUT}_from_${INIT#fit_*_s1_}" + EXTRA=(--init_from "${INIT}/tfs_fit_model_params.npz") +fi +if [[ "${NOMIX:-0}" == 1 ]]; then + OUT="${OUT}_single" + EXTRA+=(--no_mixture) +fi +if [[ -n "${PRIORS:-}" ]]; then + IFS=':' read -r -a ITEMS <<< "${PRIORS}" + EXTRA+=(--set_priors "${ITEMS[@]}") +fi +if [[ -n "${ADDB:-}" ]]; then + EXTRA+=(--add_binding_singles "${ADDB}" --binding_seed "${BSEED:-1}") +fi +if [[ -n "${CSETS:-}" ]]; then + IFS=':' read -r -a SETS <<< "${CSETS}" + EXTRA+=(--congression_sets "${SETS[@]}") +fi +if [[ -n "${TAG:-}" ]]; then + OUT="${OUT}_${TAG}" +fi +if [[ "${PIN:-0}" == 1 ]]; then + OUT="${OUT}_pinkm" + EXTRA+=(--pin_km_truth) +fi +echo ">>> stage 1 ${SIM} ${MODEL} -> ${OUT} $(date)" +python "${STUDY}/stage1.py" "${SIM}" "${OUT}" --model "${MODEL}" --seed "${SEED}" "${EXTRA[@]}" +echo ">>> done $(date)" diff --git a/planning/studies/km-anchor/README.md b/planning/studies/km-anchor/README.md new file mode 100644 index 00000000..834c6e3b --- /dev/null +++ b/planning/studies/km-anchor/README.md @@ -0,0 +1,159 @@ +# Anchoring k and m with the wt monoculture rates + +**Question.** On the full-size simulation the deep genotypes rank well (X r +0.99 above 1e4 reads) but their intervals do not cover (95% coverage 0.29 +realistic, 0.02 Poisson), because k and m are off and held at the MAP: X is +compressed by |m| 30-45% too large, and k slid up against dk_geno. Counts +measure relative frequencies, so only the tube totals pin the absolute +scale, weakly. The experiment has one more measurement of that scale, the +wt monoculture growth rates. Does a k/m prior from them bring deep coverage +up while keeping the ranking? Does the real data accept that anchor? + +**Decision it feeds.** `planning/deep-coverage.md`, step 1: whether the +real fit's k and m get their priors from the monoculture, and what the +deep genotypes' intervals can claim. + +**Why it matters for the real data.** In kanR+kan the real fit and the +simulation are off the same way against the monoculture rates, which are +the simulation's truth: + +| kanR+kan | k | m | +|---|---|---| +| wt monoculture (= sim truth) | 0.0150 | -0.0141 | +| real fit (recipe) | 0.0298 | -0.0204 | +| simulated fit, realistic | 0.0274 | -0.0203 | +| simulated fit, Poisson | 0.0189 | -0.0182 | + +In the simulation that is a property of the fit, so the real kanR curves +are likely compressed the same way. pheS+4CP differs: the real fit's k is +0.023 against 0.003 for the monoculture, and its |m| is smaller (0.0097 +against 0.0144), which may be the real selection-onset effect +(`planning/offset-mode-growth-transition.md`). + +## Arms + +All three refit an existing growth table with the recipe's model (growth- +only relative fit, level offsets at SD 0.17, log(n) population SD 0.5, +loose k/m priors in the control conditions), adding k/m priors in the two +selection conditions from the wt monoculture rates +(`--growth_priors_wt_rates`: k = wt rate at 1 mM, m = rate at 0 minus rate +at 1 mM, SD = standard error of the replicate mean floored at 0.002 per +minute). + +| arm | growth table | rates | +|---|---|---| +| `anchor_sim` | `sim_realistic_s2` | the monoculture rates (the simulation's truth, at the monoculture's SD) | +| `anchor_sim_perturbed` | `sim_realistic_s2` | the same, each rate moved once by a draw of its SD (seed 1) | +| `anchor_real` | the recipe's `tfs_growth.csv` | the monoculture rates | + +Only the selection conditions get monoculture priors (`make_wt_rates.py`): +in the control conditions wt's monoculture rate varies with IPTG in ways +an m of 0 cannot carry, and the simulation sets their m to 0. The rate +tables are derived from lab data, so they are generated on the cluster and +never committed. + +## How to run + +On the cluster, after pulling this commit. Simulation arms, from the +full-size simulation's working directory (it holds `sim_realistic_s2/` and +the `processed/` link): + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/km-anchor/{run_anchor.srun,make_wt_rates.py} . +``` + +```bash +python make_wt_rates.py processed/wt_control_rate_summary.csv --seed 1 +``` + +It prints the 0 and 1 mM rates and their perturbed values; check that it +wrote `wt_rates_selection.csv` and `wt_rates_selection_perturbed.csv`. +`growth_priors_loose.csv` is already in this directory from the full-size +run. + +```bash +mkdir anchor_sim && cd anchor_sim && sbatch --export=ALL,GROWTH=../sim_realistic_s2/tfs_growth.csv,LIBRARY=../sim_realistic_s2/library_config.yaml,WT_RATES=../wt_rates_selection.csv,TRUTH_DIR=../sim_realistic_s2 ../run_anchor.srun && cd .. +``` + +```bash +mkdir anchor_sim_perturbed && cd anchor_sim_perturbed && sbatch --export=ALL,GROWTH=../sim_realistic_s2/tfs_growth.csv,LIBRARY=../sim_realistic_s2/library_config.yaml,WT_RATES=../wt_rates_selection_perturbed.csv,TRUTH_DIR=../sim_realistic_s2 ../run_anchor.srun && cd .. +``` + +Real arm, next to the recipe run: + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/km-anchor/{run_anchor.srun,make_wt_rates.py} . +``` + +```bash +python make_wt_rates.py ../processed/wt_control_rate_summary.csv --seed 1 +``` + +```bash +mkdir anchor_real && cd anchor_real && sbatch --export=ALL,GROWTH=../recipe/tfs_growth.csv,LIBRARY=../../processed/library_config.yaml,WT_RATES=../wt_rates_selection.csv ../run_anchor.srun && cd .. +``` + +`growth_priors_loose.csv` is already in `real_fit/` from the recipe. Each +arm ends with `>>> Done` in its `run.out` and a +`summary/tfs_summarize_fit_summary.json`. + +## Inputs + +`sim_realistic_s2` (`planning/studies/full-size-sim/`), the recipe's growth +table (`planning/studies/real-data-fit/`), and +`processed/wt_control_rate_summary.csv` (the wt monoculture rates, from +`prep_dev_data.py`). + +## Commit + +The commit that adds this study. Every step writes its provenance. + +## Results + +Run 2026-10-09 (all three arms converged at every stage; offsets clean in +all). **The data reject the anchor, in the simulation and in the real +data.** + +| arm | condition | k prior (SD) | k fit | m prior (SD) | m fit | truth k, m | +|---|---|---|---|---|---|---| +| unanchored sim (`sim_realistic_s2`) | kanR+kan | 0.015 (0.01) | 0.0274 | 0 (0.01) | -0.0203 | 0.0149, -0.0138 | +| `anchor_sim` | kanR+kan | 0.0150 (0.002) | 0.0247 | -0.0141 (0.002) | -0.0184 | 0.0149, -0.0138 | +| `anchor_sim_perturbed` | kanR+kan | 0.0156 (0.002) | 0.0255 | -0.0140 (0.002) | -0.0199 | 0.0149, -0.0138 | +| `anchor_sim` | pheS+4CP | 0.0031 (0.002) | 0.0065 | 0.0144 (0.002) | 0.0183 | 0.0033, 0.0141 | +| recipe (real) | kanR+kan | 0.015 (0.01) | 0.0298 | 0 (0.01) | -0.0204 | | +| `anchor_real` | kanR+kan | 0.0150 (0.002) | 0.0249 | -0.0141 (0.002) | -0.0202 | | +| recipe (real) | pheS+4CP | 0.015 (0.01) | 0.0226 | 0 (0.01) | 0.0097 | | +| `anchor_real` | pheS+4CP | 0.0031 (0.002) | 0.0172 | 0.0144 (0.002) | 0.0097 | | + +- k moves by a pure slide: on the real data it fell by 0.005 in every + condition, the unanchored controls included, with dk_geno up by 0.0048 + everywhere; it still ends five prior SDs above the monoculture. +- m does not move. The real kanR m stays at -0.020 against a prior of + -0.014 +/- 0.002 (pheS 0.0097 against 0.0144), so the likelihood pins it. + In the simulation, given the truth as its prior, the fit still puts m at + -0.018 to -0.020 against -0.0138: the fitted model prefers the wrong m. +- Deep coverage barely improves (`anchor_sim`, X 95% coverage 0.65 / + 0.35 / 0.22 at 1e3-1e4 / 1e4-1e5 / >1e5 reads, against 0.56 / 0.29 / + 0.08 unanchored); X slope 1.08-1.23 against 1.19-1.37. +- The real fit is otherwise unchanged (X shifts by 0.03, log K and n by + under 0.01, 9 held directions). + +So the |m| inflation is a misspecification of the fit against how the +simulation makes its data, not a weakly pinned ridge, and the simulation +reproduces the real kanR numbers, so the real kanR X is most likely +compressed by the same factor (~1.4). Ruled out: the OD-derived tube totals +(unbiased in the simulation, drift <= 0.001 per minute, 96% of tubes in the +calibrated range) and the 2026-10-08 simulator features (the bias is in the +s1b arms, which had neither unassigned reads nor the excess wt). Suspects: +congression (the simulation's lambda 0.357 with the homodimer rule; the fit +uses `single`), per-tube rate noise (the fit has level offsets), and the +dk_geno prior's shape. Next: a reduced-library factorial that switches each +off (`planning/deep-coverage.md`, step 1b). diff --git a/planning/studies/km-anchor/growth_priors_loose.csv b/planning/studies/km-anchor/growth_priors_loose.csv new file mode 100644 index 00000000..4cfb4e48 --- /dev/null +++ b/planning/studies/km-anchor/growth_priors_loose.csv @@ -0,0 +1,5 @@ +condition_rep,k_loc,k_scale,m_loc,m_scale +kanR+kan,0.015,0.01,0.0,0.01 +kanR-kan,0.015,0.01,0.0,0.01 +pheS+4CP,0.015,0.01,0.0,0.01 +pheS-4CP,0.015,0.01,0.0,0.01 diff --git a/planning/studies/km-anchor/make_wt_rates.py b/planning/studies/km-anchor/make_wt_rates.py new file mode 100644 index 00000000..6f2ab0fe --- /dev/null +++ b/planning/studies/km-anchor/make_wt_rates.py @@ -0,0 +1,50 @@ +""" +wt monoculture rate tables for the k/m anchor arms. + + python make_wt_rates.py [--seed 1] + +Writes, in the current directory: + +- wt_rates_selection.csv: the selection conditions only (condition_sel + containing '+'). The control conditions are left to the loose priors: in + them wt's monoculture rate varies with IPTG for reasons the fit's m = 0 + cannot carry, and the full-size simulation sets their m to 0. +- wt_rates_selection_perturbed.csv: the same, with each rate_mean moved + once by a Normal draw of the SD the prior assumes (the standard error of + the replicate mean, floored at 0.002 per minute, as + priors_edit.growth_priors_from_wt_rates does). The honest-anchor arm: a + monoculture is a measurement, not the truth. +""" + +import argparse + +import numpy as np +import pandas as pd + +SD_FLOOR = 0.002 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("rates") + ap.add_argument("--seed", type=int, default=1) + args = ap.parse_args() + + r = pd.read_csv(args.rates) + sel = r[r["condition_sel"].str.contains("+", regex=False)].copy() + sel.to_csv("wt_rates_selection.csv", index=False) + + rng = np.random.default_rng(args.seed) + se = sel["rate_sd"] / np.sqrt(sel["num_replicates"]) + sd = np.maximum(se.to_numpy(), SD_FLOOR) + pert = sel.copy() + pert["rate_mean"] = sel["rate_mean"].to_numpy() + rng.normal(0.0, sd) + pert.to_csv("wt_rates_selection_perturbed.csv", index=False) + + show = sel[["condition_sel", "titrant_conc", "rate_mean"]].assign( + perturbed=pert["rate_mean"].to_numpy(), sd=sd) + print(show[show["titrant_conc"].isin([0.0, 1.0])].round(5).to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/km-anchor/run_anchor.srun b/planning/studies/km-anchor/run_anchor.srun new file mode 100644 index 00000000..fbf593a8 --- /dev/null +++ b/planning/studies/km-anchor/run_anchor.srun @@ -0,0 +1,68 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_anchor +#SBATCH --output=run.out +#SBATCH --error=run.err +#SBATCH --partition=gpulong +#SBATCH --time=02-00:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# One k/m anchor arm (planning/studies/km-anchor): the recipe's model and +# chain, refit on an existing growth table with k and m priors in the +# selection conditions from the wt monoculture rates. Run from a new arm +# directory; the paths below are relative to it. +# +# GROWTH growth table (tfs-process-counts output) to refit +# LIBRARY library config the fit is configured with +# WT_RATES wt_rates_selection.csv or wt_rates_selection_perturbed.csv +# TRUTH_DIR (simulation arms) directory holding tfs_sim_*.csv; they are +# linked in so tfs-summarize-fit scores against the truth +# +# See README.md for the exact sbatch lines. + +module load cuda/12.4.1 + +set -e + +GROWTH="${GROWTH:?set GROWTH}" +LIBRARY="${LIBRARY:?set LIBRARY}" +WT_RATES="${WT_RATES:?set WT_RATES}" +SEED="${SEED:-1}" + +if [[ -n "${TRUTH_DIR:-}" ]]; then + for f in "${TRUTH_DIR}"/tfs_sim_*.csv; do + ln -sf "${f}" . + done +fi + +echo ">>> Configure (k/m priors from ${WT_RATES}) $(date)" +tfs-configure-model \ + --allow_missing_spikes \ + --growth_df "${GROWTH}" \ + --library_config "${LIBRARY}" \ + --theta_model hill_relative \ + --growth_shares_replicates \ + --batch_size 4096 \ + --growth_priors ../growth_priors_loose.csv \ + --growth_priors_wt_rates "${WT_RATES}" \ + --set_priors sigma_fixed=0.17 theta_log_hill_n_hyper_scale_fixed=0.5 + +echo ">>> Fit (staged MAP) $(date)" +tfs-fit-model tfs_configure_config.yaml --seed "${SEED}" --analysis_method map + +echo ">>> Arrowhead Laplace $(date)" +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --laplace arrowhead --num_posterior_samples 500 --skip_growth_observations + +echo ">>> Extract and predict $(date)" +tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 +tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Summarize $(date)" +tfs-summarize-fit . + +echo ">>> Done $(date)" diff --git a/planning/studies/m-bias-factorial/README.md b/planning/studies/m-bias-factorial/README.md new file mode 100644 index 00000000..83c0f0a3 --- /dev/null +++ b/planning/studies/m-bias-factorial/README.md @@ -0,0 +1,1515 @@ +# What makes the fit's m too steep + +**Question.** On the calibrated full-size simulation the fit's growth slope +m is about 40% too steep in kanR+kan (-0.018 to -0.020 against a true +-0.0138), even with the truth as a 0.002-SD prior +(`planning/studies/km-anchor/`); the real fit has the same m against the +wt monoculture. The fitted likelihood prefers the wrong m, so something +the simulation does is missing from the fit. Which of the simulation's +features the fit does not model causes it? + +**Decision it feeds.** `planning/deep-coverage.md`, step 1b: what the fit +has to model, or correct for, before the deep genotypes' intervals can be +honest. + +## Round 1: reduced-library simulations + +A reduced library (5 degenerate sites per tile, every spike site kept: +5,256 doubles instead of 223,000), scaled with +`genetics.library_design.scale_library_design` so each sequence keeps its +reads, cells, transformants and PCR templates, on the real design (118 +tubes) with the calibrated simulation of `planning/studies/full-size-sim/`. +The fit is the real recipe's (growth-only relative model, level offsets at +SD 0.17, loose k/m priors), MAP only; the slope does not need the Laplace. + +| arm | changes from the calibrated simulation | +|---|---| +| `base` | none | +| `no_cong` | no co-transformed cells (`transformation_poisson_lambda` 0) | +| `no_tube` | no per-tube rate noise (`tube_noise_sigma` 0) | +| `poisson` | Poisson counts (no founder, demographic or PCR noise) | +| `clean` | all three off | + +Two seeds each. `score_arms.py` puts the truth on the fit's X scale (wt's +true curve at the gauge concentrations) and reports k, m, the ratio +m_fit / m_true, and the median dk_geno error. + +`base` must reproduce the bias first; if it does not, the reduced library +changes the problem and the factorial says nothing. + +A local test (2026-10-09) ran the simulation, processing and configure in +minutes, but the staged MAP's first stage was still descending (about 1.8 +nats per window, t = 12) at step 80,000 after an hour on a laptop CPU, so +the arms run on the cluster. + +### How to run round 1 + +On the cluster, after pulling this commit, from the full-size simulation's +working directory (it holds `processed/` and `pilot_s1/`): + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_factorial.py --pilot pilot_s1 --out_dir m_bias +``` + +It builds the two full-size configs (a few minutes) and prints one line per +arm: 5256 doubles, scale 0.1612, reads 8.17e+08. Then: + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/{run_arm.sh,run_arm.srun,score_arms.py} m_bias/ +``` + +```bash +cd m_bias && for d in base_s1 base_s2 no_cong_s1 no_cong_s2 no_tube_s1 no_tube_s2 poisson_s1 poisson_s2 clean_s1 clean_s2; do (cd $d && sbatch ../run_arm.srun); done +``` + +Each arm ends with `>>> Done` in its `run.out` and a +`tfs_params_growth_m.csv`. Then, in `m_bias/`: + +```bash +python score_arms.py *_s1 *_s2 +``` + +It prints the table and writes `factorial_scores.csv`. + +Inputs: `processed/` (the dev data, via make_sim_config.py) and the +full-size pilot (`pilot_s1/`, for the cfu0 calibration). + +### Round 1 results (2026-10-09) + +**Inconclusive: `base` did not reproduce the bias.** m_fit / m_true on the +X scale (truth from wt's true curve at the gauge): + +| arm | kanR+kan s1 / s2 | pheS+4CP s1 / s2 | +|---|---|---| +| full size, realistic (`sim_realistic_s2`) | 1.47 | 1.43 | +| full size, Poisson (`sim_poisson_s2`) | 1.32 | 1.16 | +| `base` | 1.03 / 1.14 | 0.96 / 1.06 | +| `no_cong` | 0.99 / 0.96 | 1.03 / 0.99 | +| `no_tube` | 1.05 / 1.21 | 1.00 / 1.12 | +| `poisson` | 1.13 / 1.14 | 1.01 / 1.02 | +| `clean` | 1.04 / 1.05 | 1.04 / 1.05 | + +Every arm sits between 0.96 and 1.21. The seed-to-seed spread (`base` 1.03 +against 1.14) is as large as any difference between arms. Every fit +converged except `clean`, whose stage 1 and joint stage both reached the +epoch cap while still dropping a few nats per window. + +The reduced library changed the problem in two ways: + +- **Composition.** `scale_library_design` keeps each sequence's abundance, + so the spiked origin (wt repeated 271 times) keeps its mass while the + bulk shrinks. wt starts at 55% of the reduced pool against 11% at full + size, and doubles at 6% against 44% (`library_composition_table`). In + the sequenced tubes wt is 81% of reads against 31%, and doubles are 3% + against 35%. +- **Detection.** cfu0 scales with the pool, so each tube's OD600 falls by + the same factor. 25 of 118 tubes read below the detection threshold and + dropped out (93 sequenced against 118). + +The bias weakens when wt dominates the tubes. That fits the tube totals +pinning the absolute scale through wt when wt is most of the population, +and the library's genotypes setting it when they are. It does not say +which feature is misspecified. + +## Round 2: fit-side subsets of the full-size simulation + +Given the shared parameters (k, m, tube offsets, population hypers), each +genotype's count likelihood is independent: the count model observes each +genotype's reads against its predicted share of the tube's cells, with no +constraint that the shares sum to one. So refitting the full-size +`sim_realistic_s2` growth table (m ratio 1.47) with only some genotypes is a +valid fit of the same simulated data. It changes only which genotypes +inform k and m. `make_subsets.py` writes three arms: + +| arm | genotypes | +|---|---| +| `no_doubles` | wt, spikes and singles (952) | +| `deep_doubles` | those plus every double with at least 1,000 total reads (80,915) | +| `shallow_doubles` | those plus 80,915 of the 126,684 doubles below 1,000 reads, at random (seed 1) | + +Same chain as round 1 (`run_arm.sh` skips simulate and process because the +arm directory links the simulation's `tfs_sim_*` files and has its own +`tfs_growth.csv`). Readout: if `no_doubles` recovers m (ratio near 1), the +doubles drive the bias; `deep_doubles` against `shallow_doubles` says +whether it is the well-measured doubles or the noisy ones. If `no_doubles` +is still at about 1.4, the bias comes from how wt, spikes and singles are +modeled (or from the design), and congression and the dk_geno prior's +shape become the next arms at full size. + +### How to run round 2 + +On the cluster, after pulling this commit, from the full-size simulation's +working directory (it holds `sim_realistic_s2/`): + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_subsets.py sim_realistic_s2 --out_dir m_subsets +``` + +It reads the 8 GB growth table twice (about 6 minutes locally) and prints +`952 wt/spike/single genotypes, 80915 doubles >= 1000 reads, 80915 of +126684 below`, then one genotype count per arm (952, 81867, 81867). + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/{run_arm.sh,run_arm.srun,score_arms.py} m_subsets/ +``` + +```bash +cd m_subsets && for d in no_doubles deep_doubles shallow_doubles; do (cd $d && sbatch ../run_arm.srun); done +``` + +When each `run.out` ends with `>>> Done`, in `m_subsets/`: + +```bash +python score_arms.py no_doubles deep_doubles shallow_doubles +``` + +Inputs: `sim_realistic_s2/` (`planning/studies/full-size-sim/`). + +## Round 3: the full library with k and m held + +If the doubles bias m and wt, spikes and singles do not, then fitting the +full library with k and m held at the no-doubles MAP should recover X for +the doubles. `held_priors.py` turns an arm's fitted k and m into a +`--growth_priors` table, and `run_held.srun` runs the full-size chain +(staged MAP, arrowhead Laplace, predict, `tfs-summarize-fit`) with both +clamped (`--set_priors m_pinned=1 k_pinned=1`). The intervals then leave +out k and m's own uncertainty, so this checks X's slope and the coverage +given k and m, against `sim_realistic_s2`'s summary (X slope 1.19-1.37, +95% coverage 0.56 / 0.29 / 0.08 at 1e3-1e4 / 1e4-1e5 / >1e5 reads). A +20-epoch smoke test of the chain with both pins ran locally (configure, MAP, +arrowhead Laplace, extract). + +### How to run round 3 + +On the cluster, from the full-size simulation's working directory: + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +mkdir held_s2 && cd held_s2 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets/no_doubles --out growth_priors_held.csv +``` + +It prints the four conditions' k and m; kanR+kan should read k 0.0182, +m -0.0142. Then link the simulation's data and truth, and submit: + +```bash +ln -s ../sim_realistic_s2/tfs_growth.csv ../sim_realistic_s2/tfs_sim_* . && cp ../sim_realistic_s2/library_config.yaml . +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun . && sbatch run_held.srun +``` + +It ends with `>>> Done` in `run.out` and a `summary/` directory, like the +full-size runs (several hours: the full library and the Laplace). + +## Round 4: hold kanR only + +The real pheS+4CP departs from its monoculture with or without the doubles +(`planning/deep-coverage.md` step 3), so the real fit holds kanR only: +kanR+kan and kanR-kan at the no-doubles MAP by a tight prior (SD 1e-4; +the doubles' pull has a likelihood SD near 0.0014, so this gives up about +0.5% of it), pheS at the recipe's loose priors, and no pins (they act on +every condition). X is shared across markers, so the simulated arm checks +that holding kanR alone fixes X while pheS's m stays free: does pheS's m +follow the kanR-pinned X, or do the doubles pull it, and X with it, steep +again? The real arm is read only if the simulated one passes. + +| arm | data | held | +|---|---|---| +| `held_kanR_s2` | `sim_realistic_s2` | kanR at the simulated `no_doubles` MAP | +| `held_kanR_real` | the recipe's growth table | kanR at the real `no_doubles` MAP | + +### How to run round 4 + +`held_priors.py --hold` has since changed to write pin columns (round 5); +round 4 ran it at commit 75611b08, where `--hold` held the named +conditions by a tight prior (`--scale`, default 1e-4). + +Simulated arm, in the full-size simulation's working directory: + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +mkdir held_kanR_s2 && cd held_kanR_s2 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets/no_doubles --hold kanR --loose ../growth_priors_loose.csv --out growth_priors_held.csv +``` + +Check: kanR+kan k 0.018182, m -0.014221 at scale 0.0001; pheS rows 0.015, +0.01, 0, 0.01. + +```bash +ln -s ../sim_realistic_s2/tfs_growth.csv ../sim_realistic_s2/tfs_sim_* . && cp ../sim_realistic_s2/library_config.yaml . +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun . && sbatch --export=ALL,PINS= run_held.srun +``` + +Real arm, next to the recipe run: + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +mkdir held_kanR_real && cd held_kanR_real +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets_real/no_doubles --hold kanR --loose ../growth_priors_loose.csv --out growth_priors_held.csv +``` + +Check: kanR+kan k 0.016018, m -0.014136 at scale 0.0001. + +```bash +ln -s ../recipe/tfs_growth.csv . && cp ../../processed/library_config.yaml . +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun . && sbatch --export=ALL,PINS= run_held.srun +``` + +The log's configure line reads `pins: none`. Each run ends with `>>> Done` +in `run.out` and a `summary/` directory (about 2 hours, as round 3). + +## Round 5: kanR pinned per condition + +Round 4's tight prior leaked (see its result below), so round 5 repeats it +with `linear`'s per-condition pins: `held_priors.py --hold kanR` now writes +`k_pinned`/`m_pinned` columns (1 for kanR, 0 for pheS) into the growth +priors table, which `tfs-configure-model --growth_priors` turns into +per-condition clamps. kanR's k and m are then exact and leave the +Laplace; pheS's are fitted. It needs the commit that adds per-condition +pins on the cluster. + +### How to run round 5 + +Simulated arm: + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +mkdir held_kanR_pin_s2 && cd held_kanR_pin_s2 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets/no_doubles --hold kanR --loose ../growth_priors_loose.csv --out growth_priors_held.csv +``` + +Check: the table has `k_pinned` and `m_pinned` columns, 1 for both kanR +rows and 0 for pheS. + +```bash +ln -s ../sim_realistic_s2/tfs_growth.csv ../sim_realistic_s2/tfs_sim_* . && cp ../sim_realistic_s2/library_config.yaml . +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun . && sbatch --export=ALL,PINS= run_held.srun +``` + +Real arm: + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +mkdir held_kanR_pin_real && cd held_kanR_pin_real +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets_real/no_doubles --hold kanR --loose ../growth_priors_loose.csv --out growth_priors_held.csv +``` + +```bash +ln -s ../recipe/tfs_growth.csv . && cp ../../processed/library_config.yaml . +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun . && sbatch --export=ALL,PINS= run_held.srun +``` + +In `tfs_configure_priors.csv`, `growth.condition_growth.m_pinned` has one +row per condition (1, 1, 0, 0 by name). After the fit, +`tfs_params_growth_m.csv` reads kanR's held values exactly. + +## Round 6: the real data with every condition pinned + +Round 5 showed a free k on either marker reopens the k/dk_geno slide, and +the real pheS m with X set by kanR matches pheS's own no-doubles fit. So +the real fit pins all four conditions at the real no-doubles MAP, round +3's setup (`held_priors.py` without `--hold`, `run_held.srun` with its +default pins), in `real_fit/held_all_real`. The pheS curves stay +provisional until `planning/deep-coverage.md` step 3: the pool's pheS +selection is about half the wt monoculture's, and the simulation cannot +say whether that is biology or a misspecification it lacks. + +## Step 2: k and m's uncertainty in X + +`planning/deep-coverage.md` step 2. A held fit gives X intervals given k +and m. To carry their uncertainty, take joint draws of every condition's k +and m from the stage-1 fit's arrowhead Laplace, refit the full library +pinned at each draw (round 3's setup, starting from round 3's MAP with +`--init_from`), and pool the draws' posteriors into one equal-weight +mixture. The pooled intervals should cover the deepest genotypes better +than round 3 (0.65 above 1e5 reads) if stage 1's own error is what they +miss. The Laplace's k/m spread is only as honest as stage 1: on seed 1 +its pheS m was 11% off the truth, so if the spread comes out near 1%, the +draws cannot cover that and the second seed (below) says whether it was +luck. + +`km_draws.py` stops if the stage-1 Laplace holds a k or m direction at +zero width (it would give those draws no spread). `score_x.py` scores +any run directory or pooled table by depth; `depth.py` makes its read +table. + +### How to run step 2 (simulation, `sim_realistic_s2`) + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_km_draws.srun +``` + +`run_km_draws.out` prints each condition's k and m SD across the stage-1 +posterior and ends with `wrote 10 draws to km_draws/draw_*/`. If it stops +on held k/m directions instead, send me the table it prints. Then submit +the draws, each starting from round 3's MAP: + +```bash +cd km_draws && for d in draw_*; do (cd $d && ln -s ../../sim_realistic_s2/tfs_growth.csv ../../sim_realistic_s2/tfs_sim_* . && cp ../../sim_realistic_s2/library_config.yaml . && sbatch --export=ALL,INIT=../../held_s2/tfs_fit_model_params.npz /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun); done +``` + +When every `draw_*/run.out` ends with `>>> Done`, pool them (in +`km_draws/`): + +```bash +sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_pool.srun +``` + +Then score `km_draws/pooled_aggregate.csv` against round 3 with +`score_x.py` and round 3's truth table. + +## Second seed + +Stage 1's pheS m came out 11% low on the one simulation so far. +`sim_realistic_s2` is seed 1 (the "s2" is the second calibration of the +simulator). A fresh seed with the same calibration, run through both +stages by `run_two_stage.srun`, says whether that was noise and whether the +two-stage fit's X coverage holds on new data. The stage-1 Laplace is part +of the job, so step 2 can run on it too. + +### How to run the second seed + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/full-size-sim/make_sim_config.py --out_dir sim_realistic_seed2 --seed 2 --calibrate_from pilot_s1 +``` + +Check that `sim_realistic_seed2/simulate_config.yaml` reads `seed: 2`. + +```bash +cd sim_realistic_seed2 && sbatch --export=ALL,SEED=2 /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_two_stage.srun +``` + +The simulation alone takes about 85 minutes; the whole job ends with +`>>> Done` in `run_two_stage.out` and `stage2/summary/`. To also get this +seed's unheld full-library fit (the bias at 1.45 should reproduce), submit +`run_full_size.srun` in the same directory after it (it reuses +`tfs_growth.csv`): + +```bash +sbatch --export=ALL,SEED=2 --dependency=afterok: ../run_full_size.srun +``` + +## Commit + +The commit that adds this study. + +## Results + +Round 1 above (inconclusive). + +### Round 2 results (2026-10-10) + +All three arms converged at every stage (about 15 minutes for +`no_doubles`, 1.7 and 2.3 hours for the others on one GPU). m_fit / m_true +on the X scale, `sim_realistic_s2`: + +| genotypes in the fit | doubles | kanR+kan | pheS+4CP | +|---|---|---|---| +| `no_doubles` | 0 | 1.03 | 0.89 | +| `deep_doubles` | 80,915, all >= 1e3 reads | 1.23 | 1.16 | +| `shallow_doubles` | 80,915, all < 1e3 reads | 1.20 | 1.10 | +| all (the full-size fit) | 207,599 | 1.47 | 1.43 | + +**The doubles cause the bias, in proportion to how many there are, whatever +their depth.** With only wt, spikes and singles the fit recovers m in kanR +(pheS 11% shallow, within about twice round 1's seed spread). Adding 81,000 +doubles moves it to about 1.2 whether they are all deep or all shallow, and +all 208,000 move it to 1.47. k follows m along the ridge (kanR k 0.018, +0.022, 0.022 and 0.027 against a true 0.0149). The tube offsets look alike +in every arm (SD 0.31-0.35). + +The genotypes' data are independent given the shared parameters, so each +double adds a small, consistent pull on m, and the pulls add. A pull that +does not depend on depth is not plain count noise. Two candidates: + +- **The joint MAP over many per-genotype latents.** The MAP optimizes each + genotype's X, K, n and dk_geno jointly with m, along the m·X ridge. The + hierarchical priors on those latents add one term per genotype. If a + steeper m with compressed X fits the population priors slightly better + per genotype, the preference grows with the number of genotypes, while the + information on the true m (wt, spikes, the tube totals) stays fixed. This + is the classic Neyman-Scott bias of joint estimation. +- **A per-genotype misspecification of the doubles' model**, such as their + shared bulk transformation (congression; the fit uses `single`), whose + per-genotype effect would add the same way. Full-size Poisson counts + still give 1.32, so count noise accounts for at most part of it. + +**Practical route, whatever the mechanism:** estimate k and m (and the tube +offsets) from wt, spikes and singles alone, then hold them for the full +library. The first step is the same subset of the real data: does the real +kanR m move from -0.020 to the monoculture's -0.014 when the doubles are +dropped? + +### How to run the real-data `no_doubles` arm + +On the cluster, after pulling, next to the recipe run: + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_subsets.py recipe --out_dir m_subsets_real --arms no_doubles --library_config ../processed/library_config.yaml +``` + +Check that it prints about 950 wt/spike/single genotypes, then: + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/{run_arm.sh,run_arm.srun} m_subsets_real/ +``` + +```bash +cd m_subsets_real/no_doubles && sbatch ../run_arm.srun +``` + +It ends with `>>> Done` in `run.out` (about 15 minutes) and writes +`tfs_params_growth_k.csv` and `tfs_params_growth_m.csv`. There is no truth, +so compare by hand against the recipe and the monoculture (kanR+kan k +0.0150, m -0.0141; pheS+4CP k 0.0031, m 0.0144). + +### Real-data `no_doubles` result (2026-10-10) + +The recipe's growth table without the doubles (15 minutes on one GPU). +Stage 1 and the joint stage reached the 100,000-epoch cap still descending +about 1 nat per window in the joint stage, led by an ln_cfu0 hyper scale +(the hierarchical MAP's funnel, which has no finite optimum). The tube +offsets are centered and small (SD 0.23). + +| condition | fit | k | m | +|---|---|---|---| +| kanR+kan | wt monoculture | 0.0150 | -0.0141 | +| | recipe, all genotypes | 0.0298 | -0.0204 | +| | recipe, monoculture priors (`anchor_real`) | 0.0249 | -0.0202 | +| | **no doubles** | **0.0160** | **-0.0141** | +| pheS+4CP | wt monoculture | 0.0031 | 0.0144 | +| | recipe, all genotypes | 0.0226 | 0.0097 | +| | recipe, monoculture priors (`anchor_real`) | 0.0172 | 0.0097 | +| | **no doubles** | **0.0117** | **0.0061** | + +**In kanR+kan the real data behave as the simulation does.** Without the +doubles, k and m land on the wt monoculture with no prior pulling them +there (loose priors, SD 0.01), so the real kanR curves fitted with all +genotypes are compressed by about 1.45, as in the simulation. + +**pheS+4CP does not.** Dropping the doubles lowers m, as in the simulation, +but from a starting point already below the monoculture, so it moves away +from it (0.0061 against 0.0144) and k stays high. Something in the real +pheS data that the simulation lacks sets pheS's scale: the selection-onset +transient (`planning/offset-mode-growth-transition.md`) or the known +mismatch between the 4CP wt in the screen and in monoculture. That is +`planning/deep-coverage.md` step 3. + +### Round 3 result (2026-10-10) + +`held_s2`: the full `sim_realistic_s2` library with k and m clamped at the +simulated `no_doubles` MAP (kanR+kan k 0.0182, m -0.0142; pheS+4CP k +0.0057, m 0.0126). Every stage converged; about 2 hours on one GPU. X of +the doubles against the truth, by total reads, unheld (the original +full-size fit) against held: + +| reads | doubles | r | slope (truth on fit) | RMSE | 95% coverage | median 95% width | +|---|---|---|---|---|---|---| +| <= 1e2 | 37,707 | 0.26 / 0.28 | 0.05 / 0.05 | 2.16 / 2.25 | 0.97 / 0.98 | 4.08 / 5.74 | +| 1e2-1e3 | 89,022 | 0.62 / 0.64 | 0.49 / 0.38 | 0.45 / 0.57 | 0.82 / 0.90 | 0.85 / 1.29 | +| 1e3-1e4 | 64,362 | 0.93 / 0.93 | 1.19 / 0.82 | 0.20 / 0.19 | 0.56 / 0.89 | 0.35 / 0.53 | +| 1e4-1e5 | 15,283 | 0.99 / 0.99 | 1.36 / 0.95 | 0.17 / 0.07 | 0.29 / 0.90 | 0.14 / 0.22 | +| > 1e5 | 1,225 | 0.99 / 1.00 | 1.37 / 0.96 | 0.16 / 0.05 | 0.10 / 0.65 | 0.05 / 0.08 | + +**Holding k and m removes the compression.** Above 1e4 reads the slope +goes from 1.36 to 0.95, RMSE falls about threefold and 95% coverage rises +from 0.29 to 0.90 (1e4-1e5) and from 0.10 to 0.65 (above 1e5). The +k/dk_geno slide is gone too: the median dk_geno error falls from -0.0057 +to -0.0008, and the held directions shrink from the k/dk_geno slide +(eigenvalue -1.4e10) to a small dk_geno hyperparameter direction +(-2.9e3). Tube offsets are unchanged (SD 0.32). + +What is left: + +- **The deepest genotypes still undercover (0.65 above 1e5 reads)**, + because the intervals leave out k and m's own error. The no-doubles m is + 3% steep in kanR and 11% shallow in pheS, and at widths of 0.08 that is + as large as the interval. +- **Below 1e3 reads the held fit is a little worse** (RMSE 0.57 against + 0.45 at 1e2-1e3, slope 0.38): the shallow points scatter more and their + intervals widen to cover it (0.90). That is + `planning/deep-coverage.md` step 4 (hold the X population SDs). +- At 1e3-1e4 a slope below 1 (0.82) is the expected regression dilution + of a noisy point, not a scale bias. + +### Round 4 result (2026-10-10) + +Both arms converged at every stage (4.9 and 5.7 hours). The configure +lines read `pins: none`. + +| condition | truth (sim) | unheld | all pinned (round 3) | kanR held (round 4) | +|---|---|---|---|---| +| kanR+kan m | -0.0138 | -0.0203 | -0.0142 (held) | -0.0149 | +| pheS+4CP m (free in round 4) | 0.0141 | 0.0202 | 0.0126 (held) | 0.0147 | + +| doubles' X (sim) | unheld | all pinned | kanR held | +|---|---|---|---| +| 1e4-1e5 reads: slope / RMSE / 95% coverage | 1.36 / 0.17 / 0.29 | 0.95 / 0.07 / 0.90 | 1.00 / 0.08 / 0.82 | +| > 1e5 reads: slope / RMSE / 95% coverage | 1.37 / 0.16 / 0.10 | 0.96 / 0.05 / 0.65 | 1.01 / 0.07 / 0.42 | + +- **Holding kanR alone is enough.** With X fixed through kanR, the free + pheS m settles at 1.04 of the truth, closer than when it was held at its + own no-doubles value (0.89). +- **The tight prior leaked.** kanR m moved from the held -0.0142 to + -0.0149, seven prior SDs and about 11% of the doubles' pull, not the + 0.5% estimated from the k/m anchor study. That costs deep coverage + against the pinned run, and k and m came back as held Laplace directions + (eigenvalues near -1e10). +- **Real data.** kanR m leaked shallow (-0.0133 against the held -0.0141, + k 0.0162), so the real kanR X comes out about 1.5 times wider than in the + recipe fit (0.0204 / 0.0133). The free pheS m settled at 0.0064 (k + 0.0099), as in the no-doubles fit: with X set by kanR, the real pool's + pheS selection is about half the wt monoculture's, a property of the + data (`planning/deep-coverage.md` step 3). + +So round 5 pins kanR per condition instead. + +### Round 5 result (2026-10-11) + +Both arms converged at every stage (about 3.5 and 7 hours). The pins held +exactly: kanR's k and m equal the no-doubles values in both arms. + +| sim, doubles above 1e4 reads | all pinned (round 3) | kanR prior (round 4) | kanR pinned (round 5) | +|---|---|---|---| +| 1e4-1e5: slope / RMSE / 95% coverage | 0.95 / 0.07 / 0.90 | 1.00 / 0.08 / 0.82 | 0.99 / 0.10 / 0.67 | +| > 1e5: slope / RMSE / 95% coverage | 0.96 / 0.05 / 0.65 | 1.01 / 0.07 / 0.42 | 0.99 / 0.09 / 0.20 | +| mean X error at 0 / 1 mM IPTG | +0.02 / -0.05 | +0.06 / 0.00 | +0.09 / +0.04 | +| pheS+4CP k (truth 0.0033) | 0.0057 (held) | 0.0005 | -0.0009 | +| pheS+4CP m (truth 0.0141) | 0.0126 (held) | 0.0147 | 0.0140 | + +**Freeing pheS's k reopens the k/dk_geno slide.** The free pheS m is +right (1.00 of the truth), and X's scale is right (slope 0.99), but every +deep double's X shifts up by a near-constant amount. A uniform shift in X +is compensated by dk_geno and pheS's k together. kanR's pinned k no longer +closes the slide on its own, because dk_geno is shared and pheS's k can +absorb the difference. Only wt's gauge resists, weakly. The Laplace +holds that direction (pheS+4CP k, eigenvalue -1.4e10) at zero width, so +the intervals do not widen to cover the shift. Round 4 left pheS's k +free too; its MAP stopped at a smaller shift. Round 3, with every +condition pinned, has no slide. + +**Real arm:** kanR k 0.0160, m -0.0141 (pinned); pheS+4CP k 0.0093, m +0.0068, against its own no-doubles fit's 0.0117 and 0.0061. pheS's k slid +the same way, and its held direction is again pheS+4CP k (eigenvalue +-4.7e9). + +**What the data support for pheS.** In the real data, pheS's m with X +set by kanR (0.0064 in round 4, 0.0068 here) matches pheS's own +no-doubles fit (0.0061). So the half-strength pheS selection is the pool's +property, the same with or without the doubles, not an artifact of the +no-doubles fit. Pinning pheS at its no-doubles values (round 3's setup) is +what the data support, and it is the setup that closes the slide on the +simulation. + +### Round 6 result (2026-10-11) + +`held_all_real` converged at every stage. k and m sit at the real +no-doubles values in all four conditions; the only held Laplace direction +left is a small one on the dk_geno population spread (eigenvalue -3.6e3; +no k, m or slide direction). Tube offsets SD 0.21. + +Against the recipe fit, by total reads (the recipe's `tfs_pred_theta.csv` +and per-genotype reads from its growth table): + +| genotypes | reads | n | r with recipe | slope (pinned on recipe) | median 95% width, recipe / pinned | +|---|---|---|---|---|---| +| doubles | <= 1e2 | 43,713 | 0.977 | 1.48 | 1.61 / 2.44 | +| doubles | 1e2-1e3 | 95,717 | 0.990 | 1.47 | 0.63 / 0.94 | +| doubles | 1e3-1e4 | 60,542 | 0.997 | 1.47 | 0.31 / 0.46 | +| doubles | 1e4-1e5 | 15,634 | 0.999 | 1.48 | 0.18 / 0.27 | +| doubles | > 1e5 | 1,387 | 0.999 | 1.48 | 0.13 / 0.21 | +| singles | > 1e5 | 904 | 1.000 | 1.47 | 0.13 / 0.19 | + +**The real X stretches by a uniform 1.47**, at every depth, singles and +doubles alike: between kanR's m ratio (0.0204 / 0.0141 = 1.45) and pheS's +(0.0097 / 0.0061 = 1.59), as a single X shared across markers should be, +and the factor the simulation predicted for the recipe's compression +(about 1.45). Ranking is unchanged (r 0.99-1.00 above 100 reads) and the +widths stretch with X. Deep doubles' median X runs from 0.53 at no IPTG to +0.13 at 1 mM, against 0.42 to 0.14 in the recipe. + +The intervals still leave out k and m's own uncertainty +(`planning/deep-coverage.md` step 2), and the pheS-derived part of X waits +on step 3. + +### Step 2: the stage-1 Laplace (2026-10-11) + +`run_km_draws.srun` on `m_subsets/no_doubles` (seed 1). The arrowhead +Laplace held no k or m direction, and its k/m spread is far smaller than +stage 1's actual error: + +| condition | stage-1 k (SD) | k truth | stage-1 m (SD) | m truth (X scale) | +|---|---|---|---|---| +| kanR+kan | 0.0182 (0.0008) | 0.0149 | -0.01422 (0.00008) | -0.0138 | +| pheS+4CP | 0.0056 (0.0011) | 0.0033 | 0.01261 (0.00007) | 0.0141 | + +m is off by about 5 SDs in kanR and 21 in pheS; k by 4 and 2. So draws +from this Laplace vary m by about 0.6% and cannot cover a 3-11% error. +Either stage 1 is biased, or its error is noise much larger than its +Laplace (the simulation's per-tube rate noise, which the fit's level offsets do not model, is +a likely source). The second seed tells the two apart. If it is noise, +the k/m uncertainty should come from resampling stage 1 (refits over +tubes or replicates), not from its Laplace. The ten draw fits were +cancelled before they ran; `km_draws/` holds only the draw tables. + +### Second seed: stage 1 (2026-10-11) + +`sim_realistic_seed2/stage1/no_doubles` converged at every stage (about 10 +minutes); the Laplace held no k or m direction (k SD 0.0003-0.0006, m SD +under 0.0001). + +| | seed 1 | seed 2 | +|---|---|---| +| kanR+kan m / truth | 1.028 | 1.017 | +| pheS+4CP m / truth | 0.895 | 0.897 | +| kanR+kan k - truth | +0.0033 | -0.0024 | +| pheS+4CP k - truth | +0.0023 | -0.0007 | + +**Stage 1's m error is a bias, not noise**: the same on both seeds, kanR +about 2% steep and pheS about 10% shallow. Wider intervals cannot fix a +bias, so step 2's draws would not help even with an honest spread. k's +error is mostly noise: it changes sign between seeds, and its seed-to-seed +spread (about 0.003) is several times its Laplace SD. Through the +k/dk_geno slide that noise moves X by a constant. Next: find the pheS bias +with stage-1-only fits (about 10 minutes each), starting with the Poisson +simulation (`sim_poisson_s2`) to test the realistic count noise. + +### Stage 1 under Poisson counts (2026-10-11) + +`m_subsets_poisson/no_doubles` (the `sim_poisson_s2` growth table, seed +1; the joint stage reached the epoch cap still creeping about 30 nats per +window, led by `growth_phi` drifting toward the Poisson limit): kanR+kan m +1.004 of the truth, pheS+4CP 0.874. The realistic count noise is not the +cause, and neither is founder or demographic noise (the Poisson simulation +has none). The OD600-derived tube totals are not either: against the +simulation's true totals they are within 2% in every condition and IPTG +level, with no trend. + +**Suspect: congression's asymmetry between markers.** The homodimer rule +gives a co-transformed cell a θ that leans toward its higher-θ plasmid. In +pheS+4CP (m > 0) higher θ grows faster, so a low-X mutant sharing a cell +with a high-X partner grows like the partner and those cells take over: +low-X genotypes look higher, the response compresses, and m reads +shallow. In kanR+kan (m < 0) the shared cells grow slower and fall +behind, so the clean cells dominate and the bias stays small. The test: +the seed-1 simulation with `transformation_poisson_lambda` 0 (same seed, +so the same phenotypes), stage 1 only (`run_two_stage.srun`, `STAGES=1`). + +### How to run the no-congression stage 1 + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +mkdir sim_realistic_nocong && cp sim_realistic_s2/{simulate_config.yaml,design.csv,od600_calibration.yaml,library_config.yaml} sim_realistic_nocong/ +``` + +```bash +sed -i 's/^transformation_poisson_lambda: .*/transformation_poisson_lambda: 0.0/' sim_realistic_nocong/simulate_config.yaml && grep -n "lambda\|^seed" sim_realistic_nocong/simulate_config.yaml +``` + +Check: `transformation_poisson_lambda: 0.0` and `seed: 1`. + +```bash +cd sim_realistic_nocong && sbatch --export=ALL,SEED=1,STAGES=1 /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_two_stage.srun +``` + +About 85 minutes to simulate, then about 10 for stage 1. Read +`stage1/no_doubles/tfs_params_growth_m.csv` against the truth with +`score_arms.py`. + +### No-congression stage 1 (2026-10-11) + +`sim_realistic_nocong` (seed 1, `transformation_poisson_lambda` 0, else +as `sim_realistic_s2`); every stage converged; no k or m direction held. + +| stage-1 m / truth | congression on | congression off | +|---|---|---| +| kanR+kan | 1.028 | 1.032 | +| pheS+4CP | 0.895 | 1.035 | + +**Congression causes stage 1's pheS bias.** kanR does not move, as the +asymmetry predicts. Both markers keep a small steep bias (about 3%) +without congression. + +Fitting congression (`transformation: mixture`) does not fit the +two-stage idea: a mixture needs each co-resident partner's θ, so the +partners, mostly doubles, have to be in the fit, which brings back their +pull on m; and the relative-X fit accepts only the `max` rule, not the +simulation's `homodimer`. A stage 1 of clones with little congression +avoids both: wt and the spikes. In the simulated pool (with the 271 +repeated wt sequences), about 17% of a bulk genotype's cells are +congressed (`transformation_poisson_lambda` 0.357); wt is 77% spiked, so +about 4% of its cells are congressed, single spikes about 10%, double +spikes almost none. wt alone sets the X gauge and, through the tube +totals, its growth difference between 0 and 1 mM is m. `make_subsets.py +--arms spikes` keeps wt and the library config's spiked genotypes (about +six with growth data). + +### How to run the spikes-only stage 1 + +Simulation, seed 1 (congression on), in the full-size working directory: + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_subsets.py sim_realistic_s2 --arms spikes --out_dir m_subsets +``` + +```bash +cd m_subsets/spikes && sbatch ../run_arm.srun && cd ../.. +``` + +Seed 2 (`run_arm.srun` and `run_arm.sh` are not in its `stage1/` yet): + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_subsets.py sim_realistic_seed2 --arms spikes --out_dir sim_realistic_seed2/stage1 +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/{run_arm.sh,run_arm.srun} sim_realistic_seed2/stage1/ && cd sim_realistic_seed2/stage1/spikes && sbatch ../run_arm.srun +``` + +Real data, next to the recipe: + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_subsets.py recipe --arms spikes --out_dir m_subsets_real --library_config ../processed/library_config.yaml +``` + +```bash +cd m_subsets_real/spikes && sbatch ../run_arm.srun +``` + +Each `make_subsets.py` reads its 8 GB growth table (about 6 minutes) and +prints `spikes: N genotypes`; each fit takes about 10 minutes. On the +real data, whether pheS m rises toward the monoculture's 0.0144 says how +much of the real pheS gap is congression (the simulation's congression +alone takes about 14% off pheS m; the real gap is about 2x). + +### Second seed: stage 2 (2026-10-11) + +`sim_realistic_seed2/stage2` (every condition pinned at seed 2's stage 1) +converged at every stage (about 1.7 hours). The doubles' X by total reads +(`score_x.py`, reads from `depth.py` on seed 2's growth table): + +| doubles | seed 1 (round 3) | seed 2 | +|---|---|---| +| 1e3-1e4: slope / RMSE / 95% coverage | 0.82 / 0.19 / 0.89 | 0.83 / 0.19 / 0.88 | +| 1e4-1e5: slope / RMSE / 95% coverage | 0.95 / 0.07 / 0.90 | 0.97 / 0.07 / 0.87 | +| > 1e5: slope / RMSE / 95% coverage | 0.96 / 0.05 / 0.65 | 0.98 / 0.06 / 0.51 | +| deep X error at 0 / 1 mM | +0.02 / -0.05 | +0.05 / +0.01 | + +The two-stage fit's gain reproduces on a new seed: slope near 1 and RMSE +0.06-0.07 above 1e4 reads, against 1.36 and 0.16 for the one-stage fit. +What is left is an X offset of up to about 0.05 that changes between +seeds (stage 1's k noise, through the k/dk_geno slide) on top of pheS's +congression bias; the deepest genotypes' intervals (widths about 0.08) +do not cover it (0.65, 0.51). + +### Spikes-only stage 1 (2026-10-11) + +wt plus the spiked genotypes (`make_subsets.py --arms spikes`), about 10 +minutes each. Simulation (m / truth): + +| | seed 1 | seed 2 | +|---|---|---| +| all singles, kanR+kan | 1.028 | 1.017 | +| all singles, pheS+4CP | 0.895 | 0.897 | +| spikes only, kanR+kan | 1.040 | 0.969 | +| spikes only, pheS+4CP | 1.037 | 0.988 | + +**In the simulation, spikes only removes the congression bias** (pheS as +with congression off), at the cost of noise: about ±4% between seeds +against about ±1% with all singles. Seed 1's joint stage reached the +epoch cap still descending about 8 nats per window (a count-noise +parameter moving), so its numbers are provisional. + +**On the real data it fails.** All nine spikes are deep (wt 3.9e8 reads, +each spike 1.3-6.2e6), but the fit settled in another solution: kanR+kan +m -0.0051, the control kanR-kan m -0.0123 with that condition's tube +offsets spread to SD 0.64 (the others 0.14-0.23), and pheS+4CP m about 0 +(stage 1 at the epoch cap, nearly flat). With nine genotypes the model +can trade slope between the selection and control conditions, and the +loose priors give the controls' m the same SD (0.01) as selection's. Next: +the same fit with the control slopes held near 0 (SD 0.001, `linear`'s +default for control conditions). + +### How to run the real spikes-only fit with tight controls + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit/m_subsets_real +``` + +```bash +mkdir spikes_tightctl && cd spikes_tightctl && ln -s ../spikes/tfs_growth.csv . && cp ../spikes/library_config.yaml . +``` + +```bash +printf "condition_rep,k_loc,k_scale,m_loc,m_scale\nkanR+kan,0.015,0.01,0.0,0.01\nkanR-kan,0.015,0.01,0.0,0.001\npheS+4CP,0.015,0.01,0.0,0.01\npheS-4CP,0.015,0.01,0.0,0.001\n" > growth_priors_loose.csv && cat growth_priors_loose.csv +``` + +Check that the two control rows (`-kan`, `-4CP`) have m_scale 0.001. +`run_arm.sh` reads the file by that name. + +```bash +sbatch ../run_arm.srun +``` + +### Real spikes-only with tight controls, and wt's own trajectories (2026-10-11) + +`m_subsets_real/spikes_tightctl` (control m SD 0.001): the same solution +as before (kanR+kan m -0.0042, kanR-kan m -0.0093, nine prior SDs from 0, +pheS+4CP m about -0.001; kanR-kan tube offsets SD 0.54). With nine +genotypes there is no population of X values to pin X's scale, so the fit +slides along the m·X ridge to a small m. The spikes-only stage 1 is +dropped for the real data. + +Reading wt's growth straight off the growth tables (slope of ln_cfu over +time per condition and IPTG level) is too noisy to use: each replicate +has 2-3 sequenced tubes over about 40 minutes per level, and tube-level +noise of about 0.3 ln units gives a slope error near 0.008 per minute, the +size of m. On the simulation it gives kanR m of +0.039 and -0.026 in the +two replicates against a truth of -0.014, and pooling replicates +confounds their starting levels with time. The real pool's wt happened +to come out near the monoculture (kanR m -0.016, pheS +0.013, pooled over +replicates), but that is not evidence. The real pheS gap stays open. + +Options for congression in stage 1: correct the all-singles stage 1's +pheS m by the simulation's factor (about 1.15; assumes the simulation's +lambda and homodimer rule); a middle-sized stage 1 (the spikes plus a few +hundred of the deepest singles); or fit congression (`mixture`, which +needs the partners and the `max` rule). + +### Option 2 and a test of whether congression is the doubles' pull + +Option 2: `make_subsets.py --arms spikes_deep` keeps wt, the spikes and +the 300 deepest singles (`--n_deep_singles`), on both simulation seeds. + +Option 3 is feasible: the relative fit configures and runs with +`--transformation_model mixture --congression_theta_rule max` (40 steps +on a round 1 table, about 1.8 s per step on a laptop). The lab's estimate +of lambda is 0.357 with a likelihood-ratio 95% interval of 0.156-0.674 +(log-SD about 0.37; as the (mean, std) that `--transformation_lambda` +takes, about 0.38 0.15). The simulation's 0.357 sits mid-interval, but the +real lambda could be nearly twice that, and congression's pheS bias grows +with lambda. + +Before building option 3 into stage 1: round 1's reduced library hinted +that congression also drives the doubles' pull (no_cong m 0.96-0.99 +against base 1.03-1.14). If it does, one full-library fit with `mixture` +could fix both and the two stages would not be needed. The test is the +one-stage full-library fit on `sim_realistic_nocong`: m near 1.0 means +the pull is congression; m near 1.45 means it is separate. + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3/sim_realistic_nocong && sbatch --export=ALL,SEED=1 ../run_full_size.srun +``` + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_subsets.py sim_realistic_s2 --arms spikes_deep --out_dir m_subsets && (cd m_subsets/spikes_deep && sbatch ../run_arm.srun) +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/make_subsets.py sim_realistic_seed2 --arms spikes_deep --out_dir sim_realistic_seed2/stage1 && (cd sim_realistic_seed2/stage1/spikes_deep && sbatch ../run_arm.srun) +``` + +### Option 2 result (2026-10-11) + +| stage-1 m / truth | all singles | spikes only | spikes + 300 deepest singles | +|---|---|---|---| +| seed 1, kanR+kan | 1.028 | 1.040 | 1.014 | +| seed 1, pheS+4CP | 0.895 | 1.037 | 0.889 | +| seed 2, kanR+kan | 1.017 | 0.969 | 1.024 | +| seed 2, pheS+4CP | 0.897 | 0.988 | 0.908 | + +The deepest singles are as congressed as the rest; once they are in, the +pheS bias is back at the all-singles level (seed 1's joint stage at the +epoch cap, seed 2 converged). A stage 1 large enough to pin X's scale +carries congression, so congression has to be modeled (option 3). The +one-stage no-congression fit decides whether that goes into one full +fit or into stage 1. + +### One-stage fit without congression (2026-10-11) + +`sim_realistic_nocong`, the full library through `run_full_size.srun`; +every stage converged. + +| full-library m / truth | congression on | congression off | +|---|---|---| +| kanR+kan | 1.47 | 1.37 | +| pheS+4CP | 1.43 | 1.51 | + +**The doubles' pull is not congression.** Two separate problems: the +doubles steepen m about 1.45-fold (fixed by the two stages), and +congression flattens pheS's m about 0.9-fold in any stage 1 large enough to +pin X's scale. So congression is modeled inside the two-stage fit, in +stage 1. + +## Option 3: stage 1 with the congression mixture + +Stage 1 without doubles, with `--transformation_model mixture +--transformation_lambda 0.357 0.15 --congression_theta_rule max` +(`run_arm.sh` reads extra configure flags from the arm's +`configure_extra.txt`). Two approximations: the fit draws each genotype's +co-resident partners from the genotypes with growth data, so in stage 1 +from wt, the spikes and the singles, while the real partners are mostly +doubles (in the simulation singles and doubles draw their phenotypes from +the same distribution); and the fit's `max` rule stands in for the +simulation's `homodimer`. The arm is configured with the simulation's own +composition (`simulate_config.yaml` as the library config: wt's 271 +spiked copies, so wt is mostly clean, as in the simulated pool), so the test +is of modeling congression, not of knowing wt's purity. On real data, wt's +purity is unknown (the excess wt is being worked out at the bench). + +### How to run option 3 (both seeds, reusing the no-doubles tables) + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3/m_subsets +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_arm.sh . && mkdir no_doubles_mix && cd no_doubles_mix && ln -s ../no_doubles/tfs_growth.csv ../no_doubles/tfs_sim_* . && cp ../no_doubles/{simulate_config.yaml,growth_priors_loose.csv} . && cp simulate_config.yaml library_config.yaml && echo "--transformation_model mixture --transformation_lambda 0.357 0.15 --congression_theta_rule max" > configure_extra.txt && sbatch ../run_arm.srun +``` + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3/sim_realistic_seed2/stage1 +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_arm.sh . && mkdir no_doubles_mix && cd no_doubles_mix && ln -s ../no_doubles/tfs_growth.csv ../no_doubles/tfs_sim_* . && cp ../no_doubles/{simulate_config.yaml,growth_priors_loose.csv} . && cp simulate_config.yaml library_config.yaml && echo "--transformation_model mixture --transformation_lambda 0.357 0.15 --congression_theta_rule max" > configure_extra.txt && sbatch ../run_arm.srun +``` + +The `cp run_arm.sh` replaces the copy already in each folder with the one +that reads `configure_extra.txt`. In each arm's `configure.log` the +command line shows the mixture flags. + +### Option 3 result: stage 1 (2026-10-11) + +| stage-1 m / truth | seed 1 | seed 2 | +|---|---|---| +| without mixture, kanR+kan | 1.028 | 1.017 | +| without mixture, pheS+4CP | 0.895 | 0.897 | +| with mixture, kanR+kan | 1.056 | 1.047 | +| with mixture, pheS+4CP | 1.037 | 1.023 | + +**Modeling congression removes stage 1's pheS bias** on both seeds. Both +markers keep a small steep bias (2-6%). Seed 2 converged at every stage; +seed 1's first stage reached its cap, the later ones converged. The +fitted lambda is 0.27 and 0.25 against the simulation's 0.357, inside the +prior: the `max` rule lets the higher-θ plasmid set a shared cell's θ +completely, where `homodimer` only leans toward it, so fewer congressed +cells reproduce the same effect. On real data the fitted lambda is a +model parameter, not the measured one; keep it fitted, with the +measurement as its prior. + +## Stage 2 after a mixture stage 1 + +k and m pinned at the mixture stage 1, on both seeds, with stage 2 fitted +without the mixture (`single`, as rounds 3 and 6) and with it (every +genotype's partners are in the full library; a few cell classes per +genotype, so heavier). `run_held.srun` reads `configure_extra.txt` like +`run_arm.sh`. All four use the simulation's composition as the library +config, as the stage 1 did. + +### How to run + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +Seed 1 (`sim_realistic_s2`), stage 2 without and with the mixture: + +```bash +for v in single mix; do mkdir stage2_${v}_seed1 && (cd stage2_${v}_seed1 && python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets/no_doubles_mix --out growth_priors_held.csv && ln -s ../sim_realistic_s2/tfs_growth.csv ../sim_realistic_s2/tfs_sim_* . && cp ../sim_realistic_s2/simulate_config.yaml library_config.yaml && if [ $v = mix ]; then echo "--transformation_model mixture --transformation_lambda 0.357 0.15 --congression_theta_rule max" > configure_extra.txt; fi && sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun); done +``` + +Seed 2: + +```bash +for v in single mix; do mkdir stage2_${v}_seed2 && (cd stage2_${v}_seed2 && python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../sim_realistic_seed2/stage1/no_doubles_mix --out growth_priors_held.csv && ln -s ../sim_realistic_seed2/tfs_growth.csv ../sim_realistic_seed2/tfs_sim_* . && cp ../sim_realistic_seed2/simulate_config.yaml library_config.yaml && if [ $v = mix ]; then echo "--transformation_model mixture --transformation_lambda 0.357 0.15 --congression_theta_rule max" > configure_extra.txt; fi && sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun); done +``` + +Each `held_priors.py` prints the pinned table (seed 1 kanR+kan m about +-0.0146). Each `run.out` names the extra flags on its configure line +(`extra: none` for `single`). Score with `score_x.py` against +`held_s2` (seed 1) and `sim_realistic_seed2/stage2` (seed 2). + +### Stage 2 after a mixture stage 1: first results (2026-10-12) + +`stage2_single_seed1` (k and m pinned at the mixture stage 1, stage 2 +`single`) converged; against round 3 (stage 1 without the mixture): + +| doubles, seed 1 | round 3 | mixture stage 1, `single` stage 2 | +|---|---|---| +| 1e4-1e5: slope / RMSE / 95% coverage | 0.95 / 0.07 / 0.90 | 1.02 / 0.07 / 0.90 | +| > 1e5: slope / RMSE / 95% coverage | 0.96 / 0.05 / 0.65 | 1.03 / 0.05 / 0.76 | +| deep X error at 0 / 1 mM | +0.02 / -0.05 | +0.005 / -0.02 | + +The better stage 1 roughly halves the X offset and raises the deepest +genotypes' coverage from 0.65 to 0.76. + +`stage2_mix_seed1`'s MAP converged (162,000 steps, no memory trouble), but +the arrowhead Laplace refuses it: "genotype blocks are coupled (response +on unprobed genotypes 1.10e-02 of the probed ones)". The mixture couples +each genotype to its co-resident partners' X, which the block Laplace +assumes away; the coupling is about 1% of a genotype's own response. Seed +2's mixture run's MAP converged too (102,000 steps); its Laplace ran out +of GPU memory (a 26.6 GB allocation) before reaching the coupling check. `run_point.srun` finishes such a +fit at the MAP point (`--laplace point`, predict, summarize), enough to +compare X points with `single`. If the mixture stage 2 is worth keeping, +its intervals need a Laplace that tolerates (or drops) couplings this small. + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3/stage2_mix_seed1 && sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_point.srun +``` + +Once `stage2_mix_seed2/run.err` shows the same error, the same in +`stage2_mix_seed2`. + +`stage2_single_seed2` converged; against seed 2's first two-stage fit +(stage 1 without the mixture): + +| doubles, seed 2 | stage 1 without mixture | mixture stage 1, `single` stage 2 | +|---|---|---| +| 1e4-1e5: slope / RMSE / 95% coverage | 0.97 / 0.07 / 0.87 | 1.04 / 0.07 / 0.88 | +| > 1e5: slope / RMSE / 95% coverage | 0.98 / 0.06 / 0.51 | 1.05 / 0.06 / 0.67 | +| deep X error at 0 / 1 mM | +0.05 / +0.01 | +0.03 / +0.03 | + +On both seeds the mixture stage 1 raises the deepest genotypes' coverage +(0.65 to 0.76, 0.51 to 0.67). The slope now overshoots slightly +(1.03-1.05): the small steep bias the mixture stage 1 keeps in m (kanR +about 5%, pheS 2-4%) is now the main error in X's scale, plus a constant +offset near 0.03. + +### Stage 2 with the mixture, at the MAP point (2026-10-12) + +`run_point.srun` on both mixture stage 2 fits. Seed 1 finished; seed 2 +ran out of GPU memory at a 4.1 GB allocation even for the one-draw +posterior (seed 1's identical job succeeded, probably on a larger card). +Seed 1, X of the doubles: + +| doubles, seed 1 | `single` stage 2 | mixture stage 2 (MAP point) | +|---|---|---| +| 1e4-1e5: slope / RMSE / error at 0, 1 mM | 1.02 / 0.067 / +0.005, -0.020 | 1.02 / 0.070 / -0.002, -0.010 | +| > 1e5: slope / RMSE / error at 0, 1 mM | 1.03 / 0.050 / +0.013, -0.025 | 1.03 / 0.044 / +0.006, -0.013 | +| 1e2-1e3: r / RMSE | 0.65 / 0.51 | 0.50 / 0.86 | + +For the deep genotypes the mixture stage 2 trims the offsets by about +0.01 and leaves slope and RMSE unchanged. At low depth it looks worse, but +the `single` column is a Laplace median and the mixture's a bare MAP +point, which scatters more there. Not worth a coupled Laplace. + +**The recipe: a mixture stage 1, then a `single` stage 2.** Stage 1 needs +congression modeled to get pheS's m right; with k and m pinned, stage 2 +gets X right without it. + +## The real data: mixture stage 1 under two assumptions about wt + +The mixture needs the pool's composition, and wt's purity (how much of wt +is clean spike rather than bulk) is not known for these data. Two +variants of the real no-doubles stage 1 with the mixture (max rule, +lambda prior 0.38 +- 0.15, the lab's estimate 0.357 with 95% CI +0.156-0.674 in log space): + +| variant | library config | wt in the fit's pool | +|---|---|---| +| `cfg` | the real config as written (design mixture 100 / 100 / 1000 / 1) | 1.2% of the pool, 99% bulk (about 17% of its cells congressed) | +| `wtspiked` | the simulation's config (the realized mixture from the initial composition, plus wt repeated in the spiked origin) | 11% of the pool, 23% bulk (about 4% congressed) | + +`wtspiked` is the composition the simulation was built to match (the +realized starting pool has wt at about 11%); its config has the same +genetics keys as the real one, so it serves as the library config. Both +stay on the cluster (lab-derived). Readout: kanR+kan m against the +monoculture's -0.0141 (the all-singles stage 1 without the mixture sat on +it; in the simulation the mixture moved kanR about 3% steeper), and pheS+4CP +m against the monoculture's 0.0144. kanR may not separate the variants +strongly, since its congression bias is small. + +### How to run the two real variants + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit/m_subsets_real +``` + +```bash +cp /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_arm.sh . && for v in cfg wtspiked; do mkdir no_doubles_mix_${v} && (cd no_doubles_mix_${v} && ln -s ../no_doubles/tfs_growth.csv . && cp ../no_doubles/growth_priors_loose.csv . && if [ $v = cfg ]; then cp ../no_doubles/library_config.yaml .; else cp /gpfs/projects/harmslab/harms/studies/full-sized-sims-3/sim_realistic_s2/simulate_config.yaml library_config.yaml; fi && echo "--transformation_model mixture --transformation_lambda 0.38 0.15 --congression_theta_rule max" > configure_extra.txt && sbatch ../run_arm.srun); done +``` + +Check: `grep -c "" no_doubles_mix_wtspiked/library_config.yaml` is about +350 lines (the full simulate config) and `no_doubles_mix_cfg`'s is the +short real config; each `configure.log` shows the mixture flags. + +### The two real variants (2026-10-12) + +Both joint stages ended at the epoch cap creeping about 1 nat per window +(nuisance parameters moving, not k or m). + +| real stage 1 | no mixture | mixture, `cfg` | mixture, `wtspiked` | wt monoculture | +|---|---|---|---|---| +| kanR+kan k / m | 0.0160 / -0.0141 | 0.0167 / -0.0147 | 0.0160 / -0.0139 | 0.0150 / -0.0141 | +| pheS+4CP k / m | 0.0117 / 0.0061 | 0.0120 / 0.0068 | 0.0122 / 0.0062 | 0.0031 / 0.0144 | +| fitted lambda | | 0.365 | 0.174 | measured 0.357 (0.16-0.67) | +| final exact loss | | 856,117 | 856,142 | | + +- **Congression barely moves the real pheS m** (0.0061 to 0.0068 or + 0.0062; it moved the simulation's about 15%). The real pheS gap with the + monoculture (about 2x) is mostly something else: step 3, and the pheS + curves stay provisional. +- **The data lean toward wt mostly bulk (`cfg`):** about 25 nats better + on the same data (about 2 of them the lambda prior), and its fitted + lambda lands on the measurement while `wtspiked` drives it to the low + end of the interval. Neither is decisive (the `max` rule fit lambda + about 25% low in the simulation), but both point the same way. +- **kanR stays within about 4% of the monoculture** either way (the + variants differ by about 6%). + +Next: the real stage 2 (`single`, every condition pinned at the `cfg` +mixture stage 1), `held_mix_real`, to compare with round 6 +(`held_all_real`): + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +mkdir held_mix_real && cd held_mix_real && python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets_real/no_doubles_mix_cfg --out growth_priors_held.csv && ln -s ../recipe/tfs_growth.csv . && cp ../../processed/library_config.yaml . && sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun +``` + +Check: kanR+kan k 0.016722, m -0.014744 in the printed table. + +### The real stage 2 under the full recipe (2026-10-12) + +`held_mix_real`: every condition pinned at the real mixture stage 1 (wt +mostly bulk), stage 2 `single`. Every stage converged; one small held +direction (dk_geno's population mean, eigenvalue -6.8e3), none on k, m or +the slide; tube offsets SD 0.20. + +| real X, full recipe against | round 6 | the original recipe (one stage) | +|---|---|---| +| r, doubles above 100 reads | 0.999-1.000 | 0.991-0.999 | +| slope | 0.94 | 1.39 | +| deep doubles' median X, 0 to 1 mM | 0.53 to 0.13, now 0.53 to 0.15 | 0.42 to 0.14, now 0.53 to 0.15 | + +Against round 6 the real X compresses about 6%, as the mixture stage 1's +slightly steeper m predicts (kanR -0.0147 against -0.0141, pheS 0.0068 +against 0.0061); ranking and shapes are unchanged and the widths shrink +with X. Against the original recipe the real X stretches about 1.39 (1.47 +in round 6). The kanR part of the scale sits within 4% of the monoculture; +the pheS part waits on step 3. + +## The kanR-only check + +Stage 2 on the kanR tubes alone, with kanR's k and m pinned at the same +mixture stage 1, against the both-marker fit. If the pheS gap's cause +depends on genotype, the pheS data bend the X curves and the two +disagree. The seed-1 simulation shows what agreement looks like when +pheS is fine. `resample_growth.py --library kanR` keeps one library's +tubes; `held_priors.py --only kanR` keeps kanR's pinned conditions. + +Real (about 6 minutes to filter the growth table, then about 5 hours): + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +mkdir kanR_only_real && cd kanR_only_real && python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/resample_growth.py ../recipe/tfs_growth.csv --library kanR --out tfs_growth.csv && cp ../../processed/library_config.yaml . && python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets_real/no_doubles_mix_cfg --only kanR --out growth_priors_held.csv && sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun +``` + +Simulation, seed 1: + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +mkdir kanR_only_seed1 && cd kanR_only_seed1 && python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/resample_growth.py ../sim_realistic_s2/tfs_growth.csv --library kanR --out tfs_growth.csv && ln -s ../sim_realistic_s2/tfs_sim_* . && cp ../sim_realistic_s2/simulate_config.yaml library_config.yaml && python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/held_priors.py ../m_subsets/no_doubles_mix --only kanR --out growth_priors_held.csv && sbatch /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun +``` + +Each `resample_growth.py` prints the kanR tube count; each +`held_priors.py` prints two rows (kanR+kan, kanR-kan). Compare with +`held_mix_real` and `stage2_single_seed1`. + +## Stage 1's k and m uncertainty: delete-d jackknife + +`run_jackknife.srun` (an array of 30, at most 10 at once) drops a random +20% of the tubes from the stage-1 growth table, stratified by condition and +IPTG (`resample_growth.py --drop_frac`, seed = task number), and refits +the mixture stage 1 in `jk_NN/` with the same model (it copies the full +stage 1's `library_config.yaml`, `growth_priors_loose.csv` and +`configure_extra.txt`). Tubes cannot be resampled with replacement: a +duplicated tube lands on the original's tensor cell. `jackknife_summary.py` +scales the subsample spread by (n - d) / d and, on the simulation, +compares the full stage 1's error with the jackknife SE. Seed 1 first; +seed 2 the same way for a second check of the SE against a known error. + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +mkdir jk_seed1 && cd jk_seed1 && sbatch --export=ALL,SRC=$PWD/../m_subsets/no_doubles/tfs_growth.csv,CONF=$PWD/../m_subsets/no_doubles_mix /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_jackknife.srun +``` + +Each task writes `jk_NN.out` ending `>>> Done jk_NN` (about 20-30 minutes +each). Then, in `jk_seed1/`: + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/jackknife_summary.py . --full ../m_subsets/no_doubles_mix --truth ../sim_realistic_s2 +``` + +### The kanR-only check (2026-10-12) + +Both kanR-only stage 2 fits converged at every stage. On the simulation, +where pheS is modeled correctly, dropping pheS loses much of X: at 0 IPTG +wt barely grows in kanR+kan, so genotypes near the top of X are nearly +static there and pheS carries most of that information. Seed 1's doubles +above 1e5 reads: RMSE 0.15 (0.05 with both markers), 95% coverage 0.36 +(0.76), widths 2.7 times wider, X error +0.17 at 0 IPTG. So the check is +read against the simulation's agreement, not against perfect agreement. + +| deep doubles (> 1e4 reads), kanR-only against both markers | seed 1 | real | +|---|---|---| +| r, 1e4-1e5 / > 1e5 | 0.96 / 0.99 | 0.84 / 0.94 | +| slope, 1e4-1e5 / > 1e5 | 1.09 / 1.08 | 1.11 / 1.05 | +| scatter after the best linear map | 0.13 | 0.21 | +| that scatter in combined SDs (RMS) | 0.89 | 1.16 | +| share beyond 2 combined SDs | 2.5% | 8.2% | +| per-genotype scatter, median / 90th / 99th percentile | 0.08 / 0.21 / 0.38 | 0.11 / 0.32 / 0.68 | + +**The median real genotype agrees across markers about as the simulation +does; a tail does not.** About 8% of the deep real doubles disagree +beyond 2 SD, against 2.5% in the simulation. The check cannot say which +marker is off. A candidate inside the model: dk_geno is shared by the +kanR and pheS conditions, while a real mutation may slow growth +differently in kan and in 4CP; that would bend exactly such genotypes and +cannot show in a simulation that shares dk_geno too. + +For the paper: the pheS gap stays a limitation, with a per-genotype +marker-consistency flag (the z-score between the kanR-only and both-marker +fits) marking the genotypes whose curves rest on a cross-marker +inconsistency. If needed later, a marker-specific dk_geno tests whether the +tail is genotype-specific fitness. + +### The seed-1 jackknife (2026-10-12) + +30 subsamples (20 of 118 tubes dropped each, variance factor 4.90); 29 +converged, one at the epoch cap. + +| seed 1, mixture stage 1 | k, kanR+kan | k, pheS+4CP | m, kanR+kan | m, pheS+4CP | +|---|---|---|---|---| +| jackknife SE | 0.0048 | 0.0028 | 0.00009 | 0.00008 | +| Laplace SD | 0.0008 | 0.0011 | 0.00008 | 0.00007 | +| actual error | +0.0037 | +0.0014 | -0.0008 | +0.0005 | +| error in jackknife SEs | 0.8 | 0.5 | 8.8 | 6.5 | + +**The jackknife covers k; m's error is the bias.** k's jackknife SE is +about five times its Laplace SD and covers k's actual errors. m's agrees +with the Laplace (about 0.6%), and its error, 6-9 SEs, is the small steep +bias accepted as a limitation. + +One direction carries 94% of k's jackknife variance: all four conditions' +k move together (one-SD step -0.0049 kanR+kan, -0.0079 kanR-kan, -0.0023 +pheS+4CP, -0.0027 pheS-4CP). A common shift of every k is absorbed by +dk_geno and leaves X alone; only the uneven part moves X. So two stage 2 +refits measure X's sensitivity: k at plus and minus one SD along that +direction (`jackknife_summary.py --pc1_tables`), m at the full fit, +started from `stage2_single_seed1`'s MAP. `k_band.py` takes each +genotype's k-driven SD as |X(plus) - X(minus)| / 2 and adds it in +quadrature to the stage 2 Laplace SD; `score_x.py` then scores the +widened intervals against the truth. + +### How to run the k-band refits (seed 1) + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3/jk_seed1 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/jackknife_summary.py . --full ../m_subsets/no_doubles_mix --truth ../sim_realistic_s2 --pc1_tables pc1 +``` + +It prints `k variance on the first component: 0.941` and writes +`pc1/plus/` and `pc1/minus/`. Then: + +```bash +for v in plus minus; do (cd pc1/$v && ln -s ../../../sim_realistic_s2/tfs_growth.csv ../../../sim_realistic_s2/tfs_sim_* . && cp ../../../sim_realistic_s2/simulate_config.yaml library_config.yaml && sbatch --export=ALL,INIT=$PWD/../../../stage2_single_seed1/tfs_fit_model_params.npz /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/run_held.srun); done +``` + +Each `pc1/*/run.out` ends with `>>> Done` and a `summary/`. Then, locally: +`k_band.py stage2_single_seed1 jk_seed1/pc1/plus jk_seed1/pc1/minus`, and +`score_x.py` on its output with `stage2_single_seed1`'s truth table. + +### Is the marker disagreement a library-specific dk_geno? + +Split into a per-genotype level and a shape part, the kanR-only against +both-marker disagreement of the deep doubles is 92% level on seed 1 and +91% on the real data: without pheS, a genotype's X level is weakly +pinned, so dropping a marker produces level-shaped noise even in the +simulation, which shares dk_geno across libraries. The real tail is +larger (median level offset of the worst 8% 0.45 against 0.28). + +A dk_geno that differs between the kanR and pheS libraries (backbone or +marker-cassette effects, GxE visible in the library's own control) is +identifiable: each library's control condition (m near 0) measures its +own dk directly, the two-stage fit closes both libraries' k/dk slides, +and a bivariate prior with a learned correlation pools shallow genotypes. +A dk that differs only under the stressor (kan or 4CP present) is not: +inside a selection condition it trades with the level of the genotype's +X curve. + +`control_dk_check.py` tests the first: per deep double, the slope over +time of ln(counts relative to wt) in each library's control condition +estimates its dk there (weighted least squares per series, combined by +inverse variance), and delta_dk = dk_kanR - dk_pheS. If the tail's X +offsets come from a library-specific dk, they track delta_dk with a +slope near 1 / (2 |m_kanR|) (about 34). The simulation, with a shared +dk, is the no-signal benchmark; it also shows how well the two controls +recover the same dk at this depth. + +```bash +cd /gpfs/projects/harmslab/harms/studies/dev-data/real_fit +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/control_dk_check.py recipe/tfs_growth.csv held_mix_real/tfs_pred_theta.csv kanR_only_real/tfs_pred_theta.csv --out control_dk_real.csv +``` + +```bash +cd /gpfs/projects/harmslab/harms/studies/full-sized-sims-3 +``` + +```bash +python /gpfs/home/harms/tfscreen/planning/studies/m-bias-factorial/control_dk_check.py sim_realistic_s2/tfs_growth.csv stage2_single_seed1/tfs_pred_theta.csv kanR_only_seed1/tfs_pred_theta.csv --m_kan -0.0146 --out control_dk_seed1.csv +``` + +Each reads its 8 GB growth table twice (about 10 minutes) and prints the +two libraries' dk agreement and the offset-delta_dk correlations. + +#### Result (2026-10-12) + +| | seed 1 (shared dk) | real | +|---|---|---| +| deep doubles with both controls | 16,509 | 16,982 | +| r(offset, delta_dk), all / worst 8% | 0.40 / 0.65 | 0.40 / 0.52 | +| slope, all / worst 8% (about 34 expected for a real dk effect) | 5.9 / 15.7 | 6.7 / 14.1 | +| delta_dk SD across genotypes / median SE | 0.0086 / 0.0026 | 0.0118 / 0.0041 | +| r(dk_kanR, dk_pheS) | 0.26 | 0.04 | + +**No evidence for a library-specific dk_geno.** The offset-delta_dk +correlation is as strong on the simulation, where none exists: the +kanR-only fit takes its dk from the same kanR control tubes `dk_kanR` is +estimated from, so their noise is shared. The real data do not exceed +that benchmark, and the slopes are far below what a real dk difference +would give. delta_dk's spread is about three times its Poisson SE in +both (tube-level noise the counting SE does not see). The real controls +agree less on each genotype's dk (r 0.04 against 0.26), with larger +SEs; that fits a smaller real dk spread relative to the noise and does +not single out a library effect. A marker-specific dk_geno is not +supported and would be poorly constrained. The tail stays unexplained; +the per-genotype marker-consistency flag is the handling. + +### k's uncertainty in X (seed 1, 2026-10-13) + +Both k-band refits (k at plus and minus one jackknife SD along the main +direction, from `stage2_single_seed1`'s MAP) converged. `k_band.py` adds +X's k-driven SD in quadrature to the stage 2 Laplace SD: + +| seed-1 doubles | stage 2 Laplace only | with k's uncertainty | +|---|---|---| +| 1e3-1e4: 95% coverage / width | 0.883 / 0.48 | 0.897 / 0.50 | +| 1e4-1e5: 95% coverage / width | 0.896 / 0.20 | 0.939 / 0.24 | +| > 1e5: 95% coverage / width | 0.761 / 0.07 | 0.905 / 0.15 | + +**The deep genotypes' intervals come close to nominal.** Points, slope +and RMSE are unchanged; only the widths grow, and only where k matters. +What is left above 1e5 reads (0.905) is the accepted steep m bias (slope +1.03). + +**k's uncertainty is a shift shared by every genotype.** One SD moves +every non-wt genotype's X by -0.032 (low IPTG) to -0.038 (1 mM), with a +genotype-to-genotype SD of about 0.005 and no dependence on the genotype's +own X; wt is fixed by the gauge. In quadrature it is about 0.034 for +every genotype, against a Laplace SD of 1.37 at <= 100 reads and 0.018 +above 1e5. Because it is shared, it should not widen comparisons between +genotypes; epistasis on X (X_ab - X_a - X_b + X_wt) keeps one copy of it, +since wt does not shift. The pipeline should therefore carry it as one +shared random offset per posterior draw (per IPTG level) added to every +non-wt genotype, not as independent width, so that downstream draws +(`tfs-predict-epistasis`, rankings) get the correlation right. + diff --git a/planning/studies/m-bias-factorial/control_dk_check.py b/planning/studies/m-bias-factorial/control_dk_check.py new file mode 100644 index 00000000..6a48db21 --- /dev/null +++ b/planning/studies/m-bias-factorial/control_dk_check.py @@ -0,0 +1,139 @@ +""" +Do genotypes whose X disagrees between markers also grow differently in +the two libraries' control conditions? (planning/deep-coverage.md step 3) + + python control_dk_check.py tfs_growth.csv both/tfs_pred_theta.csv \ + kanR_only/tfs_pred_theta.csv --out control_dk.csv + +A library-specific dk_geno (the genotype growing differently on the kanR +and pheS backbones) shows up directly in each library's control condition +(kanR-kan, pheS-4CP), where m is near 0: there a genotype's growth +relative to wt is its dk. For each deep double (at least --min_reads total +reads) and each control series (replicate x condition x IPTG level), the +slope over time of ln((counts_g + 0.5) / (counts_wt + 0.5)) estimates dk_g +(weighted least squares, Poisson weights 1 / (1/c_g + 1/c_wt)); series +combine by inverse variance into dk_kanR and dk_pheS, and delta_dk = +dk_kanR - dk_pheS. + +The X disagreement is each genotype's mean offset between the kanR-only +and both-marker fits after the best affine map (the level part, 91-92% of +the disagreement). If the kanR library's dk differs from the shared value +by delta, the kanR-only fit's X moves by about -delta / m_kanR relative to +the shared fit; with the shared dk near the two libraries' mean, the +offset is about delta_dk / (2 |m_kanR|) (--m_kan, default -0.0147). + +Prints the correlation between the two libraries' dk estimates (with a +shared dk, how well the estimator recovers it), the correlation of offset +and delta_dk over all deep doubles and +over the worst --tail share by disagreement, the regression slope against +the expected 1 / (2 |m_kanR|), and the noise in delta_dk. Writes one row +per genotype. A simulation that shares dk_geno across libraries gives the +no-signal benchmark. +""" + +import argparse + +import numpy as np +import pandas as pd + +CHUNK = 5_000_000 +CONTROLS = {"kanR": "kanR-kan", "pheS": "pheS-4CP"} + + +def depth_and_controls(path, min_reads): + tot = None + for c in pd.read_csv(path, usecols=["genotype", "counts"], chunksize=CHUNK): + s = c.groupby("genotype")["counts"].sum() + tot = s if tot is None else tot.add(s, fill_value=0) + g = tot.index.to_series().astype(str) + deep = set(tot.index[(g.str.count("/") == 1) & (tot >= min_reads)]) + keep = deep | {"wt"} + cols = ["genotype", "library", "replicate", "condition_sel", + "titrant_conc", "t_sel", "sample", "counts"] + parts = [] + for c in pd.read_csv(path, usecols=cols, chunksize=CHUNK): + c = c[c["condition_sel"].isin(CONTROLS.values()) & c["genotype"].isin(keep)] + parts.append(c) + return deep, pd.concat(parts) + + +def control_dk(rows): + """Per genotype and library: dk relative to wt and its SE.""" + wt = rows[rows["genotype"] == "wt"][["sample", "counts"]].rename( + columns={"counts": "c_wt"}) + d = rows[rows["genotype"] != "wt"].merge(wt, on="sample") + d["y"] = np.log((d["counts"] + 0.5) / (d["c_wt"] + 0.5)) + d["w"] = 1.0 / (1.0 / (d["counts"] + 0.5) + 1.0 / (d["c_wt"] + 0.5)) + key = ["genotype", "library", "replicate", "condition_sel", "titrant_conc"] + d["wt_"] = d["w"] * d["t_sel"] + d["wy"] = d["w"] * d["y"] + s = d.groupby(key).agg(sw=("w", "sum"), swt=("wt_", "sum"), + swy=("wy", "sum"), n=("y", "size")) + d = d.join(s, on=key) + d["tc"] = d["t_sel"] - d["swt"] / d["sw"] + d["yc"] = d["y"] - d["swy"] / d["sw"] + d["wtt"] = d["w"] * d["tc"] ** 2 + d["wty"] = d["w"] * d["tc"] * d["yc"] + s = d.groupby(key).agg(stt=("wtt", "sum"), sty=("wty", "sum"), + n=("y", "size")).reset_index() + s = s[(s["n"] >= 2) & (s["stt"] > 0)] + s["slope"] = s["sty"] / s["stt"] + s["var"] = 1.0 / s["stt"] + s["iv"] = 1.0 / s["var"] + s["iv_slope"] = s["iv"] * s["slope"] + out = s.groupby(["genotype", "library"]).agg( + iv=("iv", "sum"), ivs=("iv_slope", "sum"), n_series=("slope", "size")) + out["dk"] = out["ivs"] / out["iv"] + out["dk_se"] = 1.0 / np.sqrt(out["iv"]) + return out[["dk", "dk_se", "n_series"]].unstack("library") + + +def x_offsets(both, kan, genotypes): + cols = ["genotype", "titrant_conc", "q0.5"] + m = pd.read_csv(both, usecols=cols).merge( + pd.read_csv(kan, usecols=cols), on=["genotype", "titrant_conc"], + suffixes=("_b", "_k")) + m = m[m["genotype"].isin(genotypes)] + p = np.polyfit(m["q0.5_b"], m["q0.5_k"], 1) + m["res"] = m["q0.5_k"] - np.polyval(p, m["q0.5_b"]) + g = m.groupby("genotype")["res"] + return pd.DataFrame({"offset": g.mean(), + "rms": g.apply(lambda r: np.sqrt(np.mean(r ** 2)))}) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("growth_csv") + ap.add_argument("both_pred") + ap.add_argument("kan_pred") + ap.add_argument("--min_reads", type=float, default=1e4) + ap.add_argument("--tail", type=float, default=0.08) + ap.add_argument("--m_kan", type=float, default=-0.0147) + ap.add_argument("--out", default="control_dk.csv") + args = ap.parse_args() + + deep, rows = depth_and_controls(args.growth_csv, args.min_reads) + dk = control_dk(rows) + dk.columns = [f"{a}_{b}" for a, b in dk.columns] + off = x_offsets(args.both_pred, args.kan_pred, deep) + df = off.join(dk, how="inner").dropna(subset=["dk_kanR", "dk_pheS"]) + df["delta_dk"] = df["dk_kanR"] - df["dk_pheS"] + df["delta_dk_se"] = np.hypot(df["dk_se_kanR"], df["dk_se_pheS"]) + df.to_csv(args.out) + + expect = 1.0 / (2 * abs(args.m_kan)) + tail = df[df["rms"] >= df["rms"].quantile(1 - args.tail)] + print(f"{len(df)} deep doubles with both controls; tail {len(tail)}") + print(f"delta_dk: SD across genotypes {df['delta_dk'].std():.4f}, " + f"median SE {df['delta_dk_se'].median():.4f} per minute") + print(f"r(dk_kanR, dk_pheS) {np.corrcoef(df['dk_kanR'], df['dk_pheS'])[0, 1]:.3f} " + f"(how well the two controls agree on each genotype's dk)") + for name, sub in (("all", df), ("tail", tail)): + r = np.corrcoef(sub["offset"], sub["delta_dk"])[0, 1] + slope = np.polyfit(sub["delta_dk"], sub["offset"], 1)[0] + print(f"{name}: r(offset, delta_dk) {r:.3f}, slope {slope:.1f} " + f"(expected about {expect:.0f})") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/depth.py b/planning/studies/m-bias-factorial/depth.py new file mode 100644 index 00000000..b04781c8 --- /dev/null +++ b/planning/studies/m-bias-factorial/depth.py @@ -0,0 +1,29 @@ +""" +Total reads per genotype from a growth table (for score_x.py's depth bins). + + python depth.py sim_realistic_s2/tfs_growth.csv --out depth.csv + +Reads the table in chunks (a full-size one is 8 GB). +""" + +import argparse + +import pandas as pd + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("growth_csv") + ap.add_argument("--out", default="depth.csv") + args = ap.parse_args() + tot = None + for chunk in pd.read_csv(args.growth_csv, usecols=["genotype", "counts"], + chunksize=5_000_000): + s = chunk.groupby("genotype")["counts"].sum() + tot = s if tot is None else tot.add(s, fill_value=0) + tot.rename("reads").to_csv(args.out) + print(f"{len(tot)} genotypes -> {args.out}") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/held_priors.py b/planning/studies/m-bias-factorial/held_priors.py new file mode 100644 index 00000000..e9d57539 --- /dev/null +++ b/planning/studies/m-bias-factorial/held_priors.py @@ -0,0 +1,72 @@ +""" +Growth priors that hold k and m at a subset fit's MAP (m-bias factorial). + + python held_priors.py no_doubles --out growth_priors_held.csv + python held_priors.py no_doubles --hold kanR --loose growth_priors_loose.csv \ + --out growth_priors_held.csv + python held_priors.py no_doubles_mix --only kanR --out growth_priors_held.csv + +Reads the arm's tfs_params_growth_k.csv and tfs_params_growth_m.csv (one +row per condition_rep with --growth_shares_replicates) and writes a +--growth_priors table with those values as k_loc and m_loc. + +Without --hold every condition is held: configure with --set_priors +m_pinned=1 k_pinned=1 to clamp them (the scales are then unused). With +--hold, only conditions whose name starts with one of the given prefixes +are held, clamped through the table's k_pinned/m_pinned columns (linear's +per-condition pins); every other condition takes its loc and scale from +--loose and is fitted. Do not add the --set_priors pins then: they act on +every condition. (Round 4 held kanR by a tight prior instead, SD 1e-4; it +leaked about 11% of the doubles' pull.) --only keeps just the conditions +starting with the given prefixes, for a model fitted to one library's +data (the kanR-only check). +""" + +import argparse +import os + +import pandas as pd + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("arm_dir") + ap.add_argument("--out", default="growth_priors_held.csv") + ap.add_argument("--hold", nargs="+", default=None, + help="condition_rep prefixes to hold (default: all)") + ap.add_argument("--only", nargs="+", default=None, + help="keep only conditions starting with these prefixes") + ap.add_argument("--loose", default=None, + help="growth priors for the conditions not held") + args = ap.parse_args() + km = {} + for p in ("k", "m"): + df = pd.read_csv(os.path.join(args.arm_dir, f"tfs_params_growth_{p}.csv")) + km[p] = df.groupby("condition_rep")["q0.5"].mean() + out = pd.DataFrame({"k_loc": km["k"], "k_scale": 0.002, + "m_loc": km["m"], "m_scale": 0.002}) + out.index.name = "condition_rep" + if args.only is not None: + out = out[[any(c.startswith(p) for p in args.only) for c in out.index]] + if out.empty: + ap.error(f"no condition starts with {args.only}") + if args.hold is not None: + if args.loose is None: + ap.error("--hold needs --loose for the conditions not held") + loose = pd.read_csv(args.loose).set_index("condition_rep") + held = [c for c in out.index if any(c.startswith(h) for h in args.hold)] + if not held: + ap.error(f"no condition starts with {args.hold}: {list(out.index)}") + free = [c for c in out.index if c not in held] + missing = [c for c in free if c not in loose.index] + if missing: + ap.error(f"{args.loose} has no row for {missing}") + out = pd.concat([out.loc[held], loose.loc[free, out.columns]]) + out["k_pinned"] = [int(c in held) for c in out.index] + out["m_pinned"] = out["k_pinned"] + out.to_csv(args.out) + print(out.to_string()) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/jackknife_summary.py b/planning/studies/m-bias-factorial/jackknife_summary.py new file mode 100644 index 00000000..03b03101 --- /dev/null +++ b/planning/studies/m-bias-factorial/jackknife_summary.py @@ -0,0 +1,135 @@ +""" +k and m standard errors from a delete-d jackknife of a stage-1 fit. + + python jackknife_summary.py jk_dir --full ../m_subsets/no_doubles_mix \ + --truth ../sim_realistic_s2 --out jackknife_km.csv + +jk_dir holds jk_NN/ from run_jackknife.srun (each with resample.log and +tfs_params_growth_{k,m}.csv). For n tubes, d dropped per subsample and the +subsample estimates t_i with mean t_bar, the jackknife variance is +(n - d) / d * mean((t_i - t_bar)^2). Prints, per condition, the full fit's +k and m (--full), the jackknife SE and, with --truth (a simulation +directory holding tfs_sim_parameters.csv and tfs_sim_growth_parameters.csv), +the truth on the X scale and the error in jackknife SEs. + +--pc1_tables DIR also writes DIR/plus/ and DIR/minus/growth_priors_held.csv: +k at the full fit's value plus and minus one jackknife SD along the first +principal component of the k covariance (m at the full fit's value), for +two stage 2 refits that measure how X moves with k. It prints the share +of k's variance on that component. +""" + +import argparse +import glob +import os +import re + +import numpy as np +import pandas as pd +import yaml + + +def read_km(d): + out = {} + for p in ("k", "m"): + f = os.path.join(d, f"tfs_params_growth_{p}.csv") + out[p] = pd.read_csv(f).groupby("condition_rep")["q0.5"].mean() + return pd.DataFrame(out) + + +def n_and_d(d): + text = open(os.path.join(d, "resample.log")).read() + m = re.search(r"n (\d+) tubes, dropped d (\d+)", text) + return int(m.group(1)), int(m.group(2)) + + +def truth_x(sim_dir, config): + """k and m on the fit's X scale (wt's true curve at the gauge).""" + cfg = yaml.safe_load(open(config)) + c_lo, c_hi = (float(x) for x in cfg["components"]["theta_gauge_conc"]) + par = pd.read_csv(os.path.join(sim_dir, "tfs_sim_parameters.csv")) + wt = par[par["genotype"] == "wt"].iloc[0] + + def th(c): + if c <= 0: + return wt["theta_low"] + occ = 1 / (1 + np.exp(-wt["hill_n"] * (np.log(c) - wt["log_hill_K"]))) + return wt["theta_low"] + (wt["theta_high"] - wt["theta_low"]) * occ + + tr = pd.read_csv(os.path.join(sim_dir, "tfs_sim_growth_parameters.csv")) + tr = tr.set_index("condition_rep") + m = tr["growth_m"] * (th(c_lo) - th(c_hi)) + k = tr["growth_k"] + tr["growth_m"] * th(c_hi) + return pd.DataFrame({"k": k, "m": m}) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("jk_dir") + ap.add_argument("--full", default=None, help="the full stage-1 fit's dir") + ap.add_argument("--truth", default=None, help="simulation dir") + ap.add_argument("--out", default="jackknife_km.csv") + ap.add_argument("--pc1_tables", default=None, + help="write plus/ and minus/ pinned tables here (needs --full)") + args = ap.parse_args() + + dirs = sorted(d for d in glob.glob(os.path.join(args.jk_dir, "jk_*")) + if os.path.exists(os.path.join(d, "tfs_params_growth_m.csv"))) + if len(dirs) < 2: + raise SystemExit(f"{len(dirs)} finished subsamples in {args.jk_dir}") + nd = {n_and_d(d) for d in dirs} + if len(nd) != 1: + raise SystemExit(f"subsamples differ in n and d: {sorted(nd)}") + n, d = nd.pop() + est = pd.concat({os.path.basename(x): read_km(x) for x in dirs}) + est.index.names = ["subsample", "condition_rep"] + + rows = [] + for c, g in est.groupby(level="condition_rep"): + row = {"condition_rep": c, "n_sub": len(g)} + for p in ("k", "m"): + t = g[p].to_numpy() + row[f"{p}_jk_mean"] = t.mean() + row[f"{p}_jk_se"] = np.sqrt((n - d) / d * np.mean((t - t.mean()) ** 2)) + rows.append(row) + out = pd.DataFrame(rows).set_index("condition_rep") + + if args.full: + full = read_km(args.full) + for p in ("k", "m"): + out[f"{p}_full"] = full[p] + if args.truth: + truth = truth_x(args.truth, + os.path.join(args.full, "tfs_configure_config.yaml")) + for p in ("k", "m"): + out[f"{p}_truth"] = truth[p] + out[f"{p}_err_in_se"] = (out[f"{p}_full"] - out[f"{p}_truth"]) \ + / out[f"{p}_jk_se"] + if args.pc1_tables: + if not args.full: + raise SystemExit("--pc1_tables needs --full") + k = est["k"].unstack("condition_rep") + dev = (k - k.mean()) * np.sqrt((n - d) / d) + cov = np.cov(dev.to_numpy().T) + w, v = np.linalg.eigh(cov) + top = int(np.argmax(w)) + step = pd.Series(v[:, top] * np.sqrt(w[top]), index=k.columns) + full = read_km(args.full) + print(f"k variance on the first component: {w[top] / w.sum():.3f}; " + f"one-SD step {step.round(5).to_dict()}") + for name, sign in (("plus", 1), ("minus", -1)): + dd = os.path.join(args.pc1_tables, name) + os.makedirs(dd, exist_ok=True) + table = pd.DataFrame({ + "condition_rep": full.index, + "k_loc": full["k"] + sign * step.reindex(full.index), + "k_scale": 0.002, "m_loc": full["m"], "m_scale": 0.002}) + table.to_csv(os.path.join(dd, "growth_priors_held.csv"), index=False) + out.to_csv(args.out) + print(f"{len(dirs)} subsamples, n {n} tubes, d {d} dropped " + f"(variance factor {(n - d) / d:.2f})") + print(out.to_string(float_format="%.5f")) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/k_band.py b/planning/studies/m-bias-factorial/k_band.py new file mode 100644 index 00000000..754dcb9f --- /dev/null +++ b/planning/studies/m-bias-factorial/k_band.py @@ -0,0 +1,53 @@ +""" +X intervals that carry stage 1's k uncertainty (planning/deep-coverage.md +step 2). + + python k_band.py stage2_single_seed1 pc1/plus pc1/minus --out x_kband.csv + +The first directory is the stage 2 fit at stage 1's k and m; the other two +are stage 2 refits with k moved one jackknife SD up and down along the +first principal component of k's jackknife covariance (jackknife_summary.py +--pc1_tables; it carries most of k's variance). Per genotype and +concentration, X's k-driven SD is |X(plus) - X(minus)| / 2 (the medians), +added in quadrature to the stage 2 Laplace SD ((q0.975 - q0.025) / 3.92). +Writes genotype, titrant_conc, q0.025, q0.5 (the first fit's), q0.975 and +both SDs, for score_x.py. +""" + +import argparse +import os + +import pandas as pd + +COLS = ["genotype", "titrant_name", "titrant_conc", "q0.025", "q0.5", "q0.975"] +KEY = ["genotype", "titrant_name", "titrant_conc"] + + +def pred(d): + return pd.read_csv(os.path.join(d, "tfs_pred_theta.csv"), usecols=COLS) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("base") + ap.add_argument("plus") + ap.add_argument("minus") + ap.add_argument("--out", default="x_kband.csv") + args = ap.parse_args() + base = pred(args.base) + plus = pred(args.plus)[KEY + ["q0.5"]].rename(columns={"q0.5": "x_plus"}) + minus = pred(args.minus)[KEY + ["q0.5"]].rename(columns={"q0.5": "x_minus"}) + df = base.merge(plus, on=KEY).merge(minus, on=KEY) + df["sd_laplace"] = (df["q0.975"] - df["q0.025"]) / 3.92 + df["sd_k"] = (df["x_plus"] - df["x_minus"]).abs() / 2 + sd = (df["sd_laplace"] ** 2 + df["sd_k"] ** 2) ** 0.5 + df["q0.025"] = df["q0.5"] - 1.96 * sd + df["q0.975"] = df["q0.5"] + 1.96 * sd + df[KEY + ["q0.025", "q0.5", "q0.975", "sd_laplace", "sd_k"]].to_csv( + args.out, index=False) + print(f"{len(df)} rows; median sd_laplace {df['sd_laplace'].median():.4f}, " + f"median sd_k {df['sd_k'].median():.4f}") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/km_draws.py b/planning/studies/m-bias-factorial/km_draws.py new file mode 100644 index 00000000..eeaf04fc --- /dev/null +++ b/planning/studies/m-bias-factorial/km_draws.py @@ -0,0 +1,105 @@ +""" +Draws of k and m from a stage-1 fit's Laplace, one held-k/m arm per draw +(planning/deep-coverage.md step 2). + + python km_draws.py ../m_subsets/no_doubles --num_draws 10 --out_dir km_draws + +The stage-1 directory needs tfs_posterior.h5 (tfs-sample-posterior on its +MAP checkpoint) and tfs_configure_priors.csv (whose per-condition rows name +the conditions in the model's order). Each draw is one joint sample of every +condition's k and m, so their correlations carry over. Each draw directory +gets growth_priors_held.csv with that draw as k_loc and m_loc; run_held.srun +pins every condition at it (its default PINS). + +The k/m uncertainty is only as good as the Laplace: a k or m direction the +arrowhead Laplace held at zero width (tfs_posterior_held_directions.csv) +has no spread in the draws, so the script stops if one does, unless +--allow_held. +""" + +import argparse +import os + +import h5py +import numpy as np +import pandas as pd + +PREFIX = "growth.condition_growth." + + +def condition_order(priors_csv): + """condition_rep labels in the model's order, from the indexed k_loc rows.""" + df = pd.read_csv(priors_csv) + rows = df[df["parameter"] == PREFIX + "k_loc"].dropna(subset=["flat_index"]) + if rows.empty or "condition_rep" not in rows: + raise ValueError(f"{priors_csv} has no per-condition k_loc rows to " + "name the conditions (configure with --growth_priors).") + if "replicate" in rows and rows["replicate"].notna().any(): + raise ValueError("per-replicate conditions are not supported; configure " + "with --growth_shares_replicates.") + return list(rows.sort_values("flat_index")["condition_rep"].astype(str)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("stage1_dir") + ap.add_argument("--num_draws", type=int, default=10) + ap.add_argument("--out_dir", default="km_draws") + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--allow_held", action="store_true") + args = ap.parse_args() + s1 = args.stage1_dir + + held_file = os.path.join(s1, "tfs_posterior_held_directions.csv") + if os.path.exists(held_file): + held = pd.read_csv(held_file) + km = held[held["parameter"].str.startswith("condition_growth_") + & (held["loading"].abs() > 0.1)] + if not km.empty: + print("Held directions with k or m loadings above 0.1:") + print(km.to_string(index=False)) + if not args.allow_held: + raise SystemExit("k/m directions are held at zero width; the " + "draws would understate their uncertainty " + "(--allow_held to go on anyway).") + + labels = condition_order(os.path.join(s1, "tfs_configure_priors.csv")) + with h5py.File(os.path.join(s1, "tfs_posterior.h5"), "r") as hf: + k = np.asarray(hf["condition_growth_k"]).reshape(-1, len(labels)) + m = np.asarray(hf["condition_growth_m"]).reshape(-1, len(labels)) + + # The extracted medians must match the posterior's columns by name. + for name, arr in (("k", k), ("m", m)): + ext = pd.read_csv(os.path.join(s1, f"tfs_params_growth_{name}.csv")) + ext = ext.groupby("condition_rep")["q0.5"].mean() + med = pd.Series(np.median(arr, axis=0), index=labels) + off = (med - ext.reindex(labels)).abs().max() + if not off < 1e-3: + raise ValueError(f"{name}: posterior medians {med.round(5).to_dict()} " + f"do not match tfs_params_growth_{name}.csv " + f"{ext.round(5).to_dict()}; condition order?") + + summary = pd.DataFrame({"k_sd": k.std(axis=0), "m_sd": m.std(axis=0), + "k_median": np.median(k, axis=0), + "m_median": np.median(m, axis=0)}, index=labels) + print(f"{k.shape[0]} posterior draws of k and m:") + print(summary.to_string(float_format="%.5f")) + + rng = np.random.default_rng(args.seed) + pick = rng.choice(k.shape[0], size=args.num_draws, replace=False) + rows = [] + for i, j in enumerate(pick): + d = os.path.join(args.out_dir, f"draw_{i:02d}") + os.makedirs(d, exist_ok=True) + table = pd.DataFrame({"condition_rep": labels, "k_loc": k[j], + "k_scale": 0.002, "m_loc": m[j], "m_scale": 0.002}) + table.to_csv(os.path.join(d, "growth_priors_held.csv"), index=False) + rows.append(table.assign(draw=i, sample=int(j))) + pd.concat(rows).to_csv(os.path.join(args.out_dir, "km_draws.csv"), + index=False) + print(f"wrote {args.num_draws} draws to {args.out_dir}/draw_*/ " + f"(table: {args.out_dir}/km_draws.csv)") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/make_factorial.py b/planning/studies/m-bias-factorial/make_factorial.py new file mode 100644 index 00000000..afc019a7 --- /dev/null +++ b/planning/studies/m-bias-factorial/make_factorial.py @@ -0,0 +1,137 @@ +""" +Reduced-library factorial for the |m| bias (planning/deep-coverage.md, step 1b). + + python make_factorial.py --pilot [--out_dir m_bias] [--seeds 1 2] + +Run from planning/dev-data (needs processed/; the lab's data stay there, +C10). Builds the full-size calibrated configs with +planning/studies/full-size-sim/make_sim_config.py (realistic and Poisson +noise, cfu0 calibrated from --pilot), then shrinks the library to 5 +degenerate sites per tile (tile 1 residues 32, 37, 42, 47, 52; tile 2 +residues 74, 79, 84, 88, 93, every spike site kept: about 5,000 doubles +instead of 223,000) with genetics.library_design.scale_library_design, so +every sequence keeps its reads, cells, transformants and PCR templates. The +fit is configured with the real library config shrunk the same way. + +Arms (each switches off one feature of the simulation the fit does not +model): + +- base: the calibrated realistic simulation (congression lambda 0.357, + per-tube rate noise 0.002 per minute, founder/demographic/PCR noise) +- no_cong: transformation_poisson_lambda 0 (no co-transformed cells) +- no_tube: tube_noise_sigma 0 +- poisson: Poisson counts (the Poisson config: no founder, demographic or + PCR noise) +- clean: all three off + +Each arm directory gets simulate_config.yaml, library_config.yaml and a +copy of growth_priors_loose.csv; run_arm.sh runs the chain in it. +""" + +import argparse +import copy +import os +import shutil +import subprocess +import sys + +import yaml + +from tfscreen.genetics.library_design import scale_library_design +from tfscreen.genetics.library_manager import LibraryManager + +HERE = os.path.dirname(os.path.abspath(__file__)) +MAKE_SIM = os.path.join(HERE, "..", "full-size-sim", "make_sim_config.py") +LOOSE = os.path.join(HERE, "..", "full-size-sim", "growth_priors_loose.csv") +KEEP_RESIDUES = {32, 37, 42, 47, 52, 74, 79, 84, 88, 93} +ARMS = { + "base": ("realistic", {}), + "no_cong": ("realistic", {"transformation_poisson_lambda": 0.0}), + "no_tube": ("realistic", {"tube_noise_sigma": 0.0}), + "poisson": ("poisson", {}), + "clean": ("poisson", {"transformation_poisson_lambda": 0.0, + "tube_noise_sigma": 0.0}), +} + + +def shrink_degen(cfg): + """degen_sites with only KEEP_RESIDUES left degenerate.""" + d, r0 = cfg["degen_sites"], int(cfg["first_amplicon_residue"]) + codons = [d[i:i + 3] for i in range(0, len(d), 3)] + out = [c if (c == "..." or r0 + i in KEEP_RESIDUES) else "..." + for i, c in enumerate(codons)] + return "".join(out) + + +def shrink(sim_cf, real_lib): + """The reduced-library simulate config and library config.""" + cf = copy.deepcopy(sim_cf) + cf["degen_sites"] = shrink_degen(cf) + ref = LibraryManager(sim_cf).build_library_df() + tgt = LibraryManager(cf).build_library_df() + scaled = scale_library_design(ref, tgt, sim_cf["library_mixture"], + sim_cf["transform_sizes"], + total_num_reads=sim_cf["total_num_reads"]) + ratio = scaled["total_num_reads"] / sim_cf["total_num_reads"] + cf["library_mixture"] = scaled["library_mixture"] + cf["transform_sizes"] = scaled["transform_sizes"] + cf["total_num_reads"] = scaled["total_num_reads"] + if isinstance(cf["cfu0"], dict): + cf["cfu0"] = {k: float(v) * ratio for k, v in cf["cfu0"].items()} + else: + cf["cfu0"] = float(cf["cfu0"]) * ratio + if cf.get("pcr_template_molecules"): + cf["pcr_template_molecules"] = max( + 1, int(round(cf["pcr_template_molecules"] * ratio))) + if cf.get("max_congressed_cells"): + cf["max_congressed_cells"] = max( + 1000, int(round(cf["max_congressed_cells"] * ratio))) + lib = copy.deepcopy(real_lib) + lib["degen_sites"] = shrink_degen(lib) + n = (tgt["library_origin"] == "double-1-2").sum() + return cf, lib, ratio, n + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--pilot", required=True) + ap.add_argument("--out_dir", default="m_bias") + ap.add_argument("--seeds", type=int, nargs="+", default=[1, 2]) + args = ap.parse_args() + os.makedirs(args.out_dir, exist_ok=True) + + full = {} + for noise in ("realistic", "poisson"): + d = os.path.join(args.out_dir, f"_full_{noise}") + if not os.path.exists(os.path.join(d, "simulate_config.yaml")): + subprocess.run([sys.executable, MAKE_SIM, "--out_dir", d, + "--seed", "1", "--noise", noise, + "--calibrate_from", args.pilot], check=True) + with open(os.path.join(d, "simulate_config.yaml")) as fh: + full[noise] = (d, yaml.safe_load(fh)) + with open(os.path.join("processed", "library_config.yaml")) as fh: + real_lib = yaml.safe_load(fh) + + for arm, (noise, changes) in ARMS.items(): + d_full, sim_cf = full[noise] + cf, lib, ratio, n = shrink(sim_cf, real_lib) + cf.update(changes) + for seed in args.seeds: + d = os.path.join(args.out_dir, f"{arm}_s{seed}") + os.makedirs(d, exist_ok=True) + cf["seed"] = seed + with open(os.path.join(d, "simulate_config.yaml"), "w") as fh: + fh.write(f"# make_factorial.py arm {arm}: {noise} noise, " + f"changes {changes}, seed {seed}\n") + yaml.safe_dump(cf, fh, sort_keys=False) + with open(os.path.join(d, "library_config.yaml"), "w") as fh: + yaml.safe_dump(lib, fh, sort_keys=False) + for f in ("design.csv", "od600_calibration.yaml"): + shutil.copyfile(os.path.join(d_full, f), os.path.join(d, f)) + shutil.copyfile(LOOSE, os.path.join(d, "growth_priors_loose.csv")) + print(f"{arm}: {n} doubles, scale {ratio:.4f}, reads " + f"{cf['total_num_reads']:.3g}, changes {changes}") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/make_subsets.py b/planning/studies/m-bias-factorial/make_subsets.py new file mode 100644 index 00000000..0b1ebce5 --- /dev/null +++ b/planning/studies/m-bias-factorial/make_subsets.py @@ -0,0 +1,135 @@ +""" +Fit-side subsets of a full-size simulation (m-bias factorial, round 2). + + python make_subsets.py sim_realistic_s2 --out_dir m_subsets + python make_subsets.py recipe --out_dir m_subsets_real --arms no_doubles \ + --library_config ../processed/library_config.yaml + +Each arm refits the same simulated growth table with only some genotypes. +Given the shared parameters (k, m, tube offsets, population hypers) each +genotype's count likelihood is independent, so a subset is a valid fit of +the same data; what changes is which genotypes inform k and m. + + no_doubles wt, spikes and singles + spikes wt and the spiked genotypes only (the library config's + spiked_seqs): the clones least affected by congression + spikes_deep spikes plus the --n_deep_singles deepest singles: enough + genotypes to pin X's scale, less congression than every + single + deep_doubles no_doubles plus every double with >= DEEP total reads + shallow_doubles no_doubles plus as many doubles below DEEP, at random + +Each arm directory gets the filtered tfs_growth.csv, links to the +simulation's tfs_sim_* files (truth, and so run_arm.sh skips simulate and +process), and copies of simulate_config.yaml (its seed), library_config.yaml +and growth_priors_loose.csv. Submit run_arm.srun from inside each. + +A real-data growth table works the same way: there are no tfs_sim_* files +or simulate_config.yaml to carry over, so pass --library_config; +run_arm.sh then starts at configure, with seed $SEED or 1. +""" + +import argparse +import os +import shutil + +import numpy as np +import pandas as pd +import yaml + +from tfscreen.genetics.library_design import library_composition_table + +HERE = os.path.dirname(os.path.abspath(__file__)) +LOOSE = os.path.join(HERE, "..", "full-size-sim", "growth_priors_loose.csv") +DEEP = 1000 +CHUNK = 5_000_000 + + +def genotype_class(g): + g = g.astype(str) + return np.select([g == "wt", g.str.count("/") == 1], ["wt", "double"], + "single") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("sim_dir") + ap.add_argument("--out_dir", default="m_subsets") + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--arms", nargs="+", + default=["no_doubles", "deep_doubles", "shallow_doubles"], + choices=["no_doubles", "deep_doubles", "shallow_doubles", + "spikes", "spikes_deep"]) + ap.add_argument("--n_deep_singles", type=int, default=300) + ap.add_argument("--library_config", default=None, + help="library config (default: sim_dir's)") + args = ap.parse_args() + growth = os.path.join(args.sim_dir, "tfs_growth.csv") + + tot = None + for chunk in pd.read_csv(growth, usecols=["genotype", "counts"], + chunksize=CHUNK): + s = chunk.groupby("genotype")["counts"].sum() + tot = s if tot is None else tot.add(s, fill_value=0) + cls = pd.Series(genotype_class(tot.index.to_series()), index=tot.index) + doubles = tot[cls == "double"] + deep = doubles.index[doubles >= DEEP] + shallow_all = doubles.index[doubles < DEEP] + rng = np.random.default_rng(args.seed) + shallow = rng.choice(shallow_all, size=min(len(deep), len(shallow_all)), + replace=False) + core = tot.index[cls != "double"] + arms = {"no_doubles": set(core), + "deep_doubles": set(core) | set(deep), + "shallow_doubles": set(core) | set(shallow)} + if "spikes" in args.arms or "spikes_deep" in args.arms: + lib_path = (args.library_config + or os.path.join(args.sim_dir, "library_config.yaml")) + with open(lib_path) as fh: + comp = library_composition_table(yaml.safe_load(fh)) + spiked = set(comp.loc[comp["in_spiked_origin"].astype(bool), "genotype"]) + arms["spikes"] = (spiked | {"wt"}) & set(tot.index) + singles = tot[(cls == "single") & ~tot.index.isin(arms["spikes"])] + deepest = singles.sort_values(ascending=False).index[:args.n_deep_singles] + arms["spikes_deep"] = arms["spikes"] | set(deepest) + print(f"spikes_deep: the {len(deepest)} deepest singles have " + f">= {int(singles.loc[deepest].min())} reads") + arms = {a: arms[a] for a in args.arms} + print(f"{len(core)} wt/spike/single genotypes, {len(deep)} doubles >= " + f"{DEEP} reads, {len(shallow)} of {len(shallow_all)} below") + + os.makedirs(args.out_dir, exist_ok=True) + out = {} + for arm in arms: + d = os.path.join(args.out_dir, arm) + os.makedirs(d, exist_ok=True) + out[arm] = os.path.join(d, "tfs_growth.csv") + if os.path.exists(out[arm]): + os.remove(out[arm]) + for chunk in pd.read_csv(growth, chunksize=CHUNK): + for arm, keep in arms.items(): + sub = chunk[chunk["genotype"].isin(keep)] + sub.to_csv(out[arm], mode="a", index=False, + header=not os.path.exists(out[arm])) + + sim = os.path.abspath(args.sim_dir) + for arm in arms: + d = os.path.join(args.out_dir, arm) + for f in os.listdir(sim): + if f.startswith("tfs_sim_"): + link = os.path.join(d, f) + if not os.path.lexists(link): + os.symlink(os.path.join(sim, f), link) + if os.path.exists(os.path.join(sim, "simulate_config.yaml")): + shutil.copyfile(os.path.join(sim, "simulate_config.yaml"), + os.path.join(d, "simulate_config.yaml")) + shutil.copyfile(args.library_config + or os.path.join(sim, "library_config.yaml"), + os.path.join(d, "library_config.yaml")) + shutil.copyfile(LOOSE, os.path.join(d, "growth_priors_loose.csv")) + n = pd.read_csv(out[arm], usecols=["genotype"])["genotype"].nunique() + print(f"{arm}: {n} genotypes") + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/resample_growth.py b/planning/studies/m-bias-factorial/resample_growth.py new file mode 100644 index 00000000..0dd46160 --- /dev/null +++ b/planning/studies/m-bias-factorial/resample_growth.py @@ -0,0 +1,81 @@ +""" +Filter or resample a growth table by tube (planning/deep-coverage.md). + + python resample_growth.py tfs_growth.csv --library kanR --out kanR/tfs_growth.csv + python resample_growth.py tfs_growth.csv --drop_frac 0.2 --seed 3 --out boot_03/tfs_growth.csv + +--library keeps one library's tubes (the kanR-only check). --drop_frac +drops that share of the tubes at random, stratified by condition_sel and +titrant_conc and keeping at least one tube per stratum: a delete-d +jackknife. Tubes cannot be resampled with replacement, since a duplicated +tube lands on the same tensor cell as the original. For a delete-d +jackknife with n tubes, d dropped and N subsamples, the variance of an +estimate is (n - d) / d times the mean squared deviation of the +subsample estimates from their mean (boot_summary.py applies it). + +Writes the kept tubes' rows and prints n, d and the dropped tubes. +Reads in chunks (a full-size table is 8 GB). +""" + +import argparse +import os + +import numpy as np +import pandas as pd + +CHUNK = 5_000_000 + + +def tube_table(path): + """One row per tube: sample, library, condition_sel, titrant_conc.""" + cols = ["sample", "library", "condition_sel", "titrant_conc"] + parts = [c.drop_duplicates() for c in + pd.read_csv(path, usecols=cols, chunksize=CHUNK)] + return pd.concat(parts).drop_duplicates("sample").reset_index(drop=True) + + +def choose_drops(tubes, frac, seed): + """Tubes to drop: frac of each condition_sel x titrant_conc stratum, + rounded, leaving at least one.""" + rng = np.random.default_rng(seed) + drop = [] + for _, g in tubes.groupby(["condition_sel", "titrant_conc"]): + k = min(int(round(frac * len(g))), len(g) - 1) + if k > 0: + drop.extend(rng.choice(g["sample"].to_numpy(), size=k, + replace=False)) + return set(drop) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("growth_csv") + ap.add_argument("--out", required=True) + ap.add_argument("--library", default=None) + ap.add_argument("--drop_frac", type=float, default=0.0) + ap.add_argument("--seed", type=int, default=1) + args = ap.parse_args() + + tubes = tube_table(args.growth_csv) + if args.library is not None: + if args.library not in set(tubes["library"]): + raise SystemExit(f"no library {args.library!r}: " + f"{sorted(set(tubes['library']))}") + tubes = tubes[tubes["library"] == args.library] + drop = choose_drops(tubes, args.drop_frac, args.seed) if args.drop_frac else set() + keep = set(tubes["sample"]) - drop + + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + if os.path.exists(args.out): + os.remove(args.out) + for chunk in pd.read_csv(args.growth_csv, chunksize=CHUNK): + chunk[chunk["sample"].isin(keep)].to_csv( + args.out, mode="a", index=False, header=not os.path.exists(args.out)) + print(f"n {len(tubes)} tubes, dropped d {len(drop)}, kept {len(keep)}" + + (f" (library {args.library})" if args.library else "")) + if drop: + print("dropped:", " ".join(sorted(drop))) + + +if __name__ == "__main__": + main() diff --git a/planning/studies/m-bias-factorial/run_arm.sh b/planning/studies/m-bias-factorial/run_arm.sh new file mode 100755 index 00000000..069a1753 --- /dev/null +++ b/planning/studies/m-bias-factorial/run_arm.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# One factorial arm (planning/studies/m-bias-factorial), run inside the arm's +# directory: simulate, process, configure, MAP, extract. The chain and model +# are the full-size simulation's (run_full_size.srun), without the Laplace. +set -e +# Seed from the simulation's config; a real-data arm (no simulate_config.yaml, +# its own tfs_growth.csv) uses $SEED or 1. +if [[ -f simulate_config.yaml ]]; then + SEED="$(grep '^seed:' simulate_config.yaml | awk '{print $2}')" +fi +SEED="${SEED:-1}" + +[[ -f tfs_sim_tubes.csv || -f tfs_growth.csv ]] || tfs-simulate simulate_config.yaml \ + --out_prefix tfs_sim --seed "${SEED}" --no_write_growth > sim.log 2>&1 + +[[ -f tfs_growth.csv ]] || tfs-process-counts tfs_sim_tubes.csv tfs_sim_counts \ + --od600_file tfs_sim_tube_od600.csv \ + --od600_calibration_file tfs_sim_od600_calibration.yaml \ + --tube_volume_mL 5 --out_prefix tfs_growth --no_verbose > process.log 2>&1 + +# configure_extra.txt (optional, one line): extra tfs-configure-model flags +# for this arm, e.g. the congression mixture. +EXTRA="" +[[ -f configure_extra.txt ]] && EXTRA="$(cat configure_extra.txt)" + +[[ -f tfs_configure_config.yaml ]] || tfs-configure-model ${EXTRA} \ + --allow_missing_spikes \ + --growth_df tfs_growth.csv \ + --library_config library_config.yaml \ + --theta_model hill_relative \ + --growth_shares_replicates \ + --batch_size 4096 \ + --growth_priors growth_priors_loose.csv \ + --set_priors sigma_fixed=0.17 theta_log_hill_n_hyper_scale_fixed=0.5 \ + --skip_model_stats > configure.log 2>&1 + +[[ -f tfs_fit_model_params.npz ]] || tfs-fit-model tfs_configure_config.yaml \ + --seed "${SEED}" --analysis_method map > fit.log 2>&1 + +tfs-extract-params tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + > extract.log 2>&1 +echo "done" diff --git a/planning/studies/m-bias-factorial/run_arm.srun b/planning/studies/m-bias-factorial/run_arm.srun new file mode 100644 index 00000000..4929fbc9 --- /dev/null +++ b/planning/studies/m-bias-factorial/run_arm.srun @@ -0,0 +1,22 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_mbias +#SBATCH --output=run.out +#SBATCH --error=run.err +#SBATCH --partition=gpulong +#SBATCH --time=01-00:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# One m-bias factorial arm (planning/studies/m-bias-factorial), submitted +# from inside the arm's directory; run_arm.sh (one level up) runs the chain +# and skips any step whose output exists, so a resubmit resumes. + +module load cuda/12.4.1 +set -e +echo ">>> Start $(date)" +bash ../run_arm.sh +echo ">>> Done $(date)" diff --git a/planning/studies/m-bias-factorial/run_held.srun b/planning/studies/m-bias-factorial/run_held.srun new file mode 100644 index 00000000..6359d644 --- /dev/null +++ b/planning/studies/m-bias-factorial/run_held.srun @@ -0,0 +1,68 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_held +#SBATCH --output=run.out +#SBATCH --error=run.err +#SBATCH --partition=gpulong +#SBATCH --time=02-00:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# The full library with k and m held at a no-doubles fit's MAP +# (planning/studies/m-bias-factorial, round 3): the full-size chain +# (run_full_size.srun) with k and m clamped (m_pinned, k_pinned) at the +# values in growth_priors_held.csv, from held_priors.py. Run from an arm +# directory holding tfs_growth.csv, library_config.yaml and +# growth_priors_held.csv, plus links to the simulation's tfs_sim_* files +# for tfs-summarize-fit to score against. See README.md. +# +# PINS the --set_priors pins (default "m_pinned=1 k_pinned=1", every +# condition). When held_priors.py --hold pins some conditions +# through the table's pin columns, submit with --export=ALL,PINS= +# so the global pins do not override them. +# INIT optional params.npz of an earlier MAP of the same model (e.g. a +# held fit at nearby k and m) to start the MAP from (--init_from; +# that also skips the staged MAP). Step 2's draws start from the +# round 3 MAP this way. +# configure_extra.txt (optional file, one line): extra tfs-configure-model +# flags, e.g. the congression mixture. + +module load cuda/12.4.1 + +set -e +SEED="${SEED:-1}" +PINS="${PINS-m_pinned=1 k_pinned=1}" +EXTRA="" +[[ -f configure_extra.txt ]] && EXTRA="$(cat configure_extra.txt)" + +echo ">>> Configure (k, m held; pins: ${PINS:-none}; extra: ${EXTRA:-none}) $(date)" +tfs-configure-model ${EXTRA} \ + --allow_missing_spikes \ + --growth_df tfs_growth.csv \ + --library_config library_config.yaml \ + --theta_model hill_relative \ + --growth_shares_replicates \ + --batch_size 4096 \ + --growth_priors growth_priors_held.csv \ + --set_priors sigma_fixed=0.17 theta_log_hill_n_hyper_scale_fixed=0.5 \ + ${PINS} + +echo ">>> Fit (staged MAP) $(date)" +tfs-fit-model tfs_configure_config.yaml --seed "${SEED}" --analysis_method map \ + ${INIT:+--init_from "${INIT}"} + +echo ">>> Arrowhead Laplace $(date)" +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --laplace arrowhead --num_posterior_samples 500 --skip_growth_observations + +echo ">>> Extract and predict $(date)" +tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 +tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 + +echo ">>> Summarize against the truth $(date)" +tfs-summarize-fit . + +echo ">>> Done $(date)" diff --git a/planning/studies/m-bias-factorial/run_jackknife.srun b/planning/studies/m-bias-factorial/run_jackknife.srun new file mode 100644 index 00000000..64f08a19 --- /dev/null +++ b/planning/studies/m-bias-factorial/run_jackknife.srun @@ -0,0 +1,52 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_jk +#SBATCH --output=jk_%a.out +#SBATCH --error=jk_%a.err +#SBATCH --partition=gpulong +#SBATCH --time=00-08:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 +#SBATCH --array=1-30%10 + + +# Delete-d jackknife of a stage-1 fit (planning/deep-coverage.md, k/m +# uncertainty): array task i drops a random DROP_FRAC of the tubes +# (stratified by condition and IPTG; resample_growth.py, seed i) from the +# stage-1 growth table and refits stage 1 with the same model, in jk_NN/. +# Submit from the directory that should hold jk_*/; boot_summary.py then +# turns the fits into k and m standard errors. +# +# SRC the stage-1 growth table (absolute path) +# CONF the stage-1 fit's directory: its library_config.yaml, +# growth_priors_loose.csv and configure_extra.txt (if any) are +# copied, so every subsample fits the same model +# DROP_FRAC share of tubes dropped (default 0.2) +# TFS the tfscreen checkout (default /gpfs/home/harms/tfscreen) +# +# The fit seed is 1 for every task, so the spread is the data's. + +module load cuda/12.4.1 + +set -e +SRC="${SRC:?set SRC}" +CONF="${CONF:?set CONF}" +DROP_FRAC="${DROP_FRAC:-0.2}" +TFS="${TFS:-/gpfs/home/harms/tfscreen}" +STUDY="${TFS}/planning/studies/m-bias-factorial" +i="${SLURM_ARRAY_TASK_ID:?run as an array job}" +d="$(printf 'jk_%02d' "${i}")" + +mkdir -p "${d}" && cd "${d}" +if [[ ! -f tfs_growth.csv ]]; then + python "${STUDY}/resample_growth.py" "${SRC}" --drop_frac "${DROP_FRAC}" \ + --seed "${i}" --out tfs_growth.csv > resample.log +fi +cp "${CONF}/library_config.yaml" "${CONF}/growth_priors_loose.csv" . +[[ -f "${CONF}/configure_extra.txt" ]] && cp "${CONF}/configure_extra.txt" . +echo ">>> Stage 1 fit ${d} $(date)" +SEED=1 bash "${STUDY}/run_arm.sh" +echo ">>> Done ${d} $(date)" diff --git a/planning/studies/m-bias-factorial/run_km_draws.srun b/planning/studies/m-bias-factorial/run_km_draws.srun new file mode 100644 index 00000000..e982af2a --- /dev/null +++ b/planning/studies/m-bias-factorial/run_km_draws.srun @@ -0,0 +1,40 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_kmdraw +#SBATCH --output=run_km_draws.out +#SBATCH --error=run_km_draws.err +#SBATCH --partition=gpulong +#SBATCH --time=00-08:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# Step 2 of planning/deep-coverage.md, part 1: the arrowhead Laplace of a +# stage-1 (no-doubles) fit, then km_draws.py, which writes one directory per +# joint draw of every condition's k and m (README "Step 2"). Run from the +# directory that will hold km_draws/. +# +# STAGE1 the stage-1 fit's directory (default m_subsets/no_doubles) +# DRAWS number of draws (default 10) +# TFS the tfscreen checkout (default /gpfs/home/harms/tfscreen) + +module load cuda/12.4.1 + +set -e +STAGE1="${STAGE1:-m_subsets/no_doubles}" +DRAWS="${DRAWS:-10}" +TFS="${TFS:-/gpfs/home/harms/tfscreen}" + +if [[ ! -f "${STAGE1}/tfs_posterior.h5" ]]; then + echo ">>> Stage 1 Laplace $(date)" + (cd "${STAGE1}" && tfs-sample-posterior tfs_configure_config.yaml \ + tfs_fit_model_checkpoint.pkl --laplace arrowhead \ + --num_posterior_samples 1000 --skip_growth_observations) +fi + +echo ">>> Draws $(date)" +python "${TFS}/planning/studies/m-bias-factorial/km_draws.py" "${STAGE1}" \ + --num_draws "${DRAWS}" --out_dir km_draws +echo ">>> Done $(date)" diff --git a/planning/studies/m-bias-factorial/run_point.srun b/planning/studies/m-bias-factorial/run_point.srun new file mode 100644 index 00000000..0ef303d3 --- /dev/null +++ b/planning/studies/m-bias-factorial/run_point.srun @@ -0,0 +1,30 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_point +#SBATCH --output=run_point.out +#SBATCH --error=run_point.err +#SBATCH --partition=gpulong +#SBATCH --time=00-08:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# Finish a held fit whose arrowhead Laplace refuses the model (a congression +# mixture couples genotypes through their co-resident partners): the MAP +# point as a one-draw posterior, then predict and summarize, as run_held.srun +# would. X points only (no intervals). Submit from the fit's directory. + +module load cuda/12.4.1 + +set -e +echo ">>> MAP point $(date)" +tfs-sample-posterior tfs_configure_config.yaml tfs_fit_model_checkpoint.pkl \ + --laplace point --skip_growth_observations +echo ">>> Extract and predict $(date)" +tfs-extract-params tfs_configure_config.yaml tfs_posterior.h5 +tfs-predict-theta tfs_configure_config.yaml tfs_posterior.h5 +echo ">>> Summarize against the truth $(date)" +tfs-summarize-fit . +echo ">>> Done $(date)" diff --git a/planning/studies/m-bias-factorial/run_pool.srun b/planning/studies/m-bias-factorial/run_pool.srun new file mode 100644 index 00000000..cce54c70 --- /dev/null +++ b/planning/studies/m-bias-factorial/run_pool.srun @@ -0,0 +1,24 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_pool +#SBATCH --output=run_pool.out +#SBATCH --error=run_pool.err +#SBATCH --partition=gpulong +#SBATCH --time=00-04:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# Step 2 of planning/deep-coverage.md, part 3: pool the draws' X posteriors +# into one equal-weight mixture (tfs-compare-runs' aggregate), run from +# km_draws/ once every draw_*/run.out ends with ">>> Done". Writes +# pooled_aggregate.csv (the pooled quantiles) and pooled.csv (the draws' +# disagreement per genotype). + +set -e +ls draw_*/tfs_pred_theta.csv > pooled_inputs.txt +echo ">>> Pool $(wc -l < pooled_inputs.txt) draws $(date)" +tfs-compare-runs pooled_inputs.txt --out_prefix pooled +echo ">>> Done $(date)" diff --git a/planning/studies/m-bias-factorial/run_two_stage.srun b/planning/studies/m-bias-factorial/run_two_stage.srun new file mode 100644 index 00000000..2a85a726 --- /dev/null +++ b/planning/studies/m-bias-factorial/run_two_stage.srun @@ -0,0 +1,88 @@ +#!/bin/bash -l +#SBATCH --account=harmslab +#SBATCH --job-name=tfs_2stage +#SBATCH --output=run_two_stage.out +#SBATCH --error=run_two_stage.err +#SBATCH --partition=gpulong +#SBATCH --time=02-00:00:00 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --gpus=1 +#SBATCH --mem=0 +#SBATCH --ntasks-per-node=1 + +# The two-stage fit of one full-size simulation (planning/studies/ +# m-bias-factorial, README "Second seed"), run from a directory made by +# full-size-sim/make_sim_config.py: +# +# 1. simulate and process (skipped when tfs_growth.csv exists), +# 2. stage 1: wt, spikes and singles only (make_subsets.py no_doubles, +# run_arm.sh: configure, staged MAP, extract), plus its arrowhead +# Laplace for step 2's k/m draws, +# 3. stage 2: the full library with every condition's k and m pinned at +# stage 1's MAP (held_priors.py, run_held.srun's chain). +# +# Every step is skipped when its output exists, so a resubmit resumes. +# +# SEED the simulation and fit seed (default 1; match make_sim_config's) +# TFS the tfscreen checkout (default /gpfs/home/harms/tfscreen) +# STAGES 2 (default) runs both stages; 1 stops after stage 1 and its +# Laplace (enough to read stage 1's k and m) + +module load cuda/12.4.1 + +set -e +SEED="${SEED:-1}" +TFS="${TFS:-/gpfs/home/harms/tfscreen}" +STAGES="${STAGES:-2}" +STUDY="${TFS}/planning/studies/m-bias-factorial" + +if [[ -f tfs_growth.csv ]]; then + echo ">>> Simulate and process: tfs_growth.csv exists, skipped" +else + if [[ ! -f tfs_sim_tubes.csv ]]; then + echo ">>> Simulate $(date)" + tfs-simulate simulate_config.yaml --out_prefix tfs_sim --seed "${SEED}" + fi + echo ">>> Process counts $(date)" + tfs-process-counts tfs_sim_tubes.csv tfs_sim_counts \ + --od600_file tfs_sim_tube_od600.csv \ + --od600_calibration_file tfs_sim_od600_calibration.yaml \ + --tube_volume_mL 5 \ + --out_prefix tfs_growth --no_verbose +fi + +S1=stage1/no_doubles +if [[ ! -f ${S1}/tfs_growth.csv ]]; then + echo ">>> Stage 1 subset $(date)" + python "${STUDY}/make_subsets.py" . --out_dir stage1 --arms no_doubles +fi +if [[ ! -f ${S1}/tfs_params_growth_m.csv ]]; then + echo ">>> Stage 1 fit $(date)" + (cd "${S1}" && bash "${STUDY}/run_arm.sh") +fi +if [[ ! -f ${S1}/tfs_posterior.h5 ]]; then + echo ">>> Stage 1 Laplace $(date)" + (cd "${S1}" && tfs-sample-posterior tfs_configure_config.yaml \ + tfs_fit_model_checkpoint.pkl --laplace arrowhead \ + --num_posterior_samples 1000 --skip_growth_observations \ + > laplace.log 2>&1) +fi + +if [[ "${STAGES}" == "1" ]]; then + echo ">>> Stage 2: skipped (STAGES=1)" +elif [[ -f stage2/summary/tfs_summarize_fit_summary.json ]]; then + echo ">>> Stage 2: done, skipped" +else + echo ">>> Stage 2 setup $(date)" + mkdir -p stage2 + (cd stage2 \ + && python "${STUDY}/held_priors.py" "../${S1}" --out growth_priors_held.csv \ + && ln -sf ../tfs_growth.csv . \ + && for f in ../tfs_sim_*; do ln -sf "${f}" .; done \ + && cp ../library_config.yaml .) + echo ">>> Stage 2 fit (every condition pinned) $(date)" + (cd stage2 && SEED="${SEED}" bash "${STUDY}/run_held.srun" > run.out 2> run.err) +fi + +echo ">>> Done $(date)" diff --git a/planning/studies/m-bias-factorial/score_arms.py b/planning/studies/m-bias-factorial/score_arms.py new file mode 100644 index 00000000..b43123ed --- /dev/null +++ b/planning/studies/m-bias-factorial/score_arms.py @@ -0,0 +1,65 @@ +""" +Score the factorial arms: k and m against the truth on the X scale. + + python score_arms.py m_bias/*_s* + python score_arms.py no_doubles deep_doubles shallow_doubles + +Truth on the fit's X scale (theta: hill_relative gauges wt to X = 1 at the +low and 0 at the high gauge concentration): m_X = m (theta_wt(c_lo) - +theta_wt(c_hi)), k_X = b + m theta_wt(c_hi), with wt's true Hill curve. Also +the median dk_geno error over genotypes, when tfs_params_dk_geno.csv exists +(a MAP checkpoint does not give it). Writes factorial_scores.csv. +""" + +import os +import sys + +import numpy as np +import pandas as pd +import yaml + + +def wt_theta(p, conc): + lo, hi = p["theta_low"], p["theta_high"] + if conc <= 0: + return lo + occ = 1.0 / (1.0 + np.exp(-p["hill_n"] * (np.log(conc) - p["log_hill_K"]))) + return lo + (hi - lo) * occ + + +rows = [] +for d in sys.argv[1:]: + if not os.path.exists(os.path.join(d, "tfs_params_growth_m.csv")): + print("skip (not done):", d) + continue + cfg = yaml.safe_load(open(os.path.join(d, "tfs_configure_config.yaml"))) + gauge = None + for key in ("theta_gauge_conc",): + gauge = cfg.get("components", {}).get(key) or cfg.get(key) + c_lo, c_hi = (float(x) for x in gauge) + par = pd.read_csv(os.path.join(d, "tfs_sim_parameters.csv")) + wt = par[par.genotype == "wt"].iloc[0] + t_lo, t_hi = wt_theta(wt, c_lo), wt_theta(wt, c_hi) + truth = pd.read_csv(os.path.join(d, "tfs_sim_growth_parameters.csv")).set_index("condition_rep") + k = pd.read_csv(os.path.join(d, "tfs_params_growth_k.csv")).groupby("condition_rep")["q0.5"].mean() + m = pd.read_csv(os.path.join(d, "tfs_params_growth_m.csv")).groupby("condition_rep")["q0.5"].mean() + dk_file = os.path.join(d, "tfs_params_dk_geno.csv") + dk_bias = np.nan + if os.path.exists(dk_file): + dk = pd.read_csv(dk_file)[["genotype", "q0.5"]].merge(par[["genotype", "dk_geno"]], on="genotype") + dk_bias = float(np.median(dk["q0.5"] - dk["dk_geno"])) + arm = os.path.basename(os.path.normpath(d)) + for c in ("kanR+kan", "pheS+4CP"): + b, mm = truth.loc[c, "growth_k"], truth.loc[c, "growth_m"] + m_x, k_x = mm * (t_lo - t_hi), b + mm * t_hi + name, _, seed = arm.rpartition("_s") + if not seed.isdigit(): + name, seed = arm, "" + rows.append(dict(arm=name, seed=seed, + condition=c, k_true=k_x, k_fit=k[c], m_true=m_x, m_fit=m[c], + m_ratio=m[c] / m_x, dk_bias=dk_bias)) +df = pd.DataFrame(rows) +df.to_csv("factorial_scores.csv", index=False) +print(df.round(4).to_string(index=False)) +print() +print(df.groupby(["arm", "condition"])[["m_ratio", "dk_bias"]].mean().round(3).to_string()) diff --git a/planning/studies/m-bias-factorial/score_x.py b/planning/studies/m-bias-factorial/score_x.py new file mode 100644 index 00000000..0b79be37 --- /dev/null +++ b/planning/studies/m-bias-factorial/score_x.py @@ -0,0 +1,79 @@ +""" +X against the simulation's truth by read depth (the tables in README.md). + + python score_x.py depth.csv held_s2 km_draws/pooled_aggregate.csv \ + --truth held_s2/summary/tfs_summarize_theta_corr_test.csv + +Each input is a run directory (its summary/tfs_summarize_theta_corr_test.csv, +which carries the truth as `ref`) or a quantile table (q0.025, q0.5, q0.975 +by genotype and titrant_conc, such as tfs-compare-runs' aggregate), scored +against --truth. Doubles only unless --all. Per depth bin: r, the slope of +truth on fit, RMSE, 95% coverage, median 95% width, and the mean error at +the lowest and highest concentration. +""" + +import argparse +import os + +import numpy as np +import pandas as pd + +BINS = [0, 1e2, 1e3, 1e4, 1e5, 1e13] +LABELS = ["<=1e2", "1e2-1e3", "1e3-1e4", "1e4-1e5", ">1e5"] +COLS = ["genotype", "titrant_conc", "q0.025", "q0.5", "q0.975"] + + +def load(path, truth): + if os.path.isdir(path): + return pd.read_csv(os.path.join(path, "summary", + "tfs_summarize_theta_corr_test.csv"), + usecols=COLS + ["ref"]) + if truth is None: + raise SystemExit(f"{path} is a table; give --truth for its reference") + df = pd.read_csv(path, usecols=COLS) + return df.merge(truth, on=["genotype", "titrant_conc"]) + + +def score(df, depth, doubles_only=True): + g = df["genotype"].astype(str) + df = df[g != "wt"] + if doubles_only: + df = df[df["genotype"].astype(str).str.count("/") == 1] + df = df.assign(bin=pd.cut(df["genotype"].map(depth), BINS, labels=LABELS)) + lo, hi = df["titrant_conc"].min(), df["titrant_conc"].max() + rows = [] + for b, d in df.groupby("bin", observed=True): + x, y = d["q0.5"].to_numpy(), d["ref"].to_numpy() + err = d["q0.5"] - d["ref"] + rows.append(dict( + bin=b, n_geno=d["genotype"].nunique(), r=np.corrcoef(x, y)[0, 1], + slope=np.polyfit(x, y, 1)[0], rmse=np.sqrt(np.mean((x - y) ** 2)), + cover95=np.mean((d["q0.025"] <= y) & (y <= d["q0.975"])), + width95=np.median(d["q0.975"] - d["q0.025"]), + err_lo=err[d["titrant_conc"] == lo].mean(), + err_hi=err[d["titrant_conc"] == hi].mean())) + return pd.DataFrame(rows) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("depth_csv") + ap.add_argument("inputs", nargs="+") + ap.add_argument("--truth", default=None, + help="a run's summary/tfs_summarize_theta_corr_test.csv") + ap.add_argument("--all", action="store_true", help="singles too") + args = ap.parse_args() + depth = pd.read_csv(args.depth_csv).set_index("genotype")["reads"] + truth = None + if args.truth: + truth = pd.read_csv(args.truth, + usecols=["genotype", "titrant_conc", "ref"]) + for path in args.inputs: + print(path) + print(score(load(path, truth), depth, not args.all) + .round(3).to_string(index=False)) + print() + + +if __name__ == "__main__": + main() diff --git a/planning/studies/noise-anatomy/README.md b/planning/studies/noise-anatomy/README.md new file mode 100644 index 00000000..f089783f --- /dev/null +++ b/planning/studies/noise-anatomy/README.md @@ -0,0 +1,135 @@ +# Study 0b: anatomy of read-count noise + +**Status: provisional (2026-09-26).** Run on the 2026-07-23 snapshot, whose +counts came from a FASTQ processor that assigned every unknown read to wt +and used a very stringent Q cutoff; a new processing batch is running. +Affected here: section 3 (ratios to wt) and the `__unknown__` shares, which +are wrong (unknown reads sat in wt). Expected to survive, to be re-checked: +the low-count prevalence (may improve with a looser Q cutoff), the +composition offset, and the 5-18x excess count variance (uniform read +thinning does not create overdispersion). Next analysis: the variance +multiplier by condition and IPTG column against the tube's fold-growth +since the split; it rises with growth if founder sampling dominates and is +flat if a template bottleneck does. + +## Question + +How much of the real data sits in the low-count regime, how large is the +per-tube composition offset `u_s`, and what does per-tube count noise look +like compared with Poisson? Feeds roadmap steps 4 (simulator), 6 (`u_s` +prior) and 7 (the count likelihood's dispersion model) +(`planning/analysis-roadmap.md`). + +## How to run + +```bash +python noise_anatomy.py ~/work/programming/git-clones/tfscreen/dev/real-2026-07-23 out/ +``` + +Inputs from `planning/studies/step0-data/`. About 15 s. + +## Method + +Each replicate x condition x IPTG concentration has three tubes +(timepoints). A per-genotype straight line in `t_sel` through three points +leaves one residual degree of freedom; its projection `z` on the unit +vector orthogonal to `[1, t]` carries whatever straight-line growth does not +explain. The shared composition offset is the mean of `z` over the 494 +genotypes with at least 1000 reads in every tube; its variance is corrected +for the finite number of genotypes. Count noise compares `E[z^2]` with the +Poisson expectation `sum_i v_i^2 / c_i`, on ln frequency with the shared +offset removed, using lines with at least 100 reads in every tube (below +that, requiring nonzero counts truncates the draws and ln counts are +biased). + +## Commit + +Scripts as in this directory on top of `b949095`. + +## Results (2026-09-26) + +**1. The library is overwhelmingly low-count.** 113 tubes, median 17.9M +reads each (listed genotypes); `__unknown__` 0.4-3% of reads. + +| class | median reads per tube | 0 reads | < 5 | < 20 | +|---|---|---|---|---| +| doubles (kan+ / 4CP+) | 4 / 3 | 23% / 33% | 53% / 55% | 79% / 76% | +| singles | ~5,300 | 0.5-1% | 1-3% | 1.5-4.5% | +| spiked (5 genotypes) | 29,000-56,000 | 0 | 0 | 0 | + +Doubles are about 97% of rows. At 3-5 reads, `ln(counts + pseudocount)` is +far from Gaussian and the pseudocount dominates. + +**2. The per-tube composition offset is small:** SD 0.018 ln units, shared +by every genotype in a tube (consistent with the earlier denominator review, +0.026). Correlation with the tube's `__unknown__` share: -0.34. Per-genotype +residual SD for well-measured genotypes: 0.047. + +**3. Per-tube count noise is 5-10 times Poisson at high counts.** On ln +frequency, lines with >= 100 reads: + +`E[z^2] = 0.0055 + 8.4 x Poisson` (pooled): a variance multiplier of about +8 on top of counting, plus a floor of about 0.07 ln units. By condition the +split between the two trades off (multiplier 3-5 with floors 0.07-0.15) +because each line has one residual degree of freedom; the robust statement +is that at 100-3000 reads the observed variance is 5-18 times the Poisson +variance. The ratio to wt gives the same picture (13 times Poisson at ~500 +reads). + +**4. Low counts are mostly low abundance, not crashes.** Each double's +row in a selective tube, classified by the genotype's median reads in the +no-drug tubes of the same library and replicate (its abundance before +selection): + +| reads without drug | share of selective double rows | of which < 5 reads (kan+ / 4CP+) | +|---|---|---| +| < 5 | 59% | 74% / 88% | +| 5-20 | 22% | 19% / 38% | +| 20-100 | 13% | 2% / 9% | +| >= 100 | 6% | 0% / 0.2% | + +About 88% of the low-count (< 5) double rows belong to genotypes that are +already under 5 reads without the drug. Of the 69,245 doubles with >= 20 +reads without the drug, 39% dip below 5 reads in at least one selective +tube, but only 0.1% do so in half of their selective tubes. The double +sub-library's design gives about 1.5 transformants per double +(`transform_sizes` 300,000 for ~200,000 doubles), so a double's abundance +is set early and varies widely between doubles. + +## What it means + +- **The count likelihood (roadmap step 7) is the main event, not a + refinement.** The doubles, which carry the mutant cycles, are almost all + at 0-20 reads. +- **Today's `ln_cfu_var` (binomial counting) understates the noise 5-10 + fold** for well-measured genotypes, so fits are overconfident by roughly + 2-3x in SD. This is a leading candidate for the low theta coverage seen + in simulation-based calibration being worse on real data. +- **The dispersion model must allow variance proportional to the mean** + (NB1 / quasi-Poisson, `var = phi * mu`), not only a constant-CV floor + (NB2, `var = mu + mu^2 / r`). Variance scaling with 1/reads at 5-10x + Poisson is the signature of a bottleneck upstream of the reads: fewer + template molecules than reads (PCR input), and/or few founder cells per + genotype per tube. +- **Founder sampling is a physical per-tube noise the model and simulator + both lack.** A typical double has ~4 reads at harvest, ~15 cells in the + tube, and so only a few cells when the tube was seeded from the split. + Each tube draws its own founders, so a rare genotype's starting abundance + differs from tube to tube, independently across timepoints; `ln_cfu0` + shared by all tubes of a replicate x `condition_pre` is wrong at low + abundance. The simulator seeds every tube deterministically + (`_sim_growth`: `total_cfu0 * freq`) and samples reads once + (multinomial), so simulated noise is pure Poisson. +- **`u_s` needs only a tight prior** (SD ~0.02). +- **Most of the doubles' weak data is weak everywhere.** Low counts come + from low abundance before selection, not from genotypes crashing under + it, so a count likelihood mostly makes their uncertainty honest (and + lets the hierarchical priors carry them) rather than recovering a lost + crash signal. The crash information exists for the better-represented + doubles (39% of them dip below 5 reads somewhere) and is where the count + likelihood beats a pseudocount. The design lever is more cells per double: + deeper transformation of the double sub-library and more founders per + tube at the split (which also shrinks founder noise). +- Separating template bottleneck from founder sampling needs a technical + replicate: re-amplify and re-sequence the same extracted DNA for a few + tubes (see the roadmap). diff --git a/planning/studies/noise-anatomy/noise_anatomy.py b/planning/studies/noise-anatomy/noise_anatomy.py new file mode 100644 index 00000000..0798e2c4 --- /dev/null +++ b/planning/studies/noise-anatomy/noise_anatomy.py @@ -0,0 +1,260 @@ +""" +Step 0b of planning/analysis-roadmap.md: the anatomy of read-count noise. + +Within one replicate x condition x concentration there are three tubes +(timepoints). A per-genotype straight line in t_sel through three points +leaves one residual degree of freedom per line; its projection +z = r . v (v the unit vector orthogonal to [1, t]) carries everything the +straight-growth model does not explain in that group. That gives: + + 1. Low-count prevalence: fraction of rows at 0, < 5 and < 20 reads, by + genotype class and selection condition. + 2. The per-tube composition offset u_s (PCR, calling efficiency): a shift + shared by every genotype's ln(frequency) in a tube. Its projection is + the same for every genotype in a group, so the mean of z over many + well-measured genotypes estimates it, and + var(u) ~ mean_groups(zbar^2) - mean_groups(var(z)) / K. + Compared with the tube's __unknown__ share. + 3. The count noise model: for ln(reads_g / reads_wt) (tube total and u + cancel), E[z^2] against the Poisson expectation + sum_i v_i^2 (1/c_g,i + 1/c_wt,i). A fit E[z^2] = a + b * poisson gives + b (1 = Poisson; > 1 = effective depth smaller than the reads) and a + (a floor independent of counts: PCR jackpotting and real + genotype-by-tube differences). + +Usage: + python noise_anatomy.py DATA_DIR OUT_DIR +""" + +import sys +from pathlib import Path + +import numpy as np +import pandas as pd + +SPIKED = {"wt", "M42I", "H74A", "K84L", "M42I/H74A"} +GROUP = ["replicate", "condition_sel", "titrant_conc"] + + +def genotype_class(g): + if g in SPIKED: + return "spiked" + return "double" if "/" in g else "single" + + +def residual_projector(t): + """Unit vector orthogonal to [1, t] (3 points -> exactly one).""" + X = np.column_stack([np.ones_like(t), t]) + q, _ = np.linalg.qr(X, mode="complete") + return q[:, 2] + + +def main(data_dir, out_dir): + data_dir, out_dir = Path(data_dir), Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + samples = pd.read_parquet(data_dir / "samples.parquet") + counts = pd.read_parquet(data_dir / "counts.parquet") + genos = counts["genotype"].cat.categories + samples["unknown_share"] = 1 - samples["reads_listed"] / samples["freq_denominator"] + print(f"{len(samples)} samples, {len(genos):,} genotypes, " + f"{len(counts):,} rows") + print(f"reads per tube (listed genotypes): median " + f"{samples['reads_listed'].median():,.0f}; __unknown__ share by " + f"condition:") + print(samples.groupby("condition_sel")["unknown_share"] + .describe()[["mean", "min", "max"]].to_string()) + + # --- 1. low-count prevalence ------------------------------------------- + counts = counts.merge(samples[["sample", "condition_sel"]], on="sample") + cls = pd.Series({g: genotype_class(g) for g in genos}) + counts["class"] = counts["genotype"].map(cls).astype("category") + low = counts.groupby(["class", "condition_sel"], observed=True)["counts"].agg( + rows="size", + zero=lambda c: (c == 0).mean(), + lt5=lambda c: (c < 5).mean(), + lt20=lambda c: (c < 20).mean(), + median="median") + print("\n1. low-count prevalence (fraction of rows):") + print(low.to_string(float_format=lambda x: f"{x:.3f}")) + low.to_csv(out_dir / "low_counts.csv") + + # Matrix: tube x genotype. + mat = counts.pivot_table(index="sample", columns="genotype", + values="counts", aggfunc="sum", observed=True, + fill_value=0) + s = samples.set_index("sample").loc[mat.index] + lnf = np.log(mat.to_numpy(dtype=float) + 0.5) - \ + np.log(s["freq_denominator"].to_numpy())[:, None] + + # --- 2. composition offset u_s ------------------------------------------ + rows = [] + well = mat.columns[(mat.to_numpy() >= 1000).all(axis=0)] + wi = mat.columns.get_indexer(well) + print(f"\n2. genotypes with >= 1000 reads in every tube: {len(well)}") + for key, g in s.reset_index().groupby(GROUP): + if len(g) != 3: + continue + idx = mat.index.get_indexer(g["sample"]) + v = residual_projector(g["t_sel"].to_numpy(dtype=float)) + z = v @ lnf[idx][:, wi] # one z per well-measured genotype + uz = v @ g["unknown_share"].to_numpy() + rows.append({**dict(zip(GROUP, key)), "zbar": z.mean(), + "var_z": z.var(ddof=1), "K": len(z), "unknown_proj": uz}) + u = pd.DataFrame(rows) + var_u = (u["zbar"] ** 2).mean() - (u["var_z"] / u["K"]).mean() + print(f" groups with 3 tubes: {len(u)}") + print(f" composition offset SD (shared by all genotypes in a tube): " + f"{np.sqrt(max(var_u, 0)):.4f} ln units") + print(f" per-genotype residual SD (well-measured, projection): " + f"{np.sqrt(u['var_z'].mean()):.4f}") + r = np.corrcoef(u["zbar"], u["unknown_proj"])[0, 1] + print(f" correlation of the shared offset with the __unknown__ share " + f"(projected): {r:+.2f}") + u.to_csv(out_dir / "composition_offset.csv", index=False) + + # --- 3. count noise: ln(reads_g / reads_wt) ----------------------------- + wt = mat["wt"].to_numpy(dtype=float) + M = mat.to_numpy(dtype=float) + rec = [] + for key, g in s.reset_index().groupby(GROUP): + if len(g) != 3: + continue + idx = mat.index.get_indexer(g["sample"]) + v = residual_projector(g["t_sel"].to_numpy(dtype=float)) + c = M[idx] + ok = (c > 0).all(axis=0) + cg = c[:, ok] + y = np.log(cg) - np.log(wt[idx])[:, None] + z = v @ y + pois = (v[:, None] ** 2 * (1 / cg + 1 / wt[idx][:, None])).sum(axis=0) + rec.append(pd.DataFrame({"z2": z ** 2, "poisson": pois, + "min_reads": cg.min(axis=0), + "condition_sel": key[1]})) + rec = pd.concat(rec, ignore_index=True) + rec = rec[rec["poisson"] > 0] + bins = np.quantile(rec["poisson"], np.linspace(0, 1, 21)) + rec["bin"] = pd.cut(rec["poisson"], bins, include_lowest=True) + b = rec.groupby("bin", observed=True).agg(poisson=("poisson", "mean"), + z2=("z2", "mean"), + min_reads=("min_reads", "median"), + n=("z2", "size")) + b["ratio"] = b["z2"] / b["poisson"] + print(f"\n3. ln(reads_g/reads_wt), {len(rec):,} genotype-group lines " + f"(all 3 tubes > 0 reads)") + print(b.to_string(float_format=lambda x: f"{x:.4g}")) + A = np.column_stack([np.ones(len(b)), b["poisson"]]) + coef, *_ = np.linalg.lstsq(A, b["z2"], rcond=None) + print(f" E[z^2] = {coef[0]:.4g} + {coef[1]:.3f} * poisson " + f"(floor SD {np.sqrt(max(coef[0], 0)):.3f} ln units; " + f"Poisson multiplier {coef[1]:.2f})") + for cond, d in rec.groupby("condition_sel"): + bb = d.groupby(pd.cut(d["poisson"], bins, include_lowest=True), + observed=True).agg(p=("poisson", "mean"), z2=("z2", "mean")) + A = np.column_stack([np.ones(len(bb)), bb["p"]]) + cc, *_ = np.linalg.lstsq(A, bb["z2"], rcond=None) + print(f" {cond:10s} floor SD {np.sqrt(max(cc[0], 0)):.3f}, " + f"Poisson multiplier {cc[1]:.2f}, n={len(d):,}") + b.to_csv(out_dir / "count_noise_bins.csv") + + # --- 3b. same, on ln(frequency) with the shared offset removed ---------- + # The ratio to wt carries wt's own residual in every line. Here each + # genotype's projection has the group's shared offset (zbar over the + # well-measured genotypes) subtracted instead, and only lines with at + # least MIN_READS reads in every tube are used (below that, ln counts + # are biased and the Poisson expansion fails). + MIN_READS = 30 + zbar = u.set_index(GROUP)["zbar"] + rec = [] + for key, g in s.reset_index().groupby(GROUP): + if len(g) != 3 or key not in zbar.index: + continue + idx = mat.index.get_indexer(g["sample"]) + v = residual_projector(g["t_sel"].to_numpy(dtype=float)) + c = M[idx] + ok = (c >= MIN_READS).all(axis=0) + cg = c[:, ok] + lnD = np.log(s.loc[g["sample"], "freq_denominator"].to_numpy()) + z = v @ (np.log(cg) - lnD[:, None]) - zbar.loc[key] + pois = (v[:, None] ** 2 / cg).sum(axis=0) + rec.append(pd.DataFrame({"z2": z ** 2, "poisson": pois, + "min_reads": cg.min(axis=0), + "condition_sel": key[1], + "genotype": mat.columns[ok]})) + rec = pd.concat(rec, ignore_index=True) + rec["class"] = rec["genotype"].map(cls) + bins = np.quantile(rec["poisson"], np.linspace(0, 1, 16)) + rec["bin"] = pd.cut(rec["poisson"], bins, include_lowest=True) + b = rec.groupby("bin", observed=True).agg(poisson=("poisson", "mean"), + z2=("z2", "mean"), + min_reads=("min_reads", "median"), + n=("z2", "size")) + b["ratio"] = b["z2"] / b["poisson"] + print(f"\n3b. ln(frequency) minus the shared offset, lines with >= " + f"{MIN_READS} reads in every tube ({len(rec):,} lines)") + print(b.to_string(float_format=lambda x: f"{x:.4g}")) + # Fit only where the >= MIN_READS selection does not truncate the draws. + FIT_READS = 100 + bf = b[b["min_reads"] >= FIT_READS] + rec = rec[rec["min_reads"] >= FIT_READS] + print(f" fit on bins with median min reads >= {FIT_READS} " + f"({len(bf)} bins)") + A = np.column_stack([np.ones(len(bf)), bf["poisson"]]) + coef, *_ = np.linalg.lstsq(A, bf["z2"], rcond=None) + print(f" E[z^2] = {coef[0]:.4g} + {coef[1]:.2f} * poisson: variance " + f"multiplier (phi) {coef[1]:.2f}, floor SD " + f"{np.sqrt(max(coef[0], 0)):.3f} ln units") + for (cond), d in rec.groupby("condition_sel"): + bb = d.groupby(pd.cut(d["poisson"], bins, include_lowest=True), + observed=True).agg(p=("poisson", "mean"), z2=("z2", "mean")) + bb = bb.dropna() + A = np.column_stack([np.ones(len(bb)), bb["p"]]) + cc, *_ = np.linalg.lstsq(A, bb["z2"], rcond=None) + print(f" {cond:10s} phi {cc[1]:.2f}, floor SD " + f"{np.sqrt(max(cc[0], 0)):.3f}, n={len(d):,}") + b.to_csv(out_dir / "count_noise_bins_lnfreq.csv") + + # --- 4. why is a low count low? ----------------------------------------- + # A double that is rare everywhere carries little information in any + # tube; one that is abundant without the drug and crashes under + # selection carries its signal in the crash. Classify each low-count + # row in a selective tube by the genotype's median reads in the no-drug + # tubes of the same library (same replicate), which stand in for its + # abundance before selection. + low_rows = counts.merge(samples[["sample", "library", "replicate"]], + on="sample") + nodrug = low_rows[low_rows["condition_sel"].isin(["kanR-kan", "pheS-4CP"])] + base = (nodrug.groupby(["library", "replicate", "genotype"], observed=True) + ["counts"].median().rename("nodrug_median")) + sel = low_rows[low_rows["condition_sel"].isin(["kanR+kan", "pheS+4CP"])] + sel = sel.join(base, on=["library", "replicate", "genotype"]) + edges = [-0.1, 5, 20, 100, np.inf] + labels = ["rare (<5)", "low (5-20)", "moderate (20-100)", "abundant (>=100)"] + sel["abundance_without_drug"] = pd.cut(sel["nodrug_median"], edges, + labels=labels) + print("\n4. doubles in selective tubes: reads in the tube vs the " + "genotype's median reads without the drug") + d = sel[sel["class"] == "double"] + tab = pd.crosstab(d["abundance_without_drug"], + pd.cut(d["counts"], [-1, 0, 4, 19, np.inf], + labels=["0", "1-4", "5-19", ">=20"]), + normalize="all") + print(" fraction of all double rows in selective tubes:") + print(tab.round(3).to_string()) + by_cond = (d.assign(low=d["counts"] < 5) + .groupby(["condition_sel", "abundance_without_drug"], + observed=True)["low"].agg(["mean", "size"])) + print(" fraction below 5 reads, by condition and abundance without drug:") + print(by_cond.round(3).to_string()) + g = (d.assign(low=d["counts"] < 5) + .groupby(["library", "genotype"], observed=True) + .agg(nodrug=("nodrug_median", "median"), frac_low=("low", "mean"))) + g = g[g["nodrug"] >= 20] + print(f" doubles with >= 20 median reads without the drug: {len(g):,}; " + f"of these, share with some selective tube below 5 reads: " + f"{(g['frac_low'] > 0).mean():.3f}; with at least half below 5: " + f"{(g['frac_low'] >= 0.5).mean():.3f}") + + +if __name__ == "__main__": + main(*sys.argv[1:3]) diff --git a/planning/studies/od600-population/README.md b/planning/studies/od600-population/README.md new file mode 100644 index 00000000..0d6ed610 --- /dev/null +++ b/planning/studies/od600-population/README.md @@ -0,0 +1,72 @@ +# Study 0c: what per-tube OD600 says about each tube's total + +**Status: first pass (2026-09-26)** on the two sequenced replicates' +per-tube OD600; waiting on the third bioreplicate, the repeated OD600 runs +and the presplit OD600 and dilution. + +## Question + +How do bioreplicates differ (level vs slope), how far do tubes scatter +about a smooth curve compared with the reader's own noise, which tubes are +near or below the detection threshold, and how fast does the total grow in +each condition? Feeds roadmap steps 2, 4 and 6 and open item D9 (the +population curve form) (`planning/analysis-roadmap.md`). + +## How to run + +```bash +python od600_anatomy.py ~/Desktop/2026-07-23_onward/combined_fit_sample_df.csv \ + ~/Desktop/tfscreen-notebooks/od600-to-cfu/od600_to_cfu.yaml out/ +``` + +## Method + +Per-tube OD600 through the lab calibration to ln(cfu); reading noise from +the calibration's 2% OD600 SD alone (the calibration curve's own error is +shared by every tube and left out). Nested least-squares models over the +118 sequenced tubes (2 replicates x 20 conditions x 3 timepoints, a few +missing): one curve per condition (a line in `t_sel`), plus a replicate +level, plus replicate x condition levels, plus replicate-specific slopes. + +## Commit + +Scripts as in this directory on top of `b949095`. + +## Results (2026-09-26) + +- **Replicates differ by a level:** replicate 2 reads +0.24 ln units above + replicate 1 (SE 0.03; F = 52, p = 3e-10). Separate levels per condition + add nothing (p = 0.14), consistent with one split density per replicate. +- **Replicate-specific slopes are significant** (F = 4.2, p = 7e-5, residual + SD 0.167 -> 0.115). This contradicts slopes being shared across + bioreplicates for these two replicates; with three tubes per line it + could also be structured noise. +- **Tube scatter dwarfs reading noise:** 0.176 ln units about a shared + curve plus replicate level, against 0.031 from OD600 reading noise. The + excess, 0.17, is tube-to-tube (growth or handling). It does not grow + with time (0.14, 0.11, 0.18 by tercile of `t_sel`), so it is not + `sigma * t` tube growth noise alone. +- **No tube is below the threshold** (0.096). The lowest are kan+ at 0 mM + (1.5x threshold), kan- at 1 mM and kan+ at 0.001 mM (1.6-1.7x). +- **Population growth per condition** (per min, shared across replicates + with a replicate level): kan+ rises from -0.001 +/- 0.006 at 0 mM to + about 0.011 at >= 0.01 mM; 4CP+ falls from 0.020 at 0 mM to 0.014-0.015 + at >= 0.01 mM; no drug 0.016-0.030. Standard errors are 0.003-0.007 per + condition: three tubes per replicate over 45-120 min. + +## What it means + +- **Per-tube OD600 pins condition growth rates only to +/- 0.003-0.007 per + min,** comparable to `m` (~0.01-0.025). The absolute scale (`k_c`, `m_c`; + roadmap C4) cannot rest on these tubes alone; the earlier smoothed total + drew on repeated OD600 runs, and those data (and the third bioreplicate) + are the next input for this study. +- **The population model needs a replicate level** and probably + per-replicate slope deviations (D9), and the OD600 likelihood a per-tube + scatter term of about 0.17 ln units that is not proportional to time. +- **The sequenced tubes were chosen to be visible;** strong selection + shows up as low but readable OD600 (kan+ 0 mM barely grows), not as + censored tubes, in this dataset. +- Missing for the rest of 0c: OD600 for the third (unsequenced) + bioreplicate and the repeated OD600 runs behind `fit_lncfu`, and the + presplit OD600 and dilution factor (the known start). diff --git a/planning/studies/od600-population/od600_anatomy.py b/planning/studies/od600-population/od600_anatomy.py new file mode 100644 index 00000000..00c6f35e --- /dev/null +++ b/planning/studies/od600-population/od600_anatomy.py @@ -0,0 +1,138 @@ +""" +Step 0c of planning/analysis-roadmap.md: what the per-tube OD600 readings +say about each tube's total population. + +Questions: + 1. How much of the difference between bioreplicates is a level (shared + split density) and how much a slope? + 2. How far do tubes scatter about a smooth curve, against what the + reader's own noise (2% of OD600, through the calibration) predicts? + The excess is tube-to-tube growth noise (or protocol noise). + 3. How close are tubes to the detection threshold, by condition? + 4. The population growth rate per condition (does strong selection + stall or shrink the total?). + +Usage: + python od600_anatomy.py SAMPLE_DF CALIBRATION_YAML OUT_DIR + +SAMPLE_DF has one row per sequenced tube with od600, rep, library, +condition_sel, titrant_conc, t_sel (combined_fit_sample_df.csv). +""" + +import sys +from pathlib import Path + +import numpy as np +import pandas as pd +import statsmodels.formula.api as smf +import yaml +from matplotlib import pyplot as plt + + +def od_to_ln_cfu(od, cal): + """ln(cfu/mL) and the SD of ln(cfu/mL) from OD600 reading noise alone + (the calibration curve's own error is excluded: it is shared by every + tube, roadmap C11).""" + cfu = cal["A_CFU"] + cal["B_CFU"] * od + cal["C_CFU"] * od ** 2 + dcfu_dod = cal["B_CFU"] + 2 * cal["C_CFU"] * od + read_sd = np.abs(dcfu_dod) * cal["OD600_PCT_STD"] * od / cfu + return np.log(cfu), read_sd + + +def main(sample_df, cal_yaml, out_dir): + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + cal = yaml.safe_load(open(cal_yaml)) + + df = pd.read_csv(sample_df, index_col=0) + df["condition_sel"] = df["condition_sel"].str.strip() + df["cond"] = (df["condition_sel"] + "@" + df["titrant_conc"].astype(str)) + df["rep"] = df["rep"].astype(str) + df["lnN"], df["read_sd"] = od_to_ln_cfu(df["od600"].to_numpy(), cal) + df["t"] = df["t_sel"].astype(float) + + print(f"{len(df)} tubes, {df['cond'].nunique()} conditions, " + f"reps {sorted(df['rep'].unique())}") + print(f"OD600 reading noise in ln units: median " + f"{df['read_sd'].median():.4f} (range {df['read_sd'].min():.4f}-" + f"{df['read_sd'].max():.4f})") + + # --- 1 and 2: nested models ------------------------------------------- + forms = { + "shared curve": "lnN ~ C(cond) + C(cond):t", + "+ replicate level": "lnN ~ C(rep) + C(cond) + C(cond):t", + "+ replicate x cond level": "lnN ~ C(rep)*C(cond) + C(cond):t", + "+ replicate slopes": "lnN ~ C(rep)*C(cond) + C(rep):C(cond):t", + } + fits = {k: smf.ols(f, data=df).fit() for k, f in forms.items()} + print("\nmodel dof resid SD ") + for k, f in fits.items(): + print(f"{k:25s} {int(f.df_resid):4d} {np.sqrt(f.scale):.4f}") + names = list(fits) + for a, b in zip(names[:-1], names[1:]): + ft = fits[b].compare_f_test(fits[a]) + print(f"F-test {b!r} vs {a!r}: F={ft[0]:.2f} p={ft[1]:.3g} " + f"df={int(ft[2])}") + + rep_level = fits["+ replicate level"].params.filter(like="C(rep)") + print(f"\nreplicate level offset (rep 2 - rep 1): " + f"{rep_level.iloc[0]:+.4f} ln units " + f"(SE {fits['+ replicate level'].bse.filter(like='C(rep)').iloc[0]:.4f})") + + resid_sd = np.sqrt(fits["+ replicate level"].scale) + read_sd = df["read_sd"].median() + excess = np.sqrt(max(resid_sd ** 2 - read_sd ** 2, 0)) + print(f"scatter about shared curve + replicate level: {resid_sd:.4f}; " + f"reading noise {read_sd:.4f}; excess (tube noise) {excess:.4f}") + + # does the scatter grow with time (tube growth noise ~ sigma * t)? + r = fits["+ replicate level"].resid + by_t = df.assign(r2=r ** 2).groupby(pd.cut(df["t"], 3), observed=True)["r2"] + print("residual SD by t_sel tercile:") + for k, v in by_t: + print(f" {str(k):16s} n={len(v):3d} SD={np.sqrt(v.mean()):.4f}") + + # --- 3: distance from the detection threshold -------------------------- + thr = cal["OD600_MEAS_THRESHOLD"] + near = (df.groupby(["cond"])["od600"].min() + .rename("min_od600").to_frame()) + near["fold_above_threshold"] = near["min_od600"] / thr + print(f"\ndetection threshold {thr:.4f}; tubes below: " + f"{(df['od600'] < thr).sum()}") + print("lowest OD600 per condition (fold above threshold):") + print(near.sort_values("min_od600").head(10).to_string()) + + # --- 4: population growth rate per condition --------------------------- + slopes = [] + for cond, g in df.groupby("cond"): + f = smf.ols("lnN ~ C(rep) + t", data=g).fit() + slopes.append({"cond": cond, "slope_per_min": f.params["t"], + "slope_se": f.bse["t"], "n": len(g)}) + slopes = pd.DataFrame(slopes).sort_values("cond") + slopes.to_csv(out_dir / "population_slopes.csv", index=False) + print("\npopulation growth rate (per min) by condition, shared across " + "replicates with a replicate level:") + print(slopes.to_string(index=False, float_format=lambda x: f"{x:.5f}")) + + # --- figure -------------------------------------------------------------- + conds = sorted(df["cond"].unique()) + ncol = 8 + nrow = int(np.ceil(len(conds) / ncol)) + fig, axes = plt.subplots(nrow, ncol, figsize=(2.4 * ncol, 2.2 * nrow), + sharex=True, sharey=True) + for ax, cond in zip(axes.flat, conds): + g = df[df["cond"] == cond] + for rep, gg in g.groupby("rep"): + ax.errorbar(gg["t"], gg["lnN"], yerr=gg["read_sd"], fmt="o", + ms=3, label=f"rep {rep}") + ax.set_title(cond, fontsize=7) + axes.flat[0].legend(fontsize=6) + fig.supxlabel("t_sel (min)") + fig.supylabel("ln(cfu) from OD600") + fig.tight_layout() + fig.savefig(out_dir / "od600_by_condition.pdf") + df.to_csv(out_dir / "tubes.csv", index=False) + + +if __name__ == "__main__": + main(*sys.argv[1:4]) diff --git a/planning/studies/real-data-fit/README.md b/planning/studies/real-data-fit/README.md new file mode 100644 index 00000000..21e7016e --- /dev/null +++ b/planning/studies/real-data-fit/README.md @@ -0,0 +1,203 @@ +# Real-data fit of the dev-data screen + +**Question.** Does the growth-only relative fit (`theta: hill_relative`), +with the count likelihood and tube offsets, run on a full screen? What do +its curves say about the relationship between in vitro binding and in vivo +growth? + +**Decisions it fed.** +- Roadmap step 5: the full-library inference route. The answer is the MAP + plus the arrowhead Laplace; the SVI guides slid along the m·X ridge. +- Roadmap step 8: whether a simple growth-binding map exists. It does not. +- The new options added along the way: `--init_from`, `sigma_fixed`, the + held Hill population SDs, per-condition m prior scales, and the block + and arrowhead Laplace. + +The results and their reasoning are in `planning/analysis-roadmap-summary.md`. +The full lab notebook, with every run including the dead ends, is +`planning/dev-data/README.md` and `planning/dev-data/real_fit/README.md`. +Those are gitignored, because they sit with the data. + +## Layout: tracked scripts, untracked data + +The lab's data never enter the repository (roadmap C10). They live in +`planning/dev-data/` (gitignored), and these scripts expect that layout: + +``` +planning/dev-data/ + counts/ tfs-process-fastq output, one dir per tube + counts/run_config.yaml processing library config (3 spikes; see below) + sample_df.csv, all_reps_screen_od600.xlsx, 2025-10-02_control-experiments.xlsx + binding/ anisotropy tables + od600-to-cfu/ calibration spreadsheets + processed/ written by processing/prep_dev_data.py + real_fit/inputs/ growth.csv.gz, library_config.yaml, + wt_control_rate_summary.csv, init_*.npz + real_fit// one directory per fit +``` + +In `planning/dev-data/` the scripts are symlinks to the copies here, so they +run in place: `prep_dev_data.py` from `dev-data/`, the `fit/` and +`analysis/` scripts from `real_fit/`, as `../