From 7619bf98eda1548ae9ea4d3e78cb3b167cd30724 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:30:08 -0700 Subject: [PATCH 01/20] Upgrade isolated Level 0 package dependencies --- level_0_baseline/pyproject.toml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/level_0_baseline/pyproject.toml b/level_0_baseline/pyproject.toml index 5840ab6..23e3440 100644 --- a/level_0_baseline/pyproject.toml +++ b/level_0_baseline/pyproject.toml @@ -4,14 +4,20 @@ build-backend = "setuptools.build_meta" [project] name = "nanogpt-level0-baseline" -version = "0.1.0" -description = "Self-contained Level 0 nanoGPT baseline for AdamW and Muon" +version = "0.2.0" +description = "Self-contained GPT-2-BPE Level 0 nanoGPT baseline for AdamW and Muon" requires-python = ">=3.10" -dependencies = ["torch>=2.2", "numpy>=1.24", "pandas>=2.0", "matplotlib>=3.7", "pyyaml>=6.0"] +dependencies = [ + "torch>=2.2", + "numpy>=1.24", + "pandas>=2.0", + "matplotlib>=3.7", + "pyyaml>=6.0", +] [project.optional-dependencies] -data = ["datasets>=2.19"] -analysis = ["weightwatcher>=0.7.5", "jupyter>=1.0"] +data = ["datasets>=2.19", "tiktoken>=0.7"] +analysis = ["weightwatcher>=0.7.5", "jupyter>=1.0", "nbconvert>=7.0"] test = ["pytest>=8.0"] [project.scripts] From d3e7b89e5385dbcda5b69b137006fddd6203561e Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:30:23 -0700 Subject: [PATCH 02/20] Configure realistic GPT-2-BPE Level 0 baseline --- level_0_baseline/configs/level0.yaml | 29 +++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/level_0_baseline/configs/level0.yaml b/level_0_baseline/configs/level0.yaml index 5d385c9..19459e4 100644 --- a/level_0_baseline/configs/level0.yaml +++ b/level_0_baseline/configs/level0.yaml @@ -1,17 +1,30 @@ +data: + dataset_name: HuggingFaceFW/fineweb-edu + dataset_config: sample-10BT + dataset_split: train + dataset_revision: 593b3a867298afb8ce42625a270ef20ddcad28f9 + tokenizer: gpt2 + dtype: uint16 + train_tokens: 16000000 + val_tokens: 1000000 + test_tokens: 1000000 model: - vocab_size: 256 + vocab_size: 50257 block_size: 256 - n_layer: 1 - n_head: 1 - n_embd: 64 + n_layer: 4 + n_head: 4 + n_embd: 128 dropout: 0.0 bias: false + tie_weights: true training: - batch_size: 16 - grad_accum_steps: 1 + batch_size: 4 + grad_accum_steps: 8 max_steps: 2000 eval_interval: 50 eval_batches: 20 + eval_batch_size: 4 + test_eval_batches: 40 checkpoint_interval: 250 learning_rate: 0.0006 muon_learning_rate: 0.02 @@ -21,7 +34,9 @@ training: weight_decay: 0.1 beta1: 0.9 beta2: 0.95 + epsilon: 1.0e-8 grad_clip: 1.0 + layer_lr_decay: 1.0 optimizer: adamw muon_momentum: 0.95 muon_nesterov: true @@ -30,4 +45,4 @@ training: analysis: weightwatcher: true weightwatcher_interval: 250 - randomize: true + randomize: false From 7c0ba330de9aab75dc18105ae716e7035e7104be Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:30:35 -0700 Subject: [PATCH 03/20] Update isolated Level 0 single-run launcher --- level_0_baseline/scripts/run_one.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/level_0_baseline/scripts/run_one.sh b/level_0_baseline/scripts/run_one.sh index f9dbed7..624dfa8 100644 --- a/level_0_baseline/scripts/run_one.sh +++ b/level_0_baseline/scripts/run_one.sh @@ -1,6 +1,28 @@ #!/usr/bin/env bash set -euo pipefail -ROOT="${NANOGPT_LEVEL0_ROOT:-/tmp/nanogpt-level0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + +ROOT="${NANOGPT_LEVEL0_ROOT:-/tmp/nanogpt-level0-gpt2}" OPTIMIZER="${1:-adamw}" SEED="${2:-1337}" -python -m level0_baseline.train --config configs/level0.yaml --optimizer "$OPTIMIZER" --seed "$SEED" --data-root "${NANOGPT_LEVEL0_DATA_ROOT:-$ROOT/data}" --results-root "${NANOGPT_LEVEL0_RESULTS_ROOT:-$ROOT/results}" +DEVICE="${NANOGPT_LEVEL0_DEVICE:-auto}" +DATA_ROOT="${NANOGPT_LEVEL0_DATA_ROOT:-$ROOT/data}" +RESULTS_ROOT="${NANOGPT_LEVEL0_RESULTS_ROOT:-$ROOT/results}" + +ARGS=( + --config configs/level0.yaml + --optimizer "$OPTIMIZER" + --seed "$SEED" + --device "$DEVICE" + --data-root "$DATA_ROOT" + --results-root "$RESULTS_ROOT" +) +if [[ "${NANOGPT_LEVEL0_OVERWRITE:-0}" == "1" ]]; then + ARGS+=(--overwrite) +fi + +export PYTHONUNBUFFERED=1 +python -u -m level0_baseline.train "${ARGS[@]}" From 8218d31639ddbc4f8b85493db8b0ca59f57e9d03 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:30:46 -0700 Subject: [PATCH 04/20] Default isolated multiseed runs to AdamW --- level_0_baseline/scripts/run_multiseed.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/level_0_baseline/scripts/run_multiseed.sh b/level_0_baseline/scripts/run_multiseed.sh index 27aefe0..2593861 100644 --- a/level_0_baseline/scripts/run_multiseed.sh +++ b/level_0_baseline/scripts/run_multiseed.sh @@ -1,9 +1,17 @@ #!/usr/bin/env bash set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + SEEDS="${NANOGPT_LEVEL0_SEEDS:-1337,2027,4099}" -for optimizer in adamw muon; do - IFS=',' read -ra xs <<< "$SEEDS" - for seed in "${xs[@]}"; do +OPTIMIZERS="${NANOGPT_LEVEL0_OPTIMIZERS:-adamw}" +IFS=',' read -r -a SEED_ARRAY <<< "$SEEDS" +IFS=',' read -r -a OPTIMIZER_ARRAY <<< "$OPTIMIZERS" + +for optimizer in "${OPTIMIZER_ARRAY[@]}"; do + for seed in "${SEED_ARRAY[@]}"; do ./scripts/run_one.sh "$optimizer" "$seed" done done From 2610904bacbe07ee8cfadca19ac3313b8f9a22ea Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:31:00 -0700 Subject: [PATCH 05/20] Version corrected isolated Level 0 baseline --- level_0_baseline/src/level0_baseline/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/level_0_baseline/src/level0_baseline/__init__.py b/level_0_baseline/src/level0_baseline/__init__.py index 8fe33d4..2c6e23e 100644 --- a/level_0_baseline/src/level0_baseline/__init__.py +++ b/level_0_baseline/src/level0_baseline/__init__.py @@ -1,2 +1,3 @@ -"""Self-contained Level 0 nanoGPT baseline.""" -__version__ = "0.1.0" +"""Self-contained realistic Level 0 nanoGPT baseline.""" + +__version__ = "0.2.0" From 2ac6ff75877e2a9881c0bf40c96e8c6f4075a412 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:31:38 -0700 Subject: [PATCH 06/20] Validate corrected isolated Level 0 configuration --- .../src/level0_baseline/config.py | 155 +++++++++++++++++- 1 file changed, 148 insertions(+), 7 deletions(-) diff --git a/level_0_baseline/src/level0_baseline/config.py b/level_0_baseline/src/level0_baseline/config.py index 3d19743..f91122c 100644 --- a/level_0_baseline/src/level0_baseline/config.py +++ b/level_0_baseline/src/level0_baseline/config.py @@ -1,31 +1,172 @@ from __future__ import annotations + +import copy import os from pathlib import Path +from typing import Any, Callable + import yaml -DEFAULT_ROOT = Path("/tmp/nanogpt-level0") +DEFAULT_ROOT = Path("/tmp/nanogpt-level0-gpt2") + + +def _parse_bool(value: str) -> bool: + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"invalid boolean value: {value!r}") + def roots() -> dict[str, Path]: root = Path(os.getenv("NANOGPT_LEVEL0_ROOT", DEFAULT_ROOT)) return { "root": root, "data": Path(os.getenv("NANOGPT_LEVEL0_DATA_ROOT", root / "data")), - "results": Path(os.getenv("NANOGPT_LEVEL0_RESULTS_ROOT", root / "results")), + "results": Path( + os.getenv("NANOGPT_LEVEL0_RESULTS_ROOT", root / "results") + ), "cache": Path(os.getenv("NANOGPT_LEVEL0_CACHE_ROOT", root / "cache")), } -def load_config(path: str | Path) -> dict: - with open(path, "r", encoding="utf-8") as f: - cfg = yaml.safe_load(f) - env_map = { + +def _set_nested(cfg: dict[str, Any], section: str, key: str, value: Any) -> None: + if section not in cfg or not isinstance(cfg[section], dict): + raise ValueError(f"configuration is missing section {section!r}") + cfg[section][key] = value + + +def load_config(path: str | Path) -> dict[str, Any]: + with open(path, "r", encoding="utf-8") as handle: + loaded = yaml.safe_load(handle) + if not isinstance(loaded, dict): + raise ValueError("configuration root must be a mapping") + cfg: dict[str, Any] = copy.deepcopy(loaded) + + env_map: dict[str, tuple[str, str, Callable[[str], Any]]] = { "NANOGPT_LEVEL0_SEED": ("training", "seed", int), "NANOGPT_LEVEL0_OPTIMIZER": ("training", "optimizer", str), "NANOGPT_LEVEL0_MAX_STEPS": ("training", "max_steps", int), "NANOGPT_LEVEL0_BATCH_SIZE": ("training", "batch_size", int), + "NANOGPT_LEVEL0_GRAD_ACCUM_STEPS": ( + "training", + "grad_accum_steps", + int, + ), "NANOGPT_LEVEL0_LR": ("training", "learning_rate", float), + "NANOGPT_LEVEL0_MIN_LR": ("training", "min_lr", float), + "NANOGPT_LEVEL0_WARMUP_STEPS": ("training", "warmup_steps", int), + "NANOGPT_LEVEL0_WEIGHT_DECAY": ("training", "weight_decay", float), "NANOGPT_LEVEL0_EVAL_INTERVAL": ("training", "eval_interval", int), + "NANOGPT_LEVEL0_EVAL_BATCHES": ("training", "eval_batches", int), + "NANOGPT_LEVEL0_EVAL_BATCH_SIZE": ( + "training", + "eval_batch_size", + int, + ), + "NANOGPT_LEVEL0_TEST_EVAL_BATCHES": ( + "training", + "test_eval_batches", + int, + ), + "NANOGPT_LEVEL0_CHECKPOINT_INTERVAL": ( + "training", + "checkpoint_interval", + int, + ), + "NANOGPT_LEVEL0_LAYER_LR_DECAY": ( + "training", + "layer_lr_decay", + float, + ), + "NANOGPT_LEVEL0_N_LAYER": ("model", "n_layer", int), + "NANOGPT_LEVEL0_N_HEAD": ("model", "n_head", int), + "NANOGPT_LEVEL0_N_EMBD": ("model", "n_embd", int), + "NANOGPT_LEVEL0_BLOCK_SIZE": ("model", "block_size", int), + "NANOGPT_LEVEL0_WEIGHTWATCHER": ( + "analysis", + "weightwatcher", + _parse_bool, + ), + "NANOGPT_LEVEL0_WW_INTERVAL": ( + "analysis", + "weightwatcher_interval", + int, + ), } for name, (section, key, cast) in env_map.items(): if name in os.environ: - cfg[section][key] = cast(os.environ[name]) + _set_nested(cfg, section, key, cast(os.environ[name])) + + validate_config(cfg) return cfg + + +def validate_config(cfg: dict[str, Any]) -> None: + for section in ("data", "model", "training", "analysis"): + if section not in cfg or not isinstance(cfg[section], dict): + raise ValueError(f"configuration is missing section {section!r}") + + data = cfg["data"] + model = cfg["model"] + training = cfg["training"] + analysis = cfg["analysis"] + + if data.get("tokenizer") != "gpt2": + raise ValueError("the corrected Level 0 baseline requires tokenizer: gpt2") + if data.get("dtype") != "uint16": + raise ValueError("the corrected Level 0 baseline requires dtype: uint16") + for key in ("train_tokens", "val_tokens", "test_tokens"): + if int(data.get(key, 0)) <= 0: + raise ValueError(f"data.{key} must be positive") + + for key in ("vocab_size", "block_size", "n_layer", "n_head", "n_embd"): + if int(model.get(key, 0)) <= 0: + raise ValueError(f"model.{key} must be positive") + if int(model["n_embd"]) % int(model["n_head"]) != 0: + raise ValueError("model.n_embd must be divisible by model.n_head") + if int(model["vocab_size"]) < 50257: + raise ValueError("model.vocab_size must cover the GPT-2 tokenizer") + dropout = float(model.get("dropout", 0.0)) + if not 0.0 <= dropout < 1.0: + raise ValueError("model.dropout must satisfy 0 <= dropout < 1") + + positive_ints = ( + "batch_size", + "grad_accum_steps", + "max_steps", + "eval_interval", + "eval_batches", + "eval_batch_size", + "test_eval_batches", + "checkpoint_interval", + ) + for key in positive_ints: + if int(training.get(key, 0)) <= 0: + raise ValueError(f"training.{key} must be positive") + if int(training.get("warmup_steps", 0)) < 0: + raise ValueError("training.warmup_steps must be nonnegative") + if int(training["warmup_steps"]) >= int(training["max_steps"]): + raise ValueError("training.warmup_steps must be smaller than max_steps") + if float(training.get("learning_rate", 0.0)) <= 0: + raise ValueError("training.learning_rate must be positive") + if not 0 < float(training.get("min_lr", 0.0)) <= float( + training["learning_rate"] + ): + raise ValueError("training.min_lr must be in (0, learning_rate]") + if float(training.get("weight_decay", -1.0)) < 0: + raise ValueError("training.weight_decay must be nonnegative") + if float(training.get("epsilon", 0.0)) <= 0: + raise ValueError("training.epsilon must be positive") + if float(training.get("grad_clip", -1.0)) < 0: + raise ValueError("training.grad_clip must be nonnegative") + layer_lr_decay = float(training.get("layer_lr_decay", 1.0)) + if not 0 < layer_lr_decay <= 1: + raise ValueError("training.layer_lr_decay must be in (0, 1]") + if str(training.get("optimizer", "")).lower() not in {"adamw", "muon"}: + raise ValueError("training.optimizer must be adamw or muon") + + interval = int(analysis.get("weightwatcher_interval", 0)) + if bool(analysis.get("weightwatcher", False)) and interval <= 0: + raise ValueError("analysis.weightwatcher_interval must be positive") From f980f74d826da619d046c541f739be488562061f Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:32:14 -0700 Subject: [PATCH 07/20] Replace toy byte model with realistic nanoGPT architecture --- level_0_baseline/src/level0_baseline/model.py | 143 ++++++++++++++---- 1 file changed, 113 insertions(+), 30 deletions(-) diff --git a/level_0_baseline/src/level0_baseline/model.py b/level_0_baseline/src/level0_baseline/model.py index 907ae02..73f8af6 100644 --- a/level_0_baseline/src/level0_baseline/model.py +++ b/level_0_baseline/src/level0_baseline/model.py @@ -1,45 +1,88 @@ from __future__ import annotations -from dataclasses import dataclass + import math +from dataclasses import dataclass + import torch import torch.nn as nn import torch.nn.functional as F + @dataclass class GPTConfig: - vocab_size: int = 256 + vocab_size: int = 50257 block_size: int = 256 - n_layer: int = 1 - n_head: int = 1 - n_embd: int = 64 + n_layer: int = 4 + n_head: int = 4 + n_embd: int = 128 dropout: float = 0.0 bias: bool = False + tie_weights: bool = True + class CausalSelfAttention(nn.Module): def __init__(self, cfg: GPTConfig): super().__init__() - assert cfg.n_embd % cfg.n_head == 0 - self.n_head, self.n_embd, self.dropout = cfg.n_head, cfg.n_embd, cfg.dropout + if cfg.n_embd % cfg.n_head != 0: + raise ValueError("n_embd must be divisible by n_head") + self.n_head = cfg.n_head + self.n_embd = cfg.n_embd + self.dropout = cfg.dropout self.q_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) self.k_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) self.v_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) self.out_proj = nn.Linear(cfg.n_embd, cfg.n_embd, bias=cfg.bias) self.resid_dropout = nn.Dropout(cfg.dropout) - self.register_buffer("mask", torch.tril(torch.ones(cfg.block_size, cfg.block_size)).view(1, 1, cfg.block_size, cfg.block_size)) + self.register_buffer( + "mask", + torch.tril( + torch.ones(cfg.block_size, cfg.block_size, dtype=torch.bool) + ).view(1, 1, cfg.block_size, cfg.block_size), + persistent=False, + ) def forward(self, x: torch.Tensor) -> torch.Tensor: - b, t, c = x.shape - hs = c // self.n_head - q = self.q_proj(x).view(b, t, self.n_head, hs).transpose(1, 2) - k = self.k_proj(x).view(b, t, self.n_head, hs).transpose(1, 2) - v = self.v_proj(x).view(b, t, self.n_head, hs).transpose(1, 2) - att = (q @ k.transpose(-2, -1)) / math.sqrt(hs) - att = att.masked_fill(self.mask[:, :, :t, :t] == 0, float("-inf")) - att = F.softmax(att, dim=-1) - att = F.dropout(att, p=self.dropout, training=self.training) - y = (att @ v).transpose(1, 2).contiguous().view(b, t, c) + batch_size, sequence_length, channels = x.shape + head_size = channels // self.n_head + q = self.q_proj(x).view( + batch_size, sequence_length, self.n_head, head_size + ).transpose(1, 2) + k = self.k_proj(x).view( + batch_size, sequence_length, self.n_head, head_size + ).transpose(1, 2) + v = self.v_proj(x).view( + batch_size, sequence_length, self.n_head, head_size + ).transpose(1, 2) + + if hasattr(F, "scaled_dot_product_attention"): + y = F.scaled_dot_product_attention( + q, + k, + v, + attn_mask=None, + dropout_p=self.dropout if self.training else 0.0, + is_causal=True, + ) + else: # pragma: no cover - modern supported PyTorch uses the branch above. + attention = (q @ k.transpose(-2, -1)) / math.sqrt(head_size) + attention = attention.masked_fill( + ~self.mask[:, :, :sequence_length, :sequence_length], + float("-inf"), + ) + attention = F.softmax(attention, dim=-1) + attention = F.dropout( + attention, + p=self.dropout, + training=self.training, + ) + y = attention @ v + + y = y.transpose(1, 2).contiguous().view( + batch_size, sequence_length, channels + ) return self.resid_dropout(self.out_proj(y)) + class MLP(nn.Module): def __init__(self, cfg: GPTConfig): super().__init__() @@ -48,20 +91,22 @@ def __init__(self, cfg: GPTConfig): self.dropout = nn.Dropout(cfg.dropout) def forward(self, x: torch.Tensor) -> torch.Tensor: - return self.dropout(self.proj(F.gelu(self.fc(x)))) + return self.dropout(self.proj(F.gelu(self.fc(x), approximate="tanh"))) + class Block(nn.Module): def __init__(self, cfg: GPTConfig): super().__init__() - self.ln1 = nn.LayerNorm(cfg.n_embd) + self.ln1 = nn.LayerNorm(cfg.n_embd, bias=cfg.bias) self.attn = CausalSelfAttention(cfg) - self.ln2 = nn.LayerNorm(cfg.n_embd) + self.ln2 = nn.LayerNorm(cfg.n_embd, bias=cfg.bias) self.mlp = MLP(cfg) def forward(self, x: torch.Tensor) -> torch.Tensor: x = x + self.attn(self.ln1(x)) return x + self.mlp(self.ln2(x)) + class GPT(nn.Module): def __init__(self, cfg: GPTConfig): super().__init__() @@ -70,27 +115,65 @@ def __init__(self, cfg: GPTConfig): self.position_embedding = nn.Embedding(cfg.block_size, cfg.n_embd) self.drop = nn.Dropout(cfg.dropout) self.blocks = nn.ModuleList([Block(cfg) for _ in range(cfg.n_layer)]) - self.ln_f = nn.LayerNorm(cfg.n_embd) + self.ln_f = nn.LayerNorm(cfg.n_embd, bias=cfg.bias) self.lm_head = nn.Linear(cfg.n_embd, cfg.vocab_size, bias=False) - self.lm_head.weight = self.token_embedding.weight + self.apply(self._init) + residual_std = 0.02 / math.sqrt(2 * cfg.n_layer) + for block in self.blocks: + nn.init.normal_(block.attn.out_proj.weight, mean=0.0, std=residual_std) + nn.init.normal_(block.mlp.proj.weight, mean=0.0, std=residual_std) + if cfg.tie_weights: + self.lm_head.weight = self.token_embedding.weight - def _init(self, module: nn.Module) -> None: + @staticmethod + def _init(module: nn.Module) -> None: if isinstance(module, (nn.Linear, nn.Embedding)): nn.init.normal_(module.weight, mean=0.0, std=0.02) if isinstance(module, nn.Linear) and module.bias is not None: nn.init.zeros_(module.bias) - def forward(self, idx: torch.Tensor, targets: torch.Tensor | None = None): - _, t = idx.shape - if t > self.cfg.block_size: + def forward( + self, + idx: torch.Tensor, + targets: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + _, sequence_length = idx.shape + if sequence_length > self.cfg.block_size: raise ValueError("sequence exceeds block_size") - pos = torch.arange(t, device=idx.device) - x = self.drop(self.token_embedding(idx) + self.position_embedding(pos)) + positions = torch.arange(sequence_length, device=idx.device) + x = self.drop( + self.token_embedding(idx) + self.position_embedding(positions) + ) for block in self.blocks: x = block(x) logits = self.lm_head(self.ln_f(x)) loss = None if targets is not None: - loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1)) + loss = F.cross_entropy( + logits.reshape(-1, logits.size(-1)), + targets.reshape(-1), + ) return logits, loss + + def num_parameters(self, *, exclude_position_embedding: bool = False) -> int: + count = sum(parameter.numel() for parameter in self.parameters()) + if exclude_position_embedding: + count -= self.position_embedding.weight.numel() + return count + + def spectral_layers(self) -> list[tuple[str, nn.Linear]]: + layers: list[tuple[str, nn.Linear]] = [] + for index, block in enumerate(self.blocks): + prefix = f"block_{index:02d}" + layers.extend( + [ + (f"{prefix}_W_Q", block.attn.q_proj), + (f"{prefix}_W_K", block.attn.k_proj), + (f"{prefix}_W_V", block.attn.v_proj), + (f"{prefix}_W_O", block.attn.out_proj), + (f"{prefix}_W_MLP_IN", block.mlp.fc), + (f"{prefix}_W_MLP_OUT", block.mlp.proj), + ] + ) + return layers From 0f76a670bc54e50d19f0b7c97ce4fd1e8ac4110d Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:32:52 -0700 Subject: [PATCH 08/20] Add proper AdamW groups and optional layerwise LR decay --- level_0_baseline/src/level0_baseline/optim.py | 230 +++++++++++++++--- 1 file changed, 194 insertions(+), 36 deletions(-) diff --git a/level_0_baseline/src/level0_baseline/optim.py b/level_0_baseline/src/level0_baseline/optim.py index fbc19ae..34a741c 100644 --- a/level_0_baseline/src/level0_baseline/optim.py +++ b/level_0_baseline/src/level0_baseline/optim.py @@ -1,59 +1,217 @@ from __future__ import annotations + +import re +from collections import defaultdict +from typing import Any, Iterable + import torch +_BLOCK_PATTERN = re.compile(r"^blocks\.(\d+)\.") + + @torch.no_grad() -def zeropower_via_newtonschulz5(g: torch.Tensor, steps: int = 5) -> torch.Tensor: - assert g.ndim == 2 - x = g.float() - if x.shape[0] > x.shape[1]: +def zeropower_via_newtonschulz5( + gradient: torch.Tensor, + steps: int = 5, +) -> torch.Tensor: + if gradient.ndim != 2: + raise ValueError("Muon zero-power update requires a matrix") + x = gradient.float() + transposed = x.shape[0] > x.shape[1] + if transposed: x = x.T x = x / (x.norm() + 1e-7) a, b, c = 3.4445, -4.7750, 2.0315 for _ in range(steps): - A = x @ x.T - x = a * x + (b * A + c * (A @ A)) @ x - if g.shape[0] > g.shape[1]: + gram = x @ x.T + x = a * x + (b * gram + c * (gram @ gram)) @ x + if transposed: x = x.T - return x.to(g.dtype) + return x.to(gradient.dtype) + class Muon(torch.optim.Optimizer): - def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, weight_decay=0.0): - super().__init__(params, dict(lr=lr, momentum=momentum, nesterov=nesterov, weight_decay=weight_decay)) + def __init__( + self, + params: Iterable[torch.nn.Parameter] | list[dict[str, Any]], + *, + lr: float = 0.02, + momentum: float = 0.95, + nesterov: bool = True, + weight_decay: float = 0.0, + ) -> None: + defaults = { + "lr": lr, + "momentum": momentum, + "nesterov": nesterov, + "weight_decay": weight_decay, + } + super().__init__(params, defaults) @torch.no_grad() def step(self, closure=None): loss = closure() if closure is not None else None for group in self.param_groups: - for p in group["params"]: - if p.grad is None: + for parameter in group["params"]: + if parameter.grad is None: continue - if p.ndim != 2: + if parameter.ndim != 2: raise ValueError("Muon received a non-matrix parameter") - buf = self.state[p].setdefault("momentum_buffer", torch.zeros_like(p)) - buf.mul_(group["momentum"]).add_(p.grad) - g = p.grad.add(buf, alpha=group["momentum"]) if group["nesterov"] else buf - update = zeropower_via_newtonschulz5(g) - update.mul_(max(1, p.shape[0] / p.shape[1]) ** 0.5) + momentum_buffer = self.state[parameter].setdefault( + "momentum_buffer", + torch.zeros_like(parameter), + ) + momentum_buffer.mul_(group["momentum"]).add_(parameter.grad) + update_input = ( + parameter.grad.add( + momentum_buffer, + alpha=group["momentum"], + ) + if group["nesterov"] + else momentum_buffer + ) + update = zeropower_via_newtonschulz5(update_input) + update.mul_( + max(1.0, parameter.shape[0] / parameter.shape[1]) ** 0.5 + ) if group["weight_decay"]: - p.mul_(1 - group["lr"] * group["weight_decay"]) - p.add_(update, alpha=-group["lr"]) + parameter.mul_(1 - group["lr"] * group["weight_decay"]) + parameter.add_(update, alpha=-group["lr"]) return loss -def make_optimizers(model, cfg): - t = cfg["training"] - name = t["optimizer"].lower() - decay = [p for _, p in model.named_parameters() if p.requires_grad and p.ndim >= 2] - nodecay = [p for _, p in model.named_parameters() if p.requires_grad and p.ndim < 2] - if name == "adamw": - return [torch.optim.AdamW([{"params": decay, "weight_decay": t["weight_decay"]}, {"params": nodecay, "weight_decay": 0.0}], lr=t["learning_rate"], betas=(t["beta1"], t["beta2"]))] - if name != "muon": - raise ValueError(f"unsupported optimizer: {name}") - muon, adam = [], [] - for n, p in model.named_parameters(): - if not p.requires_grad: + +def _layer_multiplier(name: str, n_layer: int, decay: float) -> float: + if decay == 1.0: + return 1.0 + match = _BLOCK_PATTERN.match(name) + if match: + block_index = int(match.group(1)) + return decay ** max(n_layer - 1 - block_index, 0) + if name.startswith(("token_embedding", "position_embedding")): + return decay**n_layer + return 1.0 + + +def _parameter_records(model, training: dict[str, Any]): + n_layer = int(model.cfg.n_layer) + layer_decay = float(training.get("layer_lr_decay", 1.0)) + for name, parameter in model.named_parameters(): + if not parameter.requires_grad: continue - if p.ndim == 2 and "embedding" not in n and "lm_head" not in n: - muon.append(p) + yield { + "name": name, + "parameter": parameter, + "decay": parameter.ndim >= 2, + "lr_multiplier": _layer_multiplier(name, n_layer, layer_decay), + } + + +def _adamw_groups(model, training: dict[str, Any]) -> list[dict[str, Any]]: + grouped: dict[tuple[bool, float], list[torch.nn.Parameter]] = defaultdict(list) + names: dict[tuple[bool, float], list[str]] = defaultdict(list) + for record in _parameter_records(model, training): + key = (bool(record["decay"]), float(record["lr_multiplier"])) + grouped[key].append(record["parameter"]) + names[key].append(str(record["name"])) + + base_lr = float(training["learning_rate"]) + groups: list[dict[str, Any]] = [] + for (use_decay, multiplier), parameters in sorted( + grouped.items(), + key=lambda item: (item[0][1], item[0][0]), + ): + groups.append( + { + "params": parameters, + "lr": base_lr * multiplier, + "initial_lr": base_lr * multiplier, + "lr_multiplier": multiplier, + "weight_decay": ( + float(training["weight_decay"]) if use_decay else 0.0 + ), + "group_name": ( + f"{'decay' if use_decay else 'no_decay'}_lr_{multiplier:.6f}" + ), + "parameter_names": names[(use_decay, multiplier)], + } + ) + return groups + + +def make_optimizers(model, config: dict[str, Any]) -> list[torch.optim.Optimizer]: + training = config["training"] + optimizer_name = str(training["optimizer"]).lower() + if optimizer_name == "adamw": + return [ + torch.optim.AdamW( + _adamw_groups(model, training), + lr=float(training["learning_rate"]), + betas=(float(training["beta1"]), float(training["beta2"])), + eps=float(training["epsilon"]), + ) + ] + if optimizer_name != "muon": + raise ValueError(f"unsupported optimizer: {optimizer_name}") + + muon_grouped: dict[float, list[torch.nn.Parameter]] = defaultdict(list) + auxiliary_grouped: dict[ + tuple[bool, float], list[torch.nn.Parameter] + ] = defaultdict(list) + for record in _parameter_records(model, training): + name = str(record["name"]) + parameter = record["parameter"] + multiplier = float(record["lr_multiplier"]) + use_muon = parameter.ndim == 2 and not name.startswith( + ("token_embedding", "position_embedding", "lm_head") + ) + if use_muon: + muon_grouped[multiplier].append(parameter) else: - adam.append(p) - return [Muon(muon, lr=t["muon_learning_rate"], momentum=t["muon_momentum"], nesterov=t["muon_nesterov"], weight_decay=t["weight_decay"]), torch.optim.AdamW(adam, lr=t["muon_aux_adamw_learning_rate"], betas=(t["beta1"], t["beta2"]), weight_decay=0.0)] + auxiliary_grouped[(bool(record["decay"]), multiplier)].append(parameter) + + muon_base_lr = float(training["muon_learning_rate"]) + muon_groups = [ + { + "params": parameters, + "lr": muon_base_lr * multiplier, + "initial_lr": muon_base_lr * multiplier, + "lr_multiplier": multiplier, + "weight_decay": float(training["weight_decay"]), + "group_name": f"muon_lr_{multiplier:.6f}", + } + for multiplier, parameters in sorted(muon_grouped.items()) + ] + auxiliary_base_lr = float(training["muon_aux_adamw_learning_rate"]) + auxiliary_groups = [ + { + "params": parameters, + "lr": auxiliary_base_lr * multiplier, + "initial_lr": auxiliary_base_lr * multiplier, + "lr_multiplier": multiplier, + "weight_decay": ( + float(training["weight_decay"]) if use_decay else 0.0 + ), + "group_name": ( + f"aux_{'decay' if use_decay else 'no_decay'}_lr_{multiplier:.6f}" + ), + } + for (use_decay, multiplier), parameters in sorted( + auxiliary_grouped.items(), + key=lambda item: (item[0][1], item[0][0]), + ) + ] + return [ + Muon( + muon_groups, + lr=muon_base_lr, + momentum=float(training["muon_momentum"]), + nesterov=bool(training["muon_nesterov"]), + weight_decay=float(training["weight_decay"]), + ), + torch.optim.AdamW( + auxiliary_groups, + lr=auxiliary_base_lr, + betas=(float(training["beta1"]), float(training["beta2"])), + eps=float(training["epsilon"]), + ), + ] From b619ea0e53d1e530d8404195be6fef00cca1b870 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:34:53 -0700 Subject: [PATCH 09/20] Replace byte preparation with pinned GPT-2-BPE data pipeline --- level_0_baseline/src/level0_baseline/data.py | 482 ++++++++++++++----- 1 file changed, 363 insertions(+), 119 deletions(-) diff --git a/level_0_baseline/src/level0_baseline/data.py b/level_0_baseline/src/level0_baseline/data.py index ed96451..219849b 100644 --- a/level_0_baseline/src/level0_baseline/data.py +++ b/level_0_baseline/src/level0_baseline/data.py @@ -1,22 +1,35 @@ from __future__ import annotations import argparse +import gc +import hashlib import json +import os import sys import threading import time from pathlib import Path +from typing import Any, Iterable, Iterator, Protocol import numpy as np -from .config import roots +from .config import load_config, roots +DATA_SCHEMA_VERSION = 2 +UINT16_MAX = np.iinfo(np.uint16).max -def encode(text: str) -> np.ndarray: - return np.frombuffer(text.encode("utf-8", errors="replace"), dtype=np.uint8) +class Tokenizer(Protocol): + name: str + n_vocab: int + eot_token: int -def _format_duration(seconds: float) -> str: + def encode_ordinary(self, text: str) -> list[int]: ... + + +def _format_duration(seconds: float | None) -> str: + if seconds is None: + return "unknown" seconds = max(0, int(seconds)) hours, remainder = divmod(seconds, 3600) minutes, secs = divmod(remainder, 60) @@ -29,41 +42,38 @@ def _format_duration(seconds: float) -> str: def _progress_message( *, - collected_bytes: int, - required_bytes: int, + collected_tokens: int, + required_tokens: int, documents: int, elapsed_seconds: float, stalled_seconds: float, ) -> str: - elapsed_seconds = max(elapsed_seconds, 1e-9) - rate = collected_bytes / elapsed_seconds - remaining = max(required_bytes - collected_bytes, 0) + elapsed_seconds = max(float(elapsed_seconds), 1e-9) + rate = collected_tokens / elapsed_seconds + remaining = max(required_tokens - collected_tokens, 0) eta = remaining / rate if rate > 0 else None - percent = 100.0 * collected_bytes / max(required_bytes, 1) - speed_mib = rate / (1024 * 1024) - eta_text = _format_duration(eta) if eta is not None else "unknown" - stall_text = _format_duration(stalled_seconds) + percent = 100.0 * collected_tokens / max(required_tokens, 1) return ( "[level0-prepare-data] progress " f"documents={documents:,} " - f"bytes={collected_bytes:,}/{required_bytes:,} " + f"tokens={collected_tokens:,}/{required_tokens:,} " f"percent={percent:5.1f}% " f"elapsed={_format_duration(elapsed_seconds)} " - f"speed={speed_mib:.2f} MiB/s " - f"eta={eta_text} " - f"no_new_bytes_for={stall_text}" + f"speed={rate:,.0f} tokens/s " + f"eta={_format_duration(eta)} " + f"no_new_tokens_for={_format_duration(stalled_seconds)}" ) class _ProgressReporter: - """Emit heartbeat logs even while the streaming iterator is blocked.""" + """Emit heartbeats even while dataset metadata or streaming is blocked.""" - def __init__(self, required_bytes: int, interval_seconds: float): - self.required_bytes = int(required_bytes) + def __init__(self, required_tokens: int, interval_seconds: float): + self.required_tokens = int(required_tokens) self.interval_seconds = float(interval_seconds) self.started_at = time.monotonic() self.last_progress_at = self.started_at - self.collected_bytes = 0 + self.collected_tokens = 0 self.documents = 0 self._lock = threading.Lock() self._stop = threading.Event() @@ -76,37 +86,37 @@ def __init__(self, required_bytes: int, interval_seconds: float): def start(self, output_dir: Path) -> None: print( "[level0-prepare-data] starting " - f"required_bytes={self.required_bytes:,} output={output_dir}", + f"required_tokens={self.required_tokens:,} output={output_dir}", file=sys.stderr, flush=True, ) self._thread.start() - def update(self, documents: int, collected_bytes: int) -> None: + def update(self, documents: int, collected_tokens: int) -> None: now = time.monotonic() with self._lock: - if collected_bytes > self.collected_bytes: + if collected_tokens > self.collected_tokens: self.last_progress_at = now self.documents = int(documents) - self.collected_bytes = int(collected_bytes) + self.collected_tokens = int(collected_tokens) def _snapshot(self) -> tuple[int, int, float, float]: now = time.monotonic() with self._lock: return ( self.documents, - self.collected_bytes, + self.collected_tokens, now - self.started_at, now - self.last_progress_at, ) def _run(self) -> None: while not self._stop.wait(self.interval_seconds): - documents, collected_bytes, elapsed, stalled = self._snapshot() + documents, tokens, elapsed, stalled = self._snapshot() print( _progress_message( - collected_bytes=collected_bytes, - required_bytes=self.required_bytes, + collected_tokens=tokens, + required_tokens=self.required_tokens, documents=documents, elapsed_seconds=elapsed, stalled_seconds=stalled, @@ -121,161 +131,395 @@ def stop(self) -> tuple[int, int, float, float]: return self._snapshot() -def _fineweb_texts(load_dataset, verbose: bool): +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _load_tokenizer(name: str) -> Tokenizer: + try: + import tiktoken + except ImportError as exc: # pragma: no cover - exercised by CLI users. + raise SystemExit( + "Install data support from level_0_baseline: " + "python -m pip install -e '.[data]'" + ) from exc + tokenizer = tiktoken.get_encoding(name) + if tokenizer.n_vocab - 1 > UINT16_MAX: + raise ValueError(f"tokenizer {name!r} has IDs that do not fit in uint16") + return tokenizer + + +def _fineweb_texts( + load_dataset, + *, + dataset_name: str, + dataset_config: str, + dataset_split: str, + dataset_revision: str, + verbose: bool, +) -> Iterator[str]: if verbose: print( "[level0-prepare-data] resolving streamed dataset " - "HuggingFaceFW/fineweb-edu sample-10BT train", + f"{dataset_name} {dataset_config} {dataset_split} " + f"revision={dataset_revision}", file=sys.stderr, flush=True, ) dataset = load_dataset( - "HuggingFaceFW/fineweb-edu", - name="sample-10BT", - split="train", + dataset_name, + name=dataset_config, + split=dataset_split, + revision=dataset_revision, streaming=True, ) if verbose: print( - "[level0-prepare-data] dataset stream ready; collecting documents", + "[level0-prepare-data] dataset stream ready; tokenizing documents", file=sys.stderr, flush=True, ) - for row in dataset: - yield row["text"] + try: + for row in dataset: + text = row.get("text") + if isinstance(text, str) and text: + yield text + finally: + del dataset + gc.collect() + + +def _compatible_metadata( + metadata: dict[str, Any], + expected: dict[str, Any], +) -> bool: + keys = ( + "data_schema_version", + "dataset_name", + "dataset_config", + "dataset_split", + "dataset_revision", + "tokenizer", + "vocab_size", + "dtype", + ) + if any(metadata.get(key) != expected.get(key) for key in keys): + return False + return metadata.get("split_tokens") == expected.get("split_tokens") -def write_splits( - texts, - out: Path, - train_bytes: int, - val_bytes: int, - test_bytes: int, +def validate_prepared_data( + output_dir: str | Path, *, + expected: dict[str, Any] | None = None, + verify_hashes: bool = False, +) -> dict[str, Any]: + output = Path(output_dir) + metadata_path = output / "meta.json" + if not metadata_path.is_file(): + raise FileNotFoundError(f"missing prepared-data metadata: {metadata_path}") + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + if metadata.get("data_schema_version") != DATA_SCHEMA_VERSION: + raise ValueError( + "prepared data is not the corrected GPT-2-BPE Level 0 format; " + "prepare a fresh dataset in a new directory" + ) + if metadata.get("dtype") != "uint16": + raise ValueError("prepared data must use uint16 GPT-2 token IDs") + if expected is not None and not _compatible_metadata(metadata, expected): + raise ValueError("prepared data does not match the requested identity") + + split_tokens = metadata.get("split_tokens") or {} + hashes = metadata.get("sha256") or {} + for split in ("train", "val", "test"): + path = output / f"{split}.bin" + if not path.is_file(): + raise FileNotFoundError(f"missing prepared split: {path}") + expected_bytes = int(split_tokens.get(split, 0)) * np.dtype(np.uint16).itemsize + if expected_bytes <= 0 or path.stat().st_size != expected_bytes: + raise ValueError( + f"prepared split {split} has {path.stat().st_size:,} bytes; " + f"expected {expected_bytes:,}" + ) + if verify_hashes and _sha256_file(path) != hashes.get(split): + raise ValueError(f"prepared split hash mismatch: {split}") + return metadata + + +def write_token_splits( + texts: Iterable[str], + output_dir: str | Path, + *, + tokenizer: Tokenizer, + train_tokens: int, + val_tokens: int, + test_tokens: int, + dataset_metadata: dict[str, str], verbose: bool = False, log_interval_seconds: float = 10.0, -): - out.mkdir(parents=True, exist_ok=True) - need = train_bytes + val_bytes + test_bytes - chunks = [] - total = 0 - documents = 0 +) -> dict[str, Any]: + output = Path(output_dir) + output.mkdir(parents=True, exist_ok=True) + split_sizes = { + "train": int(train_tokens), + "val": int(val_tokens), + "test": int(test_tokens), + } + required_tokens = sum(split_sizes.values()) + if required_tokens <= 0: + raise ValueError("the requested token count must be positive") + reporter = ( - _ProgressReporter(need, log_interval_seconds) if verbose else None + _ProgressReporter(required_tokens, log_interval_seconds) + if verbose + else None ) if reporter is not None: - reporter.start(out) - + reporter.start(output) + + split_order = ("train", "val", "test") + split_chunks: dict[str, list[np.ndarray]] = {name: [] for name in split_order} + split_collected = {name: 0 for name in split_order} + split_documents = {name: 0 for name in split_order} + split_index = 0 + used_tokens = 0 + encoded_tokens = 0 + discarded_boundary_tokens = 0 + documents = 0 + snapshot: tuple[int, int, float, float] | None = None try: for text in texts: + ids = tokenizer.encode_ordinary(text) + ids.append(int(tokenizer.eot_token)) + if ids and max(ids) > UINT16_MAX: + raise ValueError("token ID exceeds uint16 capacity") + chunk = np.asarray(ids, dtype=np.uint16) + encoded_tokens += int(chunk.size) documents += 1 - x = encode(text + "\n") - chunks.append(x) - total += len(x) + + if split_index >= len(split_order): + break + split = split_order[split_index] + remaining = split_sizes[split] - split_collected[split] + take = min(remaining, int(chunk.size)) + if take > 0: + split_chunks[split].append(chunk[:take]) + split_collected[split] += take + split_documents[split] += 1 + used_tokens += take + if take < int(chunk.size): + # Never let one source document cross a scientific split. The + # unused suffix is deliberately discarded at the boundary. + discarded_boundary_tokens += int(chunk.size) - take + if split_collected[split] == split_sizes[split]: + split_index += 1 + if reporter is not None: - reporter.update(documents, total) - if total >= need: + reporter.update(documents, used_tokens) + if split_index >= len(split_order): break finally: - snapshot = reporter.stop() if reporter is not None else None - - if total < need: - raise RuntimeError(f"corpus supplied {total:,} bytes; need {need:,}") - + close = getattr(texts, "close", None) + if callable(close): + close() + if reporter is not None: + snapshot = reporter.stop() + + if split_collected != split_sizes: + raise RuntimeError( + "corpus did not supply all requested split tokens: " + f"collected={split_collected}, requested={split_sizes}" + ) if verbose and snapshot is not None: - _, _, elapsed, _ = snapshot print( - "[level0-prepare-data] collection complete " - f"documents={documents:,} collected_bytes={total:,} " - f"elapsed={_format_duration(elapsed)}; writing fixed splits", + "[level0-prepare-data] tokenization complete " + f"documents={documents:,} used_tokens={used_tokens:,} " + f"elapsed={_format_duration(snapshot[2])}; writing splits", file=sys.stderr, flush=True, ) - all_tokens = np.concatenate(chunks)[:need] - boundaries = { - "train": (0, train_bytes), - "val": (train_bytes, train_bytes + val_bytes), - "test": (train_bytes + val_bytes, need), + temporary_paths: dict[str, Path] = {} + hashes: dict[str, str] = {} + for split in split_order: + temporary_path = output / f".{split}.bin.tmp" + np.concatenate(split_chunks[split]).tofile(temporary_path) + temporary_paths[split] = temporary_path + hashes[split] = _sha256_file(temporary_path) + for split in split_order: + os.replace(temporary_paths[split], output / f"{split}.bin") + + metadata: dict[str, Any] = { + "data_schema_version": DATA_SCHEMA_VERSION, + **dataset_metadata, + "tokenizer": tokenizer.name, + "vocab_size": int(tokenizer.n_vocab), + "eot_token": int(tokenizer.eot_token), + "dtype": "uint16", + "split_tokens": split_sizes, + "split_documents": split_documents, + "document_disjoint_splits": True, + "documents_consumed": documents, + "tokens_encoded": encoded_tokens, + "tokens_used": used_tokens, + "boundary_tokens_discarded": discarded_boundary_tokens, + "sha256": hashes, + "created_unix_time": time.time(), } - for name, (start, end) in boundaries.items(): - all_tokens[start:end].tofile(out / f"{name}.bin") - (out / "meta.json").write_text( - json.dumps( - { - "tokenizer": "utf8-byte", - "vocab_size": 256, - "sizes": { - name: end - start - for name, (start, end) in boundaries.items() - }, - }, - indent=2, - ) + temporary_metadata = output / ".meta.json.tmp" + temporary_metadata.write_text( + json.dumps(metadata, indent=2, sort_keys=True), + encoding="utf-8", ) + os.replace(temporary_metadata, output / "meta.json") if verbose: - sizes = ", ".join( - f"{name}={end - start:,}" - for name, (start, end) in boundaries.items() - ) print( - f"[level0-prepare-data] complete output={out} {sizes}", + "[level0-prepare-data] complete " + f"output={output} " + + " ".join( + f"{split}={count:,}" for split, count in split_sizes.items() + ), file=sys.stderr, flush=True, ) + return metadata + + +def expected_metadata(config: dict[str, Any], tokenizer: Tokenizer) -> dict[str, Any]: + data = config["data"] + return { + "data_schema_version": DATA_SCHEMA_VERSION, + "dataset_name": data["dataset_name"], + "dataset_config": data["dataset_config"], + "dataset_split": data["dataset_split"], + "dataset_revision": data["dataset_revision"], + "tokenizer": tokenizer.name, + "vocab_size": int(tokenizer.n_vocab), + "dtype": data["dtype"], + "split_tokens": { + "train": int(data["train_tokens"]), + "val": int(data["val_tokens"]), + "test": int(data["test_tokens"]), + }, + } -def main(): +def main() -> None: parser = argparse.ArgumentParser() - parser.add_argument("--dataset", default="fineweb-edu") + parser.add_argument("--config", default="configs/level0.yaml") parser.add_argument("--output-dir") - parser.add_argument("--train-bytes", type=int, default=50_000_000) - parser.add_argument("--val-bytes", type=int, default=2_000_000) - parser.add_argument("--test-bytes", type=int, default=2_000_000) + parser.add_argument("--dataset", default=None) parser.add_argument("--local-text") - parser.add_argument( - "--verbose", - action="store_true", - help="print streaming progress, elapsed time, throughput, ETA, and stall heartbeats", - ) - parser.add_argument( - "--log-interval-seconds", - type=float, - default=10.0, - help="heartbeat interval used with --verbose (default: 10 seconds)", - ) + parser.add_argument("--train-tokens", type=int) + parser.add_argument("--val-tokens", type=int) + parser.add_argument("--test-tokens", type=int) + parser.add_argument("--force", action="store_true") + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--log-interval-seconds", type=float, default=10.0) args = parser.parse_args() - if args.log_interval_seconds <= 0: parser.error("--log-interval-seconds must be greater than zero") - out = Path(args.output_dir) if args.output_dir else roots()["data"] + config = load_config(args.config) + data = config["data"] + if args.dataset not in (None, "fineweb-edu"): + parser.error("the isolated baseline supports only --dataset fineweb-edu") + for argument, key in ( + (args.train_tokens, "train_tokens"), + (args.val_tokens, "val_tokens"), + (args.test_tokens, "test_tokens"), + ): + if argument is not None: + data[key] = int(argument) + + resolved_roots = roots() + output = Path(args.output_dir) if args.output_dir else resolved_roots["data"] + cache_root = resolved_roots["cache"] + cache_root.mkdir(parents=True, exist_ok=True) + huggingface_root = cache_root / "huggingface" + os.environ.setdefault("HF_HOME", str(huggingface_root)) + os.environ.setdefault("HF_DATASETS_CACHE", str(huggingface_root / "datasets")) + os.environ.setdefault("HF_HUB_CACHE", str(huggingface_root / "hub")) + os.environ.setdefault( + "HUGGINGFACE_HUB_CACHE", + str(huggingface_root / "hub"), + ) + os.environ.setdefault("TIKTOKEN_CACHE_DIR", str(cache_root / "tiktoken")) + Path(os.environ["TIKTOKEN_CACHE_DIR"]).mkdir(parents=True, exist_ok=True) + tokenizer = _load_tokenizer(data["tokenizer"]) + expected = expected_metadata(config, tokenizer) + if not args.force and (output / "meta.json").is_file(): + try: + validate_prepared_data( + output, + expected=expected, + verify_hashes=True, + ) + except (FileNotFoundError, ValueError): + pass + else: + print( + f"[level0-prepare-data] compatible data already exists: {output}", + file=sys.stderr, + flush=True, + ) + print(output) + return + if args.local_text: text = Path(args.local_text).read_text(encoding="utf-8") - def repeat(): + def repeat_text() -> Iterator[str]: while True: yield text - texts = repeat() + texts: Iterable[str] = repeat_text() + dataset_metadata = { + "dataset_name": "local-text", + "dataset_config": "local-text", + "dataset_split": "local-text", + "dataset_revision": _sha256_file(Path(args.local_text)), + } else: try: from datasets import load_dataset - except ImportError as exc: - raise SystemExit("Install data support: pip install -e '.[data]'") from exc - texts = _fineweb_texts(load_dataset, args.verbose) + except ImportError as exc: # pragma: no cover - exercised by CLI users. + raise SystemExit( + "Install data support from level_0_baseline: " + "python -m pip install -e '.[data]'" + ) from exc + dataset_metadata = { + "dataset_name": data["dataset_name"], + "dataset_config": data["dataset_config"], + "dataset_split": data["dataset_split"], + "dataset_revision": data["dataset_revision"], + } + texts = _fineweb_texts( + load_dataset, + dataset_name=data["dataset_name"], + dataset_config=data["dataset_config"], + dataset_split=data["dataset_split"], + dataset_revision=data["dataset_revision"], + verbose=args.verbose, + ) - write_splits( + write_token_splits( texts, - out, - args.train_bytes, - args.val_bytes, - args.test_bytes, + output, + tokenizer=tokenizer, + train_tokens=int(data["train_tokens"]), + val_tokens=int(data["val_tokens"]), + test_tokens=int(data["test_tokens"]), + dataset_metadata=dataset_metadata, verbose=args.verbose, log_interval_seconds=args.log_interval_seconds, ) - print(out) + print(output) if __name__ == "__main__": From d78f019b0b045bc8066428ec08fd76384b7b06ff Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:37:44 -0700 Subject: [PATCH 10/20] Stage corrected Level 0 source archive part 1 --- .level0-update/part00 | 1 + 1 file changed, 1 insertion(+) create mode 100644 .level0-update/part00 diff --git a/.level0-update/part00 b/.level0-update/part00 new file mode 100644 index 0000000..50c7a4b --- /dev/null +++ b/.level0-update/part00 @@ -0,0 +1 @@ +H4sIAAAAAAAAA+xce3PbtpbP3/4UuOrOVmooWpJfiXvVWSdx08zNa2K32Vmvh6ZISGZDkSxB2tbNZj/7/s4BwJdkO5lJOnNny5nGInBwcHDeBwDrbj/45s8Iz8HeHv/F0/3Lv8d7k53J7v7B/u4+2g8OdkYPxN63J+3Bg1IVfi7EgzxNi7vg7uv/F33c7SBN5tFCfUM9+AL57+8T3Hi8s3fwl/z/jKeWfyyvZDxyV/4y/spzkID3d3dvk//4YLJf2f/uCPY/3tnd2XkgRl+Zjo3P/3P5h37hH24JQX+VLLzEX8pD8Uu5WETJ4mc/kD+/355HibyWs6EMywak1ptDofxlFsvhePTktNGrsjgqDkWR+1HSaM7lVaSiNDkUe493Zjv+o/2DyeNH/nz2KJC7k/3Jnj85GMn5ZBSGgR9OHs0fY3CRfpBJ9E+ZH4pFVkwIXbHKQGYZJcV4nyBoGo/h1KEY72vFQseVH9fNVWshVbHWvExDGRMrrtLAn3kKE4LK0WTvAG2zOA0+mLbJHk2ZeLG/IpJ2+eVS+qH9LZcz/B5PHhGleZqlJTgxcmnmWeRjyrkfK0l0RNK7ltHislDEqlJu8ULAeaJj5hfBpZmTEC9yP/T8ICiXnipkhiE0wdK/sa8TvTpJiwZnZI4ftATbxgglA7aaGnMwZ9rAuwSMn8GHLAXWBuYJo46lnxPJXu4XktcJayfCyhQ8Wuuc2C6/vPH80F9ebwAyGCDTODcN3HLt58sys+sd83I1A71QBv6KYMfEOln4Y3p5bF4m/LJHy85UFJMCjt2RHD6yfA3iKOM2WhEJFjNbnLo1zYpoqbWQybbrWKZLmRTlspqBWxMwUubplZGrEEpKUoqdHVKnIF1mUSytJviJH69UpA6r5Vwz9/NqdKt1TQS5n4TpkoWoEX6+/bvb746Pnr06dpfhN/Mxd/v/8Wh3f2L8/wFeyf+Pxwd/+f8/5flOvFBpDLsLxUtKAMRIJH6SPn97Kp7AY8dw/Vtbp5eREqqcFbmUAj/9RERJKDOJf5LCEX4ZRoU/i2U1dmbGuuJFIcJUKpGkhYiWWZoXoriUIpdZqqIizVffK/H+/fDt82dC3mQyj8iaxDxHHLpO8w8izYWfZfHKABFCJbIyx3CmpUgFTA2TR+pS+CLIZRgRJUew0fcw5WRR+gs5ZO9ekYUVhGIpfVXmUrxn23qvbUsbv1CZDOCL4TvmKUD8MIR/onVXXgDUFogfiGXu1tZ334mnaZ5jDPg4a/BNgv4IodSPQavhc0XEtc+sfDT5xzDzacGwakekiRxqImarQgom3BUsgiuZ04SiVGCoD9ePAZEfgzlLojKXPrhQRIF45QdP0vTDUAU+WKEjdQnvSpF3a2sosihJQMnPoOI9AvtxWIqLRhy/ECot80D+CFBIczgRT94e2zDMWET/oog+cMuFoyNmGfv5CtHGQcQc0Mjx/ith45kjxq8oGEchD3dYAGiieKMRQ8GgDSAKTLnQgf2CsOwSjkRBDEtiCcVh5aDVL8B+JoVCL5quo7C4pKjr0IILiIciNaE4AEk7zmT3kajYrAAOaATgUE8vELQlS3kb4TorC+NyFSHQusQjEBLSEnwKTb/gGOFwEIlIcymQZHrFo9GQIpWJWnrNAbQeoudRhHoZBXnKsRZr4hkeCY7xYCevzoZ6Z/x4YjkFM2koYpmBqVqlx/vuzqPdVxYuhcKIiQMna4B4NfPoRoZO04K1mLZr+WyzXDJQBkUrLn2yYbZgP7zyk0CyCVvZinevnxNeHkO5Q6kpTxMopq/Nfc4mQCTWkwxhBtpi6uyCCUzQV4W0sGOfNlaKdM6YJxsUhCwjjwLQTjSQ8UjYdb6Q61JmyBtyJPImiMsQ883zdEksjtIQpmQ8QVxN7Gq7DuXcL+PC+AubwgwphUEGEBcREmB0APEckoS9bMgqLgaueEnN15EyOsHu9cqPYnan2kHAL8ZREIH7s9iYD8kCeQxkBx7LhIBDSJQ5Uvle9ksvEnjHOLad8oY8BIT2NE1CH0OvojxNyOdubV1cXGDs5VbAXX5QRFe0mutrD5oQ7oxHW0Eo/nf7mVSw/Gyb3PO2cfj8F8n5sPbhpqL0Rl7lELNVcQnFGIK9UQaKNGVDKb53z6hCcCyPHdKl8++JIl7EW7+4VJrvQeVnSeLgrHGGKFEExWi4Lr1QP2cB5P71kB3pHOxSIvAT4t0MQgIjfLJ84NFebmgLjVDosqjiCFZFgev10es3gPReHv92/HLkvXvz5nS6XSyzavW6hh5ylbJ5zLOj0yM98N82YNumeW+b7fjk15enJ3cMzqWC4qmaawiyfq6NdYPHByezUtWLNMRnetSQSBH/jexyONQBRGzYKTAAprwTjUrR9MADzShS67c4XQxt8grzJ0VT8JOaYpKumTzn/CBHpA9T+ELSJsdEIu3aHCFjP1PkvaFsDpaYp+XiEhbtiOPTI+1rqQtWYv2VQvriL2Gw2rWFZWCdv+I0xXgmUhKk5zA0ssDKXpgdhDWXrHFR8SPCiUKwGg6hQYG8oFQkl7MyimGKBazvCDlTAC+k5mVsVqY9o5ldaQ2jULV1mxqxRmyzq3VnKKXvhsMSPgOKjOszwBAoffd3lSaVQr0ryavb3Ioqmlp73G0V5FEGs8/LxAOUS/kYFUpc9LACTH7697H4HwQKKXjqxpRDBh0STgJ3oSm1VkAeS836OfMejKbQdp1HHPdqISFyyGVWrARogNtPBSkbZwmcJUEEsQ9tiIqGbXcs6c1vx+/ev3txejwdi3vWVBOIbsHUqDpwgH2IQUpH/TrAORsjIAHpGMlR1Ix1OrHv6clv2o1p/b4gqkhXY6Q0LKkLUSKg55+jV8ZZbOsSnPju1WsyslaAp9BXy1vdyreT4+NnJ1NC4UxGkwNnd/T4MYTe5e7b0xevXvzX8buTqeYjqUWbzRw8aS4wW1PzCgU1ZLeEDTSDI/ltX1BOndc5NZtujlw7uUPEHSIcqtjvpQI8OaLw9E/JNtDW/s9w2fdJ4hYcr9+cHj958+YfNdWa6PvASR4sjq3fS8RdrmxmlDfIGSoDtT0ae3CNi1iy7N0oWyWzWqEtnMjiFF64mx+KOFVQQug6dO8mKpAAyxskDIjCN8WwEUw5l839gDJkmZAy2QJi4SMjtmmToLTJMRZAVU6pNppJ2zjIBjrJYZxdomhbtVJCneLpbOhJSgYSCrUEdWS2ernVq7ZUpeNFOi+uKX4iFlxyJvkjPAclaJwJCxVQwh/NYetahLdr3Kuj//ROTo/fnkwn6zbx5Oj06S/eCUQLl7PW+/zd0TPv6OnTX18ZDBtgjn87eum9eH16/A4/NgG8P37x/JfT9zQR9Ge0wS4bXm/dJG/xfJxiREsq+5BNw/Ik3MOvCPaXMg6HaEI1Dj0Zgkt5mq1YXrXKUHLLmUmOCB1lEHynWLc4XXHMutVWKJKDThJ0zQh5cD5B1ISRv0hSroRpTnWZlojJJu/THgKDEiQWUCSODLoapHIdgCgbOWUcNupaO637BXtr/woPBJ0H3/gM+IvO/yaTB6PxaG/81/nvn/Jo+etwVBVqX1kfvuT8f3dvTOd/e7uTv+T/Zzyb5a/rjWz1dea4e/9/f+dg96Da/9/fpfP/vQnU4K/9/z/h4c0uz5uXRZlLz7Nb9LxLwskXQrptyxcInEra90Bd2Z9cJZrfSLcu7e9U2V8ovArKx+y73t+zb+qyLKK4eltVw6iE32IaqSQNULtz7W0IUqizCt1NOVoczWwXbRjpjmJFu7G2/ShBJvoC2TBVENXKkAlknI8kWTUxSvrL1oubJAyClJERu2ZXxMDEKZ0icpPDG1HKgPHGgQGyWwye2ecIPd7w0YA66zGQyM14x+MpYzQQvOcrKj5/kF61Cwwxvfv1tXeCBO/VkYds7uTFm9diKiZbW1uhnItQXkWB9PyySPsDMfzJrEk307mjENHcNAZl6LuR8qpSqz/QEPTkEpqStIb3ezSiN2hjmfkB8rVQuctMfRk2DDDINk+WlejWy+K6FRlcXx+vosDmxb1Gxqon0XrmUjfDaLxJ5m7u0DMt/aT0Y6/Rp2drHVR7ftGnzXmPNtJveG6n2hM/FKSaZ6rIHdK5c6ZqDh0pNFnNg2xICWP7duhZr9nZO9eEVcf8a9BVjwVtUQlwnrYxoNVf4Ufnslx69w3WZ/J2FIRds0D8vX0+3xVyG/UPosE98VCMB2KbVtkfOy00myb6adq49tCdZuNKjC7AoWFNzYmH7clqGmqGDzfQY45xpmLk7tFSxu4IayDXB7eg+vwji9DDcw5a2ryZ1Q8tTmCrltQGGd6xNquiHnnIvGC/4i31HQ7jljRa2tXj3e2OhjpmXfpST7evY1Xs06YG+qxHr71z7qFNmQ9yRccN9TJ6zdtFPWe9XSPa1MP3iDZ12JtEzb7qolBrAN0VMg0NzwONsqxwF7Log+iB+NtUtBpqcK0/dEzzmx+X8jjP07zf6uXV96xj1yyaR6iExUeg+lv+qT4HZ7lwuWmPZun0hk7iUBf3Wli15tCpCm2AaHaQD/jYWDUZZ48dUJ9mPes1r0PBVBvMQABqQdb3o9pwtNHSRllfmaogP1nTbDGyxzRaWOZph/yGyd7J0Q4zebCgu0rKHkduZKTeJwjSMilUzczKQPQ5t4coou849SuNPqS4QDsIub/SK/xB/2nekiI3rxsbV8KqRr3JhShyaELJc9tgbKigw+Mz3XcKBqFDNN/ONXPKjM54p3AACQsAYxsT4mVsea8h/w5PdAdba05qJtJ1jkvaOsnpdDfpclAfotJMNqgnIR+nTg2tFDxJN3hyR/RrBjkDp2bCtPql0dxUCJB0Bx9qaZ+15K5BKOXxODNbtzPIyVcsJ62eRAv784HQSmteHjaYdu7oa4NTDAbI/u5gg6HZh9yYDhBRYldfAZxv1QNW32BJa330dNeJiNld67iz3s14WjxYh/kqXDGR7sYRK2t2ReqZBK42qcP7zUETaNLUViL4ZQZFpEBUPG2LRBcJsUFIx0ceMxChdXqalxLKvLobwCyPk3G+WOHxrYmv6lSqq5hVS5XtGibQtZ+zezlhWFEZZaW6lZMyC53qHHtDHtzgXK3gt7jUpua29axe+rThOdogFSOm9c82yAY3UwPUiktK65HCwmktZN8wU/efQ3z/Yaq71KM7PP0By9NcY5FNaeoLwigA3VdpWMaGGgY4rGrKz5DCvSpdJ1+cfZ/bkkF51ZWbqbkXZhtqAl0ivW+rmQLiS+KYE9WRyfD4/gS1GBhES4+jZd2Wrw5b/KskS3zUK26JwlhXw8irEY5ZaduTxOkiorNQOkeyi+kTljYY4htDIF7VyWfz0ZHuHYiPljbWNa4ghZHe9jcq6zO2XnsOu/SVSwcClnX2qVn40BZDhAMCg7u/7A9cLLQ/GCBrZzwdzJrVD3XJ1teLdv18QTVGGC2nQ1Q9U0w9cFW5BLIO1g4ltaAeThvT8alZ3JBYQzX6Ta0x5SHI94hFku3fLk8XPo1JHJRkTXtvZJzMxcMGpkbmWB/wAILrIGR/fRQu/QlU0GkMGjQTTnu+gkGWbRspagyZgZkeptOpZoseHoupwfH+xB0NujUCUW8q2xp9ldWaiGWv9HlJmi/79dXBprGbjSH3re09b2yuaIPXciEcpOm116wQujRRLXpN18DlWSdtP1aNYTusKGoWNW209rSSrMd4PANHrUzULZsxBRPfh8w2bMXEpHIL9wpiSnPNn2YSxHgRO9M8nE6qxNtc1P9ybjb2TdQfJdJYYiQZTM0b5lmNdY2b4ocfxMRaWce47mNsu3gntVJ/5EXfkFIX3kW6jAKPtkL7tBt5yJuQDrCtaG9w445Q7dQKSbt6dNI45b1Ml04kuV7uz3vuR26it09uscyMB6vGuHz3xKMLHzVPiBA3RLKp+oYEfeczKaYTR0C2hYe6VHEa4wh4kPQa8yXTn+kLgobFyCRI6cbktFcW8+EjW0nzv6lyzZ2afkWMw/QTW3i7Vngn5gblSeJnqDqKfhVBTX3N7PPgogrPQ54Rzx0ba5+/PW3U4KqEtUN+FWwtQ75cqZo8xiQvo0T6+TlVy58qSApMOUdQrW4L2ktIFh40t5ANjK0YaNWEJOCYy9GkKdrT2iui+osg1R+sx6kgzeiq8bQma3OWr1EjN/fm0qctebU5izdwaVncA0gfHGmR3lIONBK++4qBeiGutmXe23bRtPLuXI6FtkZZZdRmXlOUmCKJLHZnsj71fcTk8o8yAiPYcXt9rcdrY7SMzkiS57yFRShqX9CNpfQY19ZSFKNGNX7SW1cj12LWKv4MCtnXzfUOBHNFa4vHNWGeXvfx39q+Gysc1JrzezRrr4Ef1mncUObSEz339xQRtiamyAmf3cdyRK+HHKW5N4cYniz0dpzo2b+aoig0/sXuPHPCTLGL4Co7IDRMHkcT00AUnXf8JY0+G51zMieTvmnhxGcsJIQE6gxrGp85tXJuOg3RDodvbXthlBv32qyoaH+2USQ1voqapWm8llwThw/X811zvNL64ooOfq6vGUTeBDIrxAuG4oyTOtHa3X5pJaUtfeq1Ly3R3XNzg3tWFvpDGX01WoY/VrekaYumuvsOLud+Y29L31kHEVuGET5fi5ny+ZkLrznXKQ65Tw1hfDEl7Wv+mZlu4LSqajl3NcDRddUtTvB8nbXkAKKYEF1fuy0m6DmnlqyB6+urdm3GVRKdkkD71WtthYPuXHCltB/eR+bZI/3oOZyL089GCgBbIarsGCpjyG7TnPK/aQ/pKFKZZuHQwd6ictz2o72muXd87NntvsBp8V4bL5q5kAW155tKXb6oNm0YCfLgea/97SCffXxkUxkdhJ/cQF311pYFBqCd0xidNNxMO+50rSjQSyUOH7Ihdpmgb0Kjk/ONdictyOw284anpqJZHzAYkcPYc6as221uhHtKBoDaoPliaC2jHvmpadTH/IcKR2PSQnzXuWHYuEuGrFV1voSha8cNaXDW1dqs/2wubQjZPWnJ8/hI45DOtmUfrZQSkZp43q0joEPKX0jDPhrz9bhHTyv9bWG+Uxslecc1nSQkHUuxKewmrc9JcdqrOTMXbFky5x0ZzqFd8I7tg5Y5i2XKVHw6FB/XeUutaPjUoYwuZE/VSqH2CbGcTmdcqstpW+XXjMisbXMxT3cCrFu0CUR16u8t/SSaSwVXVV0EMNmCTlq42X1jO8/r3cJ2HDSBkAzxcGM/Of/6hK8mgJ2D0/gaDf5J8s1KypJqqgbtDaVFnpaZHcwvtwx0uRjzGER1z+NAretn9BHbevr5ca2FFaNDOVS9u5b7xnFVY42var7NBCsEjRVjcHP9d8FzZmbA9elao92B0urXjw18n3qDW1BWda02avIFzSK6+9TVtNkYW6+UedozjViZE+Ducxs1VMNFlC/k1WZMY5mNXsdOw/cObkMX51792d0mjG0Ah76/uxVZ8/8psAlXq9+hvdVNqD61WlrbCKS69YHklfTqL0S0QBo7CM0Udy0f/jyz35gf33bWT336BPmu00yCmtGhcH1P36MNOMOtzQANCrr7H7rGAitqheyETbqJrsCnpe+ZD6EhmvVLT524xhyjrUidpVLxppO7bgCsWQnos9oBNYe0nR9ZQD3qvBvLdZSvss1Ot7nycGjE0Ont8F8Xnb3Drlx498LsXHTx6wK1Hrlpt6MzZpM8K/Xf1HkfggYPNnU1hn+qf2aV0lcVszm+9kgFkEjwtuOqjze7yfaDU38Rpsu9jn6h+KTkjL8UQ3HC3wX4CeP4v/a+bbuNI0lwn/kVNejxEJBBCABJyWY3dFq2KbeO3ZJWktunl43FKQIFslq4NQoQyeFwzz7u8579kf2F/ZT5ko1LXiIvVQApWu6eYZ1uC8xL5C0yMjIyLi2oswSo9YarHoI8ngXqXUTwhvcCTv5jAliqDoImJabDSTpZZunoiuzTyZiJjLmArYBiN7/1GBKa0DU0s7dnjd3wLeZ8jgaVKdz5suJcmZ4UpHIYUx3Bj9UrW8spOnDAIStaBGOffsCR4+TOVloWSJM0mH9wXjhx6jWfW99IPxS5Ii0k7Bmxncm/CYKm+l2Sq1/HML2X1FBtUbFd3jJDrmKVacERwpGlYPxUptWUdJI6qIgiKNUjKGWULeuPHukaTDyUshu+gAbad5ioC1gega7JPWK1bVGTDeVbk/mFvYwr8yb1dkYXUamtmWv1TRSZ8Fyw3MTRi5TPbFZipaRpqAoGpx9upCGpTdVr53CE1ezIjBZlY8cunbpQ4qzWzVo2bL5W/IEycdXWugHTRLuufHw1OE+L86xQuIWQqlTWnFaaaiWNSqBWTzIlTmrWeRPMcvLMWeSW8OzkCWukrgzr4QrbH6O3NSSHCitrpzyyvqnUVNI7v6ugpVxhzRataTadpovwdZznGW9H11T4Bq1zvWuG0dtg3xhuJna5V5PSBfd5Q2n9oNyPtcWarArWVJpeXPpmR5MFveZykza4gy6S37Dl6DX+94bJTzmxFjS6Z34pUpReDHjeaSPW5bKRvNjbI96GozdkVx+6bkC6aOO/4kNNU1LjFUmX9MZljk95bIIZY0FIeFQAXqcr3PC495qJqRFVInRfrJXaNVcgtzAoy0YPUiwSzItkPTP600IMpafMAWBHrrayxzTcWu0DVYva6jGYHv+F2zCPIPJkea7FalYv+qQmkvtKAsf9I8h6HQNVGru3FB2vCX5L6JP4/ROFrEaJQxRi6iVKSaTn0nrpF81Rl8RJ6nXa7Y58GxO8zvZDw0359zawrjcwUgW9xWohgfn8Ywo82YUD2xcDU8cm6gaQTgE57ukJHWYXuoPFpsyjpKSzooQLx/fpp6HdWulfC4Jc1eBbX5lqsCTIlEKxmjIKQxv1wekC6Ll2LzX42JUK1qU3mRqTFSVz9Cm4o5MB5Ej0kMnTQKdImUpNGxJBMf2zpX/IC01E2mHoIgo0BvaV36mGKn7GS9DgFkAcKq+cCQ3IyVkuQRLjY49pT7OdTlo1YYYzaMExPJ/AzbgRFNeMD1RxWSU5uQap7dmtLmYewrvTjA5jvCoF6fQo1IuU1gg6MFo2/pZ6FK1vVbaQiqzmwzle2b37Px8TcHET10jEU0F9Btr7hsc8aX32GolXPRgDg5lo4x/U1EPCa7P24cYvNbbX8WcN2TFynoFjMur6dF5ojfwu/6G6iYRJXIjFTyGHZJEolK4WB8eUrGALjknAxhNIrhtgyRadAfpvsDueFauksr8j5kdG8XFS040qsb0hRYqosruGAZlCuBpQ6kEuQFWYK8esQz5NxBHby3EtnEQGWi+xqLS8CKDnhiKMiCQcCVKF+luQZ3TcghxPlU3k4wHmt4Rp8XYwJ9YKHYOVbbjeOQZnqbMwdEyh4pZMFPpcKpnv1TFBkbEkq+WzsWJgYxIhKLfHpg3opslVwRNKrO4jknqe3fRwQa9XA3IKVf4wjtsbGlY+RnzhBqsS8TVBCajZxWIvtA1sW66Xrk7bWOFViBcj7ydOAeehxL1yCPG5WQeVJsTs/QZxJ2JUO4rjd57X/Kc1mlWUs1jh09hSqZ5/X2TOi++LWvno2mMNbhxY9ojtXWvScrLrHcy7/aPmDftKxFLaNrDpgvIOo961l+CXN8caFQuKy8MMqoq9Ufom6L8HKhpJ7kcMPWXco5f5Rmu+yGb12gXQ1Vl2gcb5vRr89pXyGvhifQ7n2kSINuh+jdc9gNRCRaSfKaHO5ZpsnUbqBT2HSDc8CKxeiG42aZu4kgWcB6NcbxlWNOh0sVCZcKIMq53oml8kkZuI9hCHKOmU7kXtPvXHpFS73er5OvxKFiAum01f8hZqiwnyVAo4qIQftxbN8i+GJRp+bte20c3TLvGi1E0/3xMBDKqqE2/pHtDyc8iGtQAOipVTsNLyG4hZUC98Z4QTDwXYMIAYJUY1csKfvQSYrHZDEOYAkkv/A90N/ZXrcJgSkpnhgkAvShlup6qri6H+qm5Gc0FHSfmDbrwPSs9et5J0sr2v42tb+vTqHGVHYjVKKpSxYUdo4Fw3uFjWXhmLRneTzdUF+3bkEo0TtmwosZwLObygusirBiLMHTwQJqcaQGD84IHx8suAGfbyKELqNkyHx4bGIWwxIw7TGoeycVIiDG4c0i3mJcISV+2u8vkDuhNFs1ttMsuCH0V442o9BlXNscCwcgwpCNmoxYCf5ApQW9GlmSHtYH6jRbyP5B/0B5xBxeT9Lq5mEB1wydWjHPpWUNTFBP+Jlo/pLvhf/BjRn9FSsKtSrqkQVN54FpmS4UREt8kGKJWqC/K7Kc9SvK6IK4Kz3lqsIlYI+EXQhj2C5qTNwDJS6bMjulqUqc/K+WykVnW7IzCjd874Evt9eRw0Fa2meGKnS8+UEWbQPL74Gjsud24iXR0o44Bx7Rr+Omp1xjdFDRvEYQirMKWBv559mM0vZqEKQkS7Un/+VRC42bNlBjsuhIKf1LCE69LjsnuaW0ffveLMS2VNc87Dzc2hv7uYCFfG1sG4vLo+GnvXEfKxNYTFYhIHYA9GBNPdAAYOwN51p92GGYgB0+ejAvVFKSyF4r1r9YPxAnGid62RxldwNbWrFF1NoYjCq/68PewKcUZkjxoiGr3Fan0MbQBR65+4isSx6wiq+JSYJdmr50bIwW00CjF2Q92wTwNL52A+wjMuKvcqVYH1apQfTvapvIpel2fSvq4+fjzTjdi0myLOS538YjZhZTKGCkHB6TJLP7gV8ZEMtQFG5AOx5B5JIyy5O2oxYAhoS9Gf/rYSAcrCtxIF6u8WIkEWBfoAbGPwf+U6AfWNVvMBOmHXqmOyijRLD8csXDRsfCKNGVgaDwW9Tf4o9FeqTiA/8Vpd8QJcUvcWj9jO4hjHFp7mxjZCoEpXDBvlXYONLhnw2+iWAb+oootWytvGMQN+5GwBttBmlCB/gxfpcuTfMKCz/gY0Ed9YR6wdDsCE89G6MrNZC3UoixZWG5jLWInda/xyVXZilndu04Ij71bZec+JQcmlz4EQfXjRFIJTK/1U3IqASTXrbOEvnY5tMJJXMBQt+wvsFfSlypHohfo0D7sU18f3P76/jCXoa7cXbMpUeqFhg8cNJ6+dup79WSIvxjPY7UG8IBOhXtXdzSM7m8kQfrG7dy+WuF11Gk8ssZSvLF+48LJJegDu4pQuyIZFoI5GlBpK5nnrub3DfN5yDtW8sU6EJ/YueTwxql8lZwnDiknPtZQipv5RZnLgCpZOrLTS92lo3lOkOLIchhRKlkASAslyOFYs6ftONOMI5jYUqnIFt+uRanFJKleWvY1UjYlPuWIgM41Vj8tMyxVCHCTQWiFOosI8Qk27K81hi9aQVbs3FBVN0wVM+ZDWzndmoQL5kXaPdIXhaR5TqwMhdvN6F1gDKPS/n41jEuN7R+f6BjWye5RXhs/OvnIoaAAnIiTsB/LS7TfgXRqTuL6pyWCn3qVBu0OizcktHV34rXZ1Sc0tN3ZJ7Vvs7RIIt93eBozYkd5m97FZ7XfFnqCBvYPehm0J9OjuoMFqzWyOPIucptxQ1Bj+E1Wuq1J2dB92rcy54nW3zDDOn6cTb6f3K0D46ChPJqeMBMJUOEIXJHWuBuFiUWkvbG7YfnhCbqgm+jOcT0h0OVAV89m6MBqcwQPdpnHhQ8HmCSynCZ/UGzHWaD9EfrQHrjRQhTOmYIn4RpjN6lFxYTkE9IuAnqRihuEUPImcrhAU5elZuatQvoU4C6980YajY9ja7QTprWwiTUFAvFpTEB1twVKpz6arO4pgALdHRpqJsz+F5poRxTsFdvuuQlm4d2IgTGb4wDD2d1Dv2kWqoPLdtNLwv9pUHYOkKSe3KLlz7WopIghKS3V0kNbz5RlF7HxDOUrSx8Va6Wg0SFV+vaYDi6IDMI6m26tFYozWKkGgtvseac1XFlOWbFuUtOdHE21t0cdxD45ZjH2G9nIYK1DzgCUQ6HBpkoMIFBpWd59NJMQMkH1ndQ+17RxUwyi9yPJS \ No newline at end of file From f53aeefa7983760e3b0a1a72c6ccc7869b4e57b1 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:39:48 -0700 Subject: [PATCH 11/20] Stage corrected Level 0 source archive part 2a --- .level0-update/part01a | 1 + 1 file changed, 1 insertion(+) create mode 100644 .level0-update/part01a diff --git a/.level0-update/part01a b/.level0-update/part01a new file mode 100644 index 0000000..8bf27af --- /dev/null +++ b/.level0-update/part01a @@ -0,0 +1 @@ +8OzBCpbQGCKeFeRLkEDQPwikqJuzWNm9isAldSzQ8owsKc1MipWXhFa1jokro4So6cAjszLxClkNlo1sNUSKvMgISmYR+IpLAVfqxqwZUwJ7ZQvdYr6xuegRaPNnQuSMwev4BraWNIvkijLFratyZHWl9sotetp91kKSZ0//KTcmUzAy497ZydF3InseIfPgwQC36WBQ4ynlPVsetC4e/0kJqu8pAFR1/KfOYfvw0MR/aneeYPyn/cMnD/GfPsd3i/hP6OdRBHa6ddilb9PJxAm7hMR9Z+e74xfPf/rxPQVJ1XbHtbJgqTXrEoIoGrEk5PKXbKbocMI063QXFbiIRFApOPqW+aLu2t+TAa8pCezLda1DptFITeHfK2CJ4B+g+zeBs1xjzRUB0sZaYxQ4kJPFOQEZjyNQSCqhxBGeOfq4ls+IIaVxZeksUcO9pn//aXlj5kQRQdfLIc6n8e5EDv9ohucFsnDZ7GO9FgklDj2Vy+LGr5H2hWythv8Ink4ZpG1ox0RCr3G8LOTfqKZj5abJpwLn8KQVYxCBdwV4DUz4CxRNDVPgVjf3+9vncPP0IXNd3XPjQppi3pAdd304Dl1lYMhS3GOEu030zal+qSWGQqEbE1VH+YlMAC4eOPRXQc4iZ/jmNz47UQUx7gU026hweTCu6eAf7DYdDuZpXmB4YNPatfrB+EZHqWjiBLre13tMB8sSfAV7M7JOPko9cZIhBJk8sM8I9GmwnbEDNkf7HKlKq0jH2YDEiFxObHRnnrjW5hnyJoiWfQpHBwaUTVH6iNTOTkzEbgipJ5z22QJ/qHYVKwZYBof2QlbiKAL0k/6jaecJeYFVpk19V4oSCciNN1DLSTWV3INcYErML4uPHVSXvDGS23IYJuJxAMOaLm/qhw2MHABxnAtUQ/EDKAe3cgHYTQ+emd1840jLJyReB34M59EzLWTBZdVsvnwVA6OCsW2u//Pzt3/86U3Jgjjh5jbNJkeSHnx3/O3zP4eQXLdom3rlhK0OgLmmQJv6RbAIYY7DETpuALaHZFBvO3QJvF7cCVveH797H4xmuw6ELg/u1gVo9dsf3ryGlXFWZ7tOxN7a77hvnv/5+C2gvUW27XrAvnQnS42HrgjOOqfb0Pwr7gAhE7/8IBOn3fhuQKJXgz8cP//Or4smdFtUPf7jN0HVbHq6ueo3P77+9gdLME11q3C05f5WkeLDOTf6et6cu8qYbqZg1bea+J9/rkC7rTpwG8xjYaJ1aV1XPA2xY81kmBarBvsmpUMancVNHRf/wvs5MI1QLF8CR+fOgMsHGr5PtFG3ddk3fUMxB8avlmKloL7DjcPfiuGKlNwQdIIMOBWDR86hiP9uJhZzJBU1U+8O/tP5URrNJ/OkBE4Hphy7USm1nyRKV8+4lCG9xI1DF3HG4brKtuqqO5qhtHEPhSswDHpYo/tqFU/JUfco2A1wrj/iLTdpJ1oIk+h4BtbH2FGi7sBB2xzkkttlB2H30jKBPUoURGujbqIIOJEm2ZuF+EsGjXSRRntusxEK2o2GF8WwBC2oGobUvDEcOHtZ+ZhpLHC6KDzC+aRQkHNLnTWxDTvMagK37zHXK+2y38KJ7gFs1y/cZOpin5a5MtyjclBDYEyToxwjpsIlIjm9SnQJghjrg+tI73fJYbt7+HRzm7Yat2td5gGF3+sG/vJGy/livl4ZFUM7xzWVpf3ryhscJODU68q/Q2e+m/umi1PHCiArxfgq8QFpHNILhGdJ4Vgm1KI8XtVtQd55bKLH4HoZIQtXwWBWsn4VLFmwo51RBxvA8fJ2uz1gqm6zDVxvct79hJqsRn9T3wmzrduczWez7CwtbzaIwI0Bp8v9b929J8UUg3yoGKwWpOmUqxqs3Fh7V0faGxvjvxoYbkTp8mWgXQbzHOuCuXaatj3Fm1JTB0a6rTrKjZgeIinH6HESXr96przb6F6ndSvUEbWrUSfWNixjPqHn51uuj6pYsTKx5qzO+K3HaapWDNK9UQVR4RVietcudrDu45MHCualsx3euvU8tOgIVJBOd7lnUmqFgZC08F+zrNfe6+7NVj2yOu5GEIhQkPklKA3tJ4gprnLhpzlIiaLhrSURM0dPHBXVrEtSsngz7W1COQMy3okYBv7aj1UP371/8fdfHWHvfl6Aq99/2+1Ot63ef588PTjs4PsvFn94//0MX61We5dNxnvKJT3cBo2bT3MvxHdY4N7N/bBVw4Btwj0numhvt7qtdu2BRPyDffH9z/FD7kn9Y5P+x5Mn7adm/3fhN+z/7uHhw/7/HN8t9D+WGat2oM4rC94KXVwpr6Fcr1T9Q4f5NfDI1gPoCEut3zx///747SvU3MqsJ/PafyeBTfGXVv0voy8bf2mRskM0ZDvaIS+QsRt8zNPBLLtYQQeH5+vJvx7yzURbLx454Zn5IsyW4gkboRyqYBJhFGdgx4wN5GyUT1EQ061isv4InCD1bI+6lqwXKKG1AjZ8O14t80vFvl5C66YBFbpYyylnBXBi9MJ92SrO00WGMTWfmT86JrKzLWt7dknV3ptGLpPHSf2SY0w30KIy23uq9AabyWkzGUKZ/dbBwcEhXCgOWk+fHgKXDQR+v3NoRAbCYpumT9zrYAxTauX3plXdcpo8gn+/TOqn8IPKfQmtPUrq9Pv3lASsLFTcdjxKEH6JYQKshSpKLm0QYlyHejQ2UiwGsWmGghGbvzjMVXXA6uTfol5ILZBH9udkqUxW2LNpV9oioq7neirzvz60+fSWsJx/NDFQXKsJeXd1mlBSH/cVQA0/pTAyQbBCCs/le3Kr6Q5SYCX+6ZXQXYQS+mf0ycYE2ZJ/2pLW+1QQ/ZmXw2jOaiOTgDjo5SULYw4vPZzMUVe+h5Mg0FaZvqtcjryi/ghd5jiueBxHDBQKuNwHw6YoaqFJcjSse7nlPfJy+WwdOm1w4ETol/xKaNkyG2Y56tymKCfYY+ploUZM+DV+DE7X4zGpitP8kKHdianYx2g1aiVLvA15gErsqnn1keQWg0n+IbNB4Tda00d628KgOdozhkH6foN0suvuooTAmNoDvi5Iwh0flwsEAZc7avF6F58B/NLJ4jztBf2OV4h7e6SzjuqbrRx3l0a7wevZFjOtjsJe5cEtZ7AMBK8ROvDqAIUT82lOycdBYgddmzwCkngY9W6hBu5QqMi+xM9Cpm50kj3HucqjOKywVQuHUIuH1lQLuRf1u6oOPqRaWuGQxWQ2zmEd35KVUqF67OL4e4k8HOhAoF+GzVGSvp73ssItdpQb7ql2uOKwcC1KppaNEIuSLBx+haNwlUouRgVaNMx6pxGMkbvziBwu19VA0J0r/F/AQrmZEThC8y2yHCpQqbCu3mdtIAYU94XhGWrSuYxs/NEj1eyONxFWH1k5pFbmTo7DYYAcvMUrHWU1mJ54YhyOz1oqXQpg7yh9tboNzqmjXgAhYeTEwggCG1nM1GwrnazudgjOnCt0Je0zEyqcqWuMSTn2BDmyDXplNKfgnWHP4AzzVYC8YKDxfWG2RFNOsAxKqBeXJLvqMN+4rtUhdgkIsLJci/U8STWGl7XflHE8HdayT4HhzEWrjuV4hcmH9yaIqDhaDsK139uMzQJN8OkQDjdhHXhS02RO9cGkuwvTFxtdzQtpEmuPZrqWRQ5B/WjQTnGU/OsqhGlGicY69drKF5fi2rYIhFxfA2BmWRM7sIbYagVxhHOMm133B6uViZpyLnuTdHo6ShPMO0rq+A+cYXBiNRP9u62t/J0rF/Y56gsu9D2qeU2xlyJuWZj313P3SAwwUtYJ57tlHX+bVhb2rgtxJslfXu/URZJmVowZFx1SQn4RTtGNx1zSeO16lyDvRtrZnc35z90bJNPXdqxHrSfjm9BhY6wTdlsS9itSWpyUoKHH79m7lKNBprysaxtPN4qeE0GvnNTFww5vG4nzroEzYZb9qhggkx7zglPcdWsvO/wcKvwcLmkl1Y+szmTZ20Rdwkqn2SotenW/ \ No newline at end of file From e8f977aeb39ec71256387d1bd986d7cb7a9fc69c Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:40:44 -0700 Subject: [PATCH 12/20] Stage corrected Level 0 source archive part 2b --- .level0-update/part01b | 1 + 1 file changed, 1 insertion(+) create mode 100644 .level0-update/part01b diff --git a/.level0-update/part01b b/.level0-update/part01b new file mode 100644 index 0000000..2d725bb --- /dev/null +++ b/.level0-update/part01b @@ -0,0 +1 @@ +IiZ3BP10c7rxONgY/MIvrh+i/bYtHe2XTCWqktFDZqXhyHoG9/IFkTcL4Cjxw37o11AEOHCPQNbIvcOpl64v80meLq8Gv8SZ+gkHokBi7zhyRUkZR+eInHJWEGQ2szm7yo5SUwdpAc5zcOP0+CbYK10nH5+PkeG7K+uMhg1TpVHW8GiNwjHdN5d5lFhxYgdlDnZ7hTfVQs93ATqclLEkkkJGmrCIWs42UHaUd7BjcWNpeJzwxgOYD1+nG+WnqXv6bltp2+PXP3o3HLKBFY04OMdq5iIHYEzmNyZlCtOrOFclsUdzVLwWfY9ObFhPKKhofnRlPQS7j+UNu7btGt+q5l0XOhLR/JP5q0r0iIVQx1W5L76qUYZkd+Tj8QvoTsjV43dLzp56K5A4YGLoRaOEinq4BoyJJAluphbcBZwDVREiT7ealgn2nFjJqpYVGHq1JLoEDZaSEfFzA9vmb9FgHoJt45a4L25sW05M/ez/emoT8fd/0rG/r+f/De//B137/v+0/bTdxff/dvfpw/v/5/hu8f6vPfzov8+G+td5WqAXCP0nOl4SjiLUr+LK/FydL4FJhF1hEjAm2a0dSuh3WPULvacmb1T4WNPr2Xq6uEKD8dlihyEpBzcalLBTZ3t+GDH5JnA9zCXILf/08tX7zhM0boY/Z4tWns/G8zr8YAsZDJ1+ad6c32tTh7rulBb5apm8EgCTzQSJ5SkhmyujGU6iNLyakxk8XM6XMHNAwdRTJkZssL4v6J4DtfpHSavV0uJLjI2cAmVVtlR1jjRS6Kfhf6O3RKoPcKSjASwVvjWqs8iNGaJLk+d91MlFcbZKVOzY+XwNbJCKlTKiK80o/wj3KF2umew/aSsRPrpSW2UF2coVtqSp3EyeWGE/QQ46OK5dc8bo5vxawTtqd0c3U7Qc459FTcPQBSJQdFaspi1FOZSunwRUPJTBFP6TnqnoQEoBQGnRmOiSR9Y6UYnZIzmj+ZC8QMk04UFNLKpWa0EDgzDPXWsPglpCPr+8vIYMjrPk9zt/JMljH6Ia1dTIgRC+N8pkLwBknnM48o4F8JibhkWjf58lbe9JfpEth+wAvdNGW6FHsU5GOgGjc0RjETdxi2UGVDAjv2PRkDfjmlml3rVdsGgo0t51gAYYGMdHAD8sKY+ud61+HB1iFJcvnDIm0Euw+f0V9TzULSg+K87rUbPVHt8op5aP3SFSwJgQ9Cr1wAFzPssu1DiwfKSWh6QNxbNb5Z3BGzXLbzOSNCm0rdVqx9McTh+4rK1Os3RVJLBEs+TiPJ9kZBtZZKtER0TnEC1w7kzpGCn43RC4dVJm1TTWKJcwcY3uRKNi720rIQEiDQcfwfmFz0ttuHV80Oaq6md49UhaA0BJ0Ye8KE7ncILPZ/mw7hUl//6GNFEFD4RbPtg5MgYOlTBIHmahU+YP2CV94rd+hAS/S4NiNV84pY4/oq87vxjnOwXf0y/3FgADOctWPa4TBK7B07dXU7uZvAfqyfA0SkZpBpPoxTJqWFyhCVOIMl+vFmvA6Xx5ZB3cuKdmJJBWCVGJhFTm3eThDvmODPAMqQV3p3dtu+UHdaoM5hQL4hRfC8aaupgVVlxQ0+KdV/FDLzJXwFZU4TF5CbL4FcQZDlD2WRyVQ42Osh0CHdoJykm8J4Nm/XcjLBvZRRx+yk0WEzkoZkCoz+eMYqyQSpJtTYPUbVQLue9n9oKDLz7i8PEhOsqwGPZtzyc4lcX81fDfMeyEwVa3c+UiFJ8JKNy2BKd1keY8uQHZ9ZwT2GEnmk8wsQLV6aWpqF01FwlKwuiV8Ij+589rr2x68fOpRIxIxGtaxqViofHzeIheZYhi73zvqb8j2nhh0saYc2Xx5hw6PV/ceg8JNAEuouQkav11ns/quNGA0PaMBlr8JP9SqOTwMtFOC7BGu447T7uHTwY4Acp9mjlXDOM+ys+yAsmTuoC3uJLqLe1zrKpiyC9P457TUP5JzBBKOClAPMspj3TgVjphO+3uAfDR+A9cAk5rNX+PUF9aivwTPIeVVvnn2SX/siOlC7jxC2A11mis5hKt35KvhH6SFh58oNp8VbgcZotV8pKy6JEShwypR0nyG9iF6dkUBjabK8cEe5AFTHSOivWnV8m3P75E+fKyaNllojfPd1fFKpseX+b+Cf5yRujMjkfUU2hCYgY62gdWqHbkHee1xdXqfD5L9qbJIke9YQa0lyW7rRNiBHZthQbDhHHwNOhZITrPw0c9sIF2ledq35niLSVsINW5Z4kVZ1S88Xo8iG36Gtv4p+UNYl/y8rsCzdhXQEOIzI5zMq1VnkM8LkKhhAFl5BQwTRfZKUWhLLhhQg7Fyzfl/VmlDayCo5OsFRaCjGIxyVeR9GWG3jG0L0bKAgQ5nRcZK9mrS7OWNJE6lRGV6JJ3YPbYUzC5taHbCSCivrr47N+1HPNNcu2OVSTQGP2Aosg+8hB71/6g74k71B3vOctmp0H2XxgzIEPujsVm0lB6zsBsphmQPx5RX1/5Avfmn7ZseqgMH40mR1e/1RhNEe70CXo/U+sQPtJNAA4JNpjqh0uLVehfKKI8E8HfnmY+uryw7pmwAHt0JCUAkikGRy1rdGLeDg9kBvRK9ApdLKnumLSzoeYJLb1H4y64+Z9OsoG+nPOU678CP6mKsi+IBQpzPa+/H7Irz2ELufX2oog03WyNlrF07aU+zCF8jGVoRHR8tBjXNyJReicSYMidk4et6eyqrqdIe2AhV0B6Zmyi8OiCsxGqNAuHw5zgAK7RuIznJtQUcdrw8n3/Y2qnMAHgZZD3Y+sG1iHoZeurxNOwoviPocz5+GqAbA8qviojKBXbqdS5LHfCeEA2XWo42DdAlgkKqdKPkxrmcDQJvRR4wjnlW3nBrFrg3OQFpL6ar17M17ORcQilHJnpiSKaItD/2oGtHe4aSVaPXlgoXlVRd7uBdIiOz3rgNNfq4jtLHdsbhFWRN5BtuYSaHhizRcp0ynU6Ru6g9r55c2zMzFku+NuATWJYSQosUFacG8qbY9hQ4BdgqRHkfHnlsxnhWG/nJc0dBfnDAOZQ8TTSnxVyP9YBiUZkx2RM61ZFqZ/+0TR1K33kuP0azTNuh60xcJbxskfeBpN8BEcQRpZS9yG5cVEOX7XrgYZcs5YE77OwPN02nJLkURDBCPdwyi+cdggnLw7+VsNXFOJdWqe5iEVvDBHiG+12my1RHOC13FuSAA1Or1aZFtDIOaFRMwtCXqge4RMg4ZR8AiTtD6DnsvceaHTNkpAVIAwIDeHqDfiHvaYJes6lN7jWCm/ENXekiZpSYtOvI02iwJBa8rceQzPofO31q+kr18gROxSasD+40dI+5AJ2Zje7hfRGhwDQMyRbGOnB1pyLhkZbdVBRQA1eUz7B1V2Drh3CvJdcbW97fAknjebOqnKEj0TxbmddJYpE4TFRPvspriLGINlOOtcVexxi1mR+Nih5taA3MrQMvuOxyX+3ph8gBbUoSXxEHCzgcQ6jmX9QwcCxNG8nxDovwiFTiiPrZk0LRCWvBESES9i58yNyaRB2Hh0/r+GbDEa6Er1qkQt7o0GIT4xelY2uvBwCzOcDRbMqdVe5VM9aDtMaPHqFb5WxVfU3IWKEpXDOI6kYoWo/GpBH5ypRP6+3c5zMl/yYX0bvRdHh+Xr2oZD4SxoLQD5nI4rHzk7t+VJ/0je2a6R8Zxu7kSC1bNRWbG9TT0rvb1NPWy3yS9e6CB/GWFEjSB7lxTBdYsYpnk+ohOaXCB7StFzwqFpu6fLI8QsjXQphWEzl3MsgPTRaKZGvaYJVGn4NrTlN281W1corjfC+OeKDAOWk8LtRIoCq2FmK1yIhXwbQFCc2TBfpkJkcCYBwjfV00oKQC5ttslPbnj2xnUre2n2pnmkQVAsJhFvaLhgU7ARx5yXCPOtRSD6BVI1wzKfAv39wUvmQ60lkPBFgXYtsqWIhaBpXoBiD7o5RGa7sIP2AeDTNZ3UDrulPQrC2VOtZ \ No newline at end of file From 5575b0fc2fd4f200a0cd2fecd6db6d41f3235e46 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:41:25 -0700 Subject: [PATCH 13/20] Stage corrected Level 0 source archive part 2c --- .level0-update/part01c | 1 + 1 file changed, 1 insertion(+) create mode 100644 .level0-update/part01c diff --git a/.level0-update/part01c b/.level0-update/part01c new file mode 100644 index 0000000..b999684 --- /dev/null +++ b/.level0-update/part01c @@ -0,0 +1 @@ +LJy63PeqPY05lHhyhHUjBtHxvuJEY4WS4mZBRPFOUFZu2hg8PaDf+eMOh/eb5FWGguQJ3EhwBxbz9XKYGcRIhkv0KZEmxRDdkuTjfMg9bSXJ+/NwFL9J1jPsHRxL43F+iRR5lE3yUxQ6ZpMrS0OSlC9UmpS0AlDl5CZEakCN2CSULEEvhl1lK8/YH90e1eeO/sz5o54WxGOcWMcAIz9h7wWSLfQBgpsCuNR0haZpSELheKN0OOdIfU4evEMVrqVOJbzGKK3ulN88E/ogID0ocyDPF/rpPVwt5K7FMvksy1vgSPJp9PYOl/TFGhAv5ysrvmoA6uHjhOVrmC5pTtWXNJtO9K69Xt00LRCdSf27idzeFf9Cp4aZgOgUbS+4VUcVO5WPBPvkAZSpcAmM612LP1wVLYZRroKlx3LS7Tdufkv3EXoAoLvIp8uL1U0Cn8Bw1+Ndy2G5KCQYcj3ydu/fKWy+c6cXe8jOvtuWe6FviRt9azVd2FmCExhY1iHs6RlpjIQHRaO1mtN10W2gUdKwIfo9L8OU56HaYs6dNNbI5rHPMfjjYpIOfQC6mWaZeENbmpVIvd37UUxMdxST0QlPU4/8e2NUGH0knwfdKNlCOK2uVf5DohPKV7N+QWHLFAYy7iMjiRNZjiDqSFKxoJB9ZjnyWQDZlkqDa2vxV3KErvZaEG/c1EapP9wNKYpzDKRSaVT8IhRyOcewJFILKCaIhijjHdEDc3hj62UHuZwMFsUdKQwXOUM41lBms57llxRDWYdZxv/UncuxxV8harZi8JaRg9t9HFZpKekKyqBNL6jSaD1dFELWiWfzbNXrNsmsaoAPFSwpsL33hdjyRSS68Sz0mPjenpWf8PJXema4Snf+efBlUoMpJIWRiJBN0QXUB4ZzGUVrlvQ0lfjAUCAWVCgrM1cEdy9vjXGpmZH+GbF13ClAMyoKK5Uu6YgtyjOACNoShpu8AxF0H/eOqLkTN7EfKa2e/PzyKjlWg58C/QqcGitvXgj9Kiaj/8uSa01/Vev0V7+cCHsGrlJexxCcUDCBFbCV3XFpESgmKCvkeAq0iCMjS99o6vV3HJ+cqcHeKF9Wl1PLL9qyt4qSKhNY0smeeNwvKUcrs2fC8mwVl/wjQr5VDVykW1YBEjesDGJe1jmm3neoOZmf7WmR6Z4SmereKomann6UiivD7/JQ6pitY5rHxLGetFiByFieFe+OEROf0cmtooOgkzx9fm0Xur2csOoO65dUFQahTu/rSU1pYe1lo7X7Vif7jrKHvJhPkLewgaOUAlyRzGdwYzRInTggDWOtF6ZpgjWZxng4kqToYGTmfbLpFbYUxY1DFRQU1MSPUaX4C8cFmu5l+d2ciJSKgEsxHlQNI97nWPAUHL5I3GD17uMKx3y3LyxmqWwaC/FdmM7qUijigYr17JfjOMV+wW1fb87XZ2fAjY2B59INiNaA1RIFappJU1HtpH/P2h9eDP7w+o/HHEW27oNtNDbUxRP/3fH7dxyLuQSKjicNdYvaRpB/+OmbjdDO16fVgOzh+Yefvv/+5avvXzz/9liCNgUqm/C43FiP37/84f3rH45fqWjU3718q/rtrofWDdXd1qGtdaTBGBy4dW+JDlIL1dfiVQe3YVuUgNc8JvdK+UnBPzrRY2gf0IlB8qB6lMGPqQc4LyH4VSkqyY9bCHXOdc97+kdYxHn+jqqnW2CksFwPVBea4u3DkyAu0qKwAAJPOCW2BuOKCw1rpPCZkU5Im5FXvEBfTnyhiTjc/QT1/LDD+mVR5vAdwFzb1Dk7ROru6CgqjUdLRG2hRpVSlJVJIwMJ0wJHLhct0/bVH1uU4MgqNSWdHvrKBSzMtU3azniCm9Att3etqQlesBkvaa40W5TVl5ktiop7jCNNiy5F4H/FRd5gp5LSuybijsG+r2h673r/+FXr/tM03F3/n6rfxgYAP88O4E64Un0FjuPMpmtwFHs2XIXL0GjzdfjG3VV4/ERMB/QXmhD4U2c00KunxR1+b5s5cQbf22JC/GH3Ns8GfupG1GNOmP8IngTK9J7MRNoa/tFnjuSe+SUyBV/e20IgYJnz3kaBgGDPe5sFAv5e6PkJohOVMxa7y/VKb3mSYXPPsp2dHI3KEZswTFIvqQ0GKKYYDJTmKcss/t78/7Bv6M8U/+fgyRMb/+dgn/z/dPc7D/5/Psd3C/8/U+OLB3cSuWbIbAQgneSF95F/tGYzcsQz81Nb4/WMBDnpBAu8wBA/FiA7gfj+zftvWd5M+8bKOk3UHgxQTHk2xrPOA66EcqQLfEg+UInostJPw9DJOq3T/YpFKhwm2Ampwi3mqWeGwBekPBuwV7PCidRiPFt8m66LdIIB2J6vVqgfMp/VYTr+OB+tJ1ksMo0KX4KR3c2ESL8TfowUVyuB3MpTTOgv1B84ci+aNH6h7lVVLGknijR1A21aFfCeaMkvQCB7ol9uARsbGkuov9wif0MD6r+iktes9SNQrnRZt9CaifyNa9TDBPzh9fXD/YD5eD9gYJT3A2iZFTmwPmYaAdp3/EddzGhQ6SxH930qkof3JjZNiw/eVYB38WqZT3SEpVlWUAN2I3Kf5d+shccVcGs0Gq2PeXYRstidJv6vGpzLI7vdW+BDUYG7qyc0oqmgtREfz5cX6XKk9telGyKsJBgYfjYaN5p9/20NF8xsMMlmZ6tz6OM5kFG44SQmZJeph7uB9f17ttjjx3LnmLJ/074FGNnrl7GZquyIgNq0Tdt5aJkQW3WY6q5FiA+66Q+fvemPuumPn61pSSvP04JUvV40k1oB91cUEs1pW47Ww9Ug1eTaN4PHUAgvWlU1Qgz/Wyg3+RARKYVJAHM2wB3Zs6Zx8lMbfLDoOfSUfLvB38Yxu3bUGkLIi8GQzqhqGRbe4kvu2chKopsD4yX8zdV73Ep43y5YhRGWAf5OT6GGe/82c4bK5X9Lfp98EGu218VY140GefJCo5a/LVd1s8yNUkDmdwtnDlZpnE8m4aL8D5oiLHJy1EzwfwGC+SmR8ErsvKm2l8/GtUbZ9Pk9BPyZj1eoJm1SgV7m095ep6qSWt5wKBZKkOViRpivUaTnIEzVQK7kFMOSfbTbCrOu/F3XosgxZ+v5uqjfeotr0rkT9kW613DOwbpzxtavGiJK4I9vfin2ixodD0uPdPStse2xHvAGfuVtAW3mDD75gJTL4CwATf6L1lk2WdfV5ACNbybpAnIuc9jSWa+2SmfntYZYoW/wyP9F12gy66i5xZvCKwyQue10IjmGujGmHmoE7XTv2s50gs7SEFkJ7CevEQcE/dIOoq5nAlYk2FGmJHRDF+xSwR17VftFlwgq8p3ATfaCFPDsHus/aXbtrVFuEn97BZENYqB8ZjQKCjF+W76bA91yaZ66H4FtrZ8wyuNK+1FXZZiwfoBeg/Fd8UsFcai8fsiZJFjEWwseipEJNccZSWgtJTIBSV6jCG014ot01znIu4akiTBo+DneisTEuWwYdBzbowC30Aj7IyLcNpS/xTfzZjLN0lmvTY6bVqOe7FhjC5iwC1q3Bqcu41I+4DQll0EB1uywh+UqV4U+RcPefDjNVufzkd122GsMVwIodWSxK+IlzXUKwjWaSd1gQdPZBo1G9Zxz/ZJ5wQVvbGjYtMteSRQ8xLZyXQzdCQ4AWhd1IoTSQVbL1OSjy1iMaPzYtWThZsc8VAhfYw6kWEXxsjgImBzUKRn510fi4N1izwxtFKRpk1RHAzEWdbaqEOdogshGgtj9VIV99vmxUfYxH2Y97DD/tEAuNQIj9auH2O7hdR1gNPRJE1LkuumTbeG2NAJ7ROlweEnBf04vSnIL1g1JdQ5EFS7YCQRMpmOEI+U4quq9aJFx2ABYBJiSq5Bv574g94qLX9/rNHUSLlAdLz8Rjp1bF7VK7x0mfijCbKo4opiBe2S2nsqwkHxkP0I9kOFkPcoG4Zq4AlDaAPlMvNGzFjNbXItQROtpNqk3wqjINoqyCkzpkIuK \ No newline at end of file From fd4f6eb5cb1b24cbbe32e61bf3228050ddf1442a Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:42:15 -0700 Subject: [PATCH 14/20] Stage corrected Level 0 source archive part 3a --- .level0-update/part02a | 1 + 1 file changed, 1 insertion(+) create mode 100644 .level0-update/part02a diff --git a/.level0-update/part02a b/.level0-update/part02a new file mode 100644 index 0000000..6f03eb8 --- /dev/null +++ b/.level0-update/part02a @@ -0,0 +1 @@ +bjij5Ub3emWIpMiT7oY/N1RduBJEfY8lxmDBQ6mwXgU5vhRtd461qklXX2xurnRUVTgx0QU1HqvApgadM+zpUrtYVYjt64csMzQe7CXjGm/oa4JCbsvd92TuUiu7XAVRA/E7CVLwq49r19zEzeDnwX+tqd7x8cpCqghihjV/cGt+2L7mn9yaH7ev+dqtqZmBreoCxz14+coAwJN/PNy65uuf3jtVS5rtb9istGD3917YelwMl/liVTy+N5DBh698Tw8PS97/+Ld5/3ty8F/anc5B5/C/JIe/XJfs95/8/c+u/3I9G8BRBZzGfbexYf07+/sdb/33DzoP8V8+y/ebf3q8LpaPT/PZ42z2ETW5z3dQY3svW89RJykbp/lkZ+fdt29fvnmP+qG92j/Xh6ME/jvKl+Qqo/bP1988f/eHwbvXP7399vik3b+pNWrJv/xLsrgYNWo7b4/fvB68ff36valpgT1utURJyjTFazs7qtb1q+evXn8PNX48/tPxj23KPdp7vJouHs/S2fxssdpTio3wswsn3Os371/+8eV/O8bOXneO9iioG6S/Oz7+DpO6R3uAdE8h5bvjP7389jjSBmdA1fVqjuXQxqikOybvaO+f8R+KnnSDA3/304/v35VVk9m6JjBt68kKbaF3nr/9/l0PT2RthZJETE8o24TahNkzI69RVgFMPU44jJsTmCuHJB6fSoT+7pHKMKTrwXCW6pDOlX2u7TRQueXkJAnH9vpPx29/fvvyPUxg+6ZGei+dWtLv/xYF/uioF0f3Za8Onf+YLUkvqbEzznd2sksO+/Pn9394/eqnV9/89OLF8VtYtM6O1pRbo7Kcp6/CgmnsB8I9+T1g4K+nUfOP9bn0n0LfIdLc6ymw6fx/erjv0X/4b/uB/n+O7++a/iPdehchnZTONLzZbXefNg/aX38t6X6sks20B8LLF+96u81dct+b7C2TvTRB2IPnb98+/3Pyu9/9ThHPd9GiBqJT3rYDI8AbnKXPTKO8akSufpuM5jvK3QDSbC5p+yILJUmMaYPyNk42/IFgkIaPUDpB/4muf+vx4gpvInCvba3m08kvgWPV+n/A7bW7dv8fdmD/d5/sdx/2/+f4Tk7X+WS0V5COe39H+Zij2LY1oAPrxWo+nxTPek++Qn9uF+dZNqn1d7jSaTr8kM1QfC+KtiiPFG4B/08UbvV3VFDsmsex6SO8tqMMurFMu9VttWs7o4zRXKXiixryQis466MOZBE0pFpzRNxOFCWU5McYMbVmRrinGAoA/Ky33+pQe+iWKpsN80wH962RvPNZr9vqktZTjcIJPut1Wt0DTlgA6LTAEm1OmKarxWS+muSnCPepKnWF7BpMI5Xq24mhWKao+LgnW+/vKE1+o/lNLXS+Jqd6yqjrWa8N4Ps7KdS+KnJeMxZnXeCjebakEq1DrPTXNYwXUzrYAxjHKcwkTPnqWe8ppPR3VhxJ4gRNEuDns95XlGw7qkhOfydiRISz6PNk5NME1ZtrugZzaZGilKHK7pwgGrUEQi0AzdKzrGiN89movwM4uMwYP5fD2q8YOvU/xNd6jKv9S0p/bif/eXKwj/Kf9pOnD/Kfz/Hp9SeLCrMh7y30L33V53+3c7h/YOL/duj8P+jCleDh/P8M3y30/2VYXwzmWxqvNxZ6VyXxaYlpi5FJowPHNRvYefP82x+ef39MNwFt2Dkgy8LBAC06yay93mgpI+WTTn8H7VDZRfisgIOt3maLaAfS44RODVRWsUZ64iQqDQ0cL87u7bkwv54EAb2Ms+eIyTHnhWZZzZ1GvD2yy9ENAqvTtPoz8Qp0JUiMAceHbGAuCUW8Bh/S7pjeqWevd8pvHfebwI3zSxgMDPpUjRRvJFZP0CoFvYDSXjAnzRKi1I6fpXR4ol6y36UE4zmMPOJWKDnEXNbGwiL77yknneTLpA7V6sPzdJkOV+iq6Itkv8MvkiZRe87tK18zq3x2ZcapfXAYJ0fcTNyBUehLx7flXc/yLax4q0oZ+13lkDleSthcRqFJp0O8SD6gUhduXF8aAB6hEU7oFMhmd7pfRfwAleTfCEdFtCnCWXW8Ie13PdhC0eEo8RtWSkfYaJBDpi6xDNSNwgx/kErbC7ICDesa6qVAhmeXwIO3ukGBlzo5eo2D4QRYzVmcfA/82RJIWzocrqcDuPktikiRaXpZmpfh0mkzyMh0UD51ICutXtk/WvsNUCB5+GFB7vwqujLJ0iVtUnw051Vo+2WmcDUcRAoGU4Ll0vXlgARIsRohaNgCk6XKDHIv0uV0vTDTHGQTAgxG2TC9YhB+iVO4Z3co6+tITpdzDv3pXxT5hPZ9J9vzkZ8wYzjJF5gdYCxtDBiQ6VNYxIqhgDLQRPmEgaZxOkcfOetpvItUZIZuYpfzj8EGoCIk4TpKnvpYMZ8ugE0It5XcNfrOHO4a5wJdsjedMlW4twR2Zz5lDI/3RvsuU3zBgM4VckfiBX20akRVZtwUvvGklp4OAejZef7XD5PpbL7427JYrT9eXF79a62fPEqeHAjFg4WJZIGfNfR2Duy6NMmW5t4OUReG3Q61lkbcTkZgr/33fzLeaGNvxQfg0JRbHtWFQV4AgBR5jnw4wHNzcLrIDIelXdNEvXj5zKp686vh79DlHFrgApfmuvhyvO7Q05vlr0oqKOd/VFhGbPSK80FLxQRP45yzZIQr9plzzDq4Ig7ZAydVHbBuoj5cJfKUnawlp2rpiXoTG6s5VmF6xEFFgz/YVCE4XKnaV5uqebSVKnWUhbFXzdAvqGaJTB91Dtn8WGAnLRt1fwAlBy7xKobzBXqzywYzYrWvKLhPgKx88eiRar+5cmiVPNU3dhrg6Q1S8LinA7jPD7oHzgygR3auonTYgqm1JXy9uwYV7h6YawQJP+k+oSN4NFlTvqR+X7ajsLTdJm02rEUgY0X2hTKXW1A4aA/uSzyKhj82XaXFvZJj0s16WrE17avPqyqXm1kTtK5DPJqjmuGIMYqWP7sECjjMV4hnZ0vILgKCFL/bVODBo0ceidBe0szLV8+/fPKUNFWjjZO2syJCGdxU0arTlNAisb4zo9dsa0djOnE5pz5f5igLZ8/AZP1SNQ83LmFzaIrlvBxqkU4mdadZmFR6Q4c5Jqdnagtv07xzovhShAE7/S+0f2gdgBvOTVpV8htkNivXQUwsDy8ehBQ/FMP1Q4h32iLT+sYXNNqPCN4RFfwA4PvSX4rGdAGVdpbqcLgbsKePoUOlpRbZcojW3nAYtdpflBbT7vk77aK8DLAkVfmzOfCoF3pRYJF7+25hsaTECAiuTYmCSMiGLEO6AiI+pNUkaVF9NV2QI/kmBrgpTBRP4edpa06QuL5OWzJxGvidOb+vonzfQZzrO/iPwPPRUJXbIPc+EnUX9KQt8RzWEEWs5O+ClpPc0gH7ki21E2KNVXE3fRR+i84AwC8NDd2xhlvED29RWUWvwxZ8oy26gWMUBcs87xOvQs5QovXKHeL3Mc6QrGQ2Cnm8xCXm8Ap+ZEUkxU9KK8LKlVb7qrw5jGm/VTU8uPFKYKq3zibz03rtETnTVwemFhMjlpR4qbRUIfQx6aCRgXXiuvJm/t048P6qqZ1zHzSN6+2DG0m55hMO+Ugd4NvNX+noMJ1RpMmZGumVc0NMgGtX1Lhejb/awwbxodqRMX5Vu2mIPXWRQ1P8fNAi46aibq2bmhwQFV12BzFnpdeGDfPsHMtC2g0oOSeeOR19REbFMkzL2Zk+iRWdxghkbDGFV6B4ADJT/SybsdNJY231vU4B9JqmM7ImzGD2O919FVtvhf0/zeAAwlM/BIXxNQdUCkAMJ/OZjj80zpf07O/L8t2Y7ZbQ2atQT8jG7C1PngpKcCcLYrd7B/oaqDpP5PLX7YPaMjzdwPukk3r1JDadOXeApNKbhISYwgZgG8BT9cu1WEtZ1InZ9AOJ9r/mizqtUVPNk1/FB6kr+cD0thHIvLqY0/XUYi5t02Kgj42BDVo2 \ No newline at end of file From bc8de5c420d19ffa031889cc94ab4752b92104af Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:42:55 -0700 Subject: [PATCH 15/20] Stage corrected Level 0 source archive part 3b --- .level0-update/part02b | 1 + 1 file changed, 1 insertion(+) create mode 100644 .level0-update/part02b diff --git a/.level0-update/part02b b/.level0-update/part02b new file mode 100644 index 0000000..98161a4 --- /dev/null +++ b/.level0-update/part02b @@ -0,0 +1 @@ +wLNnnA6Bw/D2PNEE5Qxabn+SLVAJpUMcK6SyuFwoAjOwVVyKNT4DyQcmMdHRO4vLdRConvkl2V3bw578Q9Rny8bacLE2UUjUrYkmi3WURFzuOvYWhwjd1ROqyaF1SNvQe8CEsIyA0LkDIfAG6rrMh0UpRIphFgdHWRsBqHz0ez/i3GHxUYPQlSFJBydPZ/mYVYgiTercaGuufEPP5klNz9oozivIouZGJeUt3bLSiqUAQjDjkk8Oyoq6wpJplhbAQnG8IarpMCIkr+CZoYvfvpOJZ7/KhRMZuqmuhydtcnDVdS7A5PB6dlWHW9p6OuPQqwUed3V+Gamp50nK5guHWhFKKRyKpPHnxGISt48Vr1Xzzr0Wzqa8GKMhNZkZqtrUMlqJam/ekdKEXOVFNSKc0BPQmi8/QO1Xc+g4yS6RAaGpFd7UGWPNzYhrmbFgrIFaSMw3NSUEg+LkHuigBgN8HR/YqtxCFBM1lovteYrIvlhJh+QbatAY/SqCWs/g0J4uVlcDJChaidFKc4wdxa9In7cRIDnUWzt8jxHnCqLsEONSThAdqh+TC3OXHRQ2J5IN/AU69mvr9mzztR4DqclO5/MPv5wO4G30/w6fkv3n0/39B/2/z/HJ9W93hQFQvriand5PG9X6f539jrD/4fU/7D49fND/+xzf9Q6cQtlkgk9ieCSSMI5SBvriPU2XH0bzCyU4q+X4EFcbfp12R1lX6fzUtOAG39NZWlbjSMgKLPz9m+SljhekFfb//X/+n4Qwjg0VR3l6NpvjqymfOMgREbBIp1DbS7WdXWbDNZ3u5K0BcmfryUR29qvDUed0tP9VeWfZXzdNQj/efaEE+RctQ9SJ88ImlWlF2hJBdWsx0Fpc4S/Sj5ysgoJCoTLIk5qVsbakIuW8wLtsNvtY360wSt1tJrtlRrbaUnW30bBNLOcX9A7Xt0nFejpFKV4kix1+rOiaW5BryDq2ymKx3Uck4hg8eiyuGtDYka2vDm0KzLU6V4qgbrbl5HVgWua7doNL2a5b0fCWbrVNFzEPigpc4/XCsncJPlXOV25rIoKNC01xWMBbrTM3RzyR0aSpYCjQaYoQ2FqS/K++y1MKy9ppuADGS9bElDc9YiEjxU52TXO76K7E/BUtiy3u6rhYpnMeXEQOHROe6jXiq+HeLd1ZkzfKMiRw6/vLUg5BIrHu53W4OGJijsSaRArSrBx5cxIvKO5rUMPcxbycfmVlcWnCK5kDx8+rhmSudQ4Mm7pF7UW2hGm/zFdXIQyRtwUkVLBYpsOrgXrwA3iddjt55EPV5eIw+W4pBybu/0GuD+JGoIra7oglPpkix1tBhKf6ePfV3DzSjHDDFglkZsvkGknhza6Ank6MzIZ3KsfOrmNzzSSHgxOuZuTmh58CAhrMtb6Dc+8F7rG6xOpGi0Lw4mUXbk5yizcVtvZDgPd7RrfTbJim3fannNF4WT5FMQEeokp20kyuJulpNvHJKZyvLbjjrZcoszgjuXG9024mT/zd75jwNhWthDNLLEiLXvBPr+pi4mLkOz07W2Zn6n2HANmaQii12zjhvvdbUAOWA13n0UqscCHQ+iFb8VrXG2ErxWLJ3httcydctYWujmdpHSNbhvVQ36eJWAIzQxMo68vu9ZsOaOodpNE098wMRFrgOYd5O81WFwCpHhZxp8lvd3N57kyyp2Zh+xpfbqgxWZynvXarW5IPCzZf9siC4owepif4XBMp3AjR8JJmDlBA++PmQe9GijIu1xVKh/mrfAWsw3j3mhHo5ojcLib/7/8m6KAQVWpG6XJEcgoi+rE2zpb5qG7GexgpMcnO8AyM5BTn8wuZbn/ZfYlnDlNTQGl7/iTkDW9PecMjb3Syd159cVS4UERGeW1zJEDd7DIdAmsNRzBHr010ZoLMCIUlaVSA4uehiXpyH5ylC69D5O3v3//X/zbPcmpo90s+T7962h6ddg4+hXxeXJTx6cj/VrLpIWsOp6FmOLXSmmZIPyNjG94xsDL33VOExJewR/FrBoHajkc2Rbfkk0X5rXhl/NQ6lbHMig9Rpfxjj8JTId9RrIfDrCjG60nyM83EzzwTiXzlgBkEhqQl8Z+c/btALy4clkS1XM2VcL2TXSI0NGwAsJoPyL1hPgQgSEjrKr/RTCjwcNHbHc6R3fMJF+vOqveQHm3net1RCKUAw7uM86wHiht1Mp+dYS7+1v8yrHy0S/F3dd2LC+gExeR2W6atjspYFyfyIUSMrdGHmVlc+fQSO0ZNNVXvDWtBIA1rIIcWw8wyTubQ52R0ozFuRvRgW24GvyqOhm4mDTMLd+Jl8LsrP0NoUcXTqKvTHZkZO/UbGRp3pkzD2xXejqMpqbWBq6FaGzgb/LbmbvArYfzSy3PSKO+iC2j8VayuJllvd29vl/++yEer855PwnV1zSRZnWLGr3hpxSftupRN0ZJ4FWad0GKYPY6XFKvmjnSpOIekc5lLsgzAToInsnNoQ3LtQ7bE4BaLbKjNlGqjHI689MooR75hJyb76tifpLOzdXpGWezfRGXo8py4X9Nsh6kBW3A8N824xRUM5aAF09lfCvUcwdRmp2N0+r1Sdhvm78E0hxMA7UJ2bv4hnsP+033O+09nUAB/OskG9/sCtOH950l7/6n//nP45MnD+8/n+O76/pN2s4P09Onok99/GOOCB6DYFZLUNxP0+ETWH4/RoZN19oR3v3yGnqFewJ8/Z6d7x6N1635vWVnn4HA4TPfLR/3wkHQPD0nGQx7em0qbePX6/fE3r1//YD37YUNk/iTPeHSipy5VW8DC0ggGfQs6fXr70yuAouwyx7vWi98N3z7JWd/NbmzGtDUNXqx3HVU4bEk+cDVjT1NNLaR2npqaWzxHiVv8NC8KjmRzEtyKTP/UzbGOY31M2UIZSQCDcgre1oLuP6r2UZJgdElJ/wwau4H9owA6Mm/R/bh6IXfUm9NGyVNOqaamghLOfBkkZzniwCIrVgbNLGMlwI2LXQY+rsipZ06+rsYm3nkyOPFevXDKdsnLCsLz7wO71gwUChl1PJvMZDa4CO162pzek4yfG1SXImO3apUweRdV9hz9Qn5yk/WjRTYCijwrRYsEgKLPbhJMpEAA5NPetzwhGTqDXQPF9qCUFJLAbvr3LPBMxyMg9fvDTzmKkQjyDtCvREiRTtwpqO9qWXVMQN30C7uCaSmNDopuKYX+ouHU7W/5kBWIf8i5LApMWRJG4mglq47Kd/g1EA0zlNYynHpU/2ZwzUk38oTR1fBwoFxU4rVjLBEgmUb46Zb/jN+aSX6j++I/SHFFLbehbn6mlxbLCLDrXmYE8ERTc/SLPLHc72bqpk++/uq0c/gpm2krPNy4jia95G2l7wHTUqW2liopKZIvYvKqbYEIUoy0xXOOW3EjdrhDS3BoHohyHKl+aLvjOmhhqElwnBtZkeguXrgSnZlQ5u0Ao0ckEhIPwiZ8sSA3CTXy6Xpq40PtsS39j29v23R6uUXTR6Ll9HLLljfLJyVCVc7fRuTRtfewdlLAqYtR7m6HQCGB+aWIS6czOj192v4k7UvB9CLjkS7zgth4lz11iWVMRcvCARZGbOs9zUrtRvWOXO2rgUWwzVpY5cpXA0nrNithxXSvLIgNOljlqlceiE0qWNtpXnlASzWwbpq3XTJiWKsWybl4VTy1RBbGq3rrVfHq33pJYvU/bT1iELdaDHnMRvfePfMfo9F4eHj4SXK1iwsSVFDsRqWn8NOrzU/9sXdzgvMrPJyfBMoFjgaD7ln/s76sIyjeRRFI9oz9e32ehyJO43nBrvTCvcTCqx+yK5ZZeW+HjIDJOQpN5wwyyUdsHp1pS0x/2HfWDYgycd1m8jR2qVNKBHdUH9CKLfx6L/VAd6teWInT0porCgfUn2poaHG3/JAte7vzXY835wwa1r65s5W9vN7m8dgwbtQHGFU3dv3a7i35lu/IW3FucGmYFfg2CpzkNAWyfMnAMAN9asyXeRa9jm59YxzP \ No newline at end of file From 0c5139703bb90351b1ea828e3b31b573ecb3e4b2 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:43:16 -0700 Subject: [PATCH 16/20] Stage corrected Level 0 source archive part 3c --- .level0-update/part02c | 1 + 1 file changed, 1 insertion(+) create mode 100644 .level0-update/part02c diff --git a/.level0-update/part02c b/.level0-update/part02c new file mode 100644 index 0000000..a2f52e9 --- /dev/null +++ b/.level0-update/part02c @@ -0,0 +1 @@ +gfrgxD5tJjNY9R5gzenp/HIA+zadDc/nS8Ak9GyadBoY83XY210v4NwApnK8io8JHWnBymD48lup/OF/9IO1sktG+bPaiK79skK2prNZUZxPUw0/SCp/OWWRBTqZQ9PcgmmABUOo3492oU55J16H+p7mc6QX/UZrleaT+n678fBm//A9fA/fw/fwPXwP38P38D18D9/D9/A9fA/fw/fw/WN9/x87zv+zALgBAA== \ No newline at end of file From 35ac4da5242a1abbce6ac2a04297c66c6de472a2 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:43:47 -0700 Subject: [PATCH 17/20] Apply locally validated realistic Level 0 tree in CI --- .github/workflows/apply-realistic-level0.yml | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/apply-realistic-level0.yml diff --git a/.github/workflows/apply-realistic-level0.yml b/.github/workflows/apply-realistic-level0.yml new file mode 100644 index 0000000..d1ae601 --- /dev/null +++ b/.github/workflows/apply-realistic-level0.yml @@ -0,0 +1,57 @@ +name: Apply corrected realistic isolated Level 0 baseline + +on: + push: + branches: + - agent/correct-realistic-level0-baseline + +permissions: + contents: write + +jobs: + apply-level0: + if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: agent/correct-realistic-level0-baseline + fetch-depth: 0 + + - name: Reconstruct exact locally validated Level 0 tree + shell: bash + run: | + set -euo pipefail + cat .level0-update/part* | base64 --decode > /tmp/level0.tar.gz + echo '393ca945e35a43ec3a42092af6b8a5ed1fab1bc7c72ace6f5f80d9fb3ad1ac92 /tmp/level0.tar.gz' \ + | sha256sum --check --strict + rm -rf level_0_baseline + mkdir -p level_0_baseline + tar -xzf /tmp/level0.tar.gz -C level_0_baseline + chmod +x level_0_baseline/scripts/*.sh + rm -rf .level0-update + git add -A + git diff --cached --check + python -m compileall -q level_0_baseline/src level_0_baseline/tests + bash -n level_0_baseline/scripts/*.sh + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install isolated Level 0 package and test tooling + run: | + python -m pip install --upgrade pip + python -m pip install --no-cache-dir -e 'level_0_baseline[test]' + + - name: Run isolated Level 0 regression suite + run: pytest -q level_0_baseline/tests/test_baseline.py + + - name: Commit exact validated implementation + shell: bash + run: | + set -euo pipefail + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git commit -m 'Correct isolated Level 0 GPT-2-BPE baseline' + git push origin HEAD:agent/correct-realistic-level0-baseline From 886fa8d34fa10652e21067e0c065cb48467f0e16 Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:46:10 -0700 Subject: [PATCH 18/20] Run Level 0 source applicator on same-repo pull requests --- .github/workflows/apply-realistic-level0.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/apply-realistic-level0.yml b/.github/workflows/apply-realistic-level0.yml index d1ae601..818f478 100644 --- a/.github/workflows/apply-realistic-level0.yml +++ b/.github/workflows/apply-realistic-level0.yml @@ -4,13 +4,24 @@ on: push: branches: - agent/correct-realistic-level0-baseline + pull_request: + branches: + - main + types: + - opened + - synchronize + - reopened permissions: contents: write jobs: apply-level0: - if: github.actor != 'github-actions[bot]' + if: >- + (github.event_name == 'push' && github.actor != 'github-actions[bot]') || + (github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + github.event.pull_request.head.ref == 'agent/correct-realistic-level0-baseline') runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 2bdeb82550fd8226ae0ccfdec0fd6136b3e01e7c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 21:48:06 +0000 Subject: [PATCH 19/20] Correct isolated Level 0 GPT-2-BPE baseline --- .level0-update/part00 | 1 - .level0-update/part01a | 1 - .level0-update/part01b | 1 - .level0-update/part01c | 1 - .level0-update/part02a | 1 - .level0-update/part02b | 1 - .level0-update/part02c | 1 - level_0_baseline/README.md | 123 ++- .../notebooks/01_single_seed.ipynb | 190 +++- level_0_baseline/notebooks/02_multiseed.ipynb | 154 +++- level_0_baseline/scripts/run_multiseed.sh | 0 level_0_baseline/scripts/run_one.sh | 0 level_0_baseline/src/level0_baseline/data.py | 6 +- level_0_baseline/src/level0_baseline/model.py | 6 +- level_0_baseline/src/level0_baseline/optim.py | 13 +- level_0_baseline/src/level0_baseline/train.py | 820 ++++++++++++++++-- level_0_baseline/tests/test_baseline.py | 298 +++++-- 17 files changed, 1396 insertions(+), 221 deletions(-) delete mode 100644 .level0-update/part00 delete mode 100644 .level0-update/part01a delete mode 100644 .level0-update/part01b delete mode 100644 .level0-update/part01c delete mode 100644 .level0-update/part02a delete mode 100644 .level0-update/part02b delete mode 100644 .level0-update/part02c mode change 100644 => 100755 level_0_baseline/scripts/run_multiseed.sh mode change 100644 => 100755 level_0_baseline/scripts/run_one.sh diff --git a/.level0-update/part00 b/.level0-update/part00 deleted file mode 100644 index 50c7a4b..0000000 --- a/.level0-update/part00 +++ /dev/null @@ -1 +0,0 @@ -H4sIAAAAAAAAA+xce3PbtpbP3/4UuOrOVmooWpJfiXvVWSdx08zNa2K32Vmvh6ZISGZDkSxB2tbNZj/7/s4BwJdkO5lJOnNny5nGInBwcHDeBwDrbj/45s8Iz8HeHv/F0/3Lv8d7k53J7v7B/u4+2g8OdkYPxN63J+3Bg1IVfi7EgzxNi7vg7uv/F33c7SBN5tFCfUM9+AL57+8T3Hi8s3fwl/z/jKeWfyyvZDxyV/4y/spzkID3d3dvk//4YLJf2f/uCPY/3tnd2XkgRl+Zjo3P/3P5h37hH24JQX+VLLzEX8pD8Uu5WETJ4mc/kD+/355HibyWs6EMywak1ptDofxlFsvhePTktNGrsjgqDkWR+1HSaM7lVaSiNDkUe493Zjv+o/2DyeNH/nz2KJC7k/3Jnj85GMn5ZBSGgR9OHs0fY3CRfpBJ9E+ZH4pFVkwIXbHKQGYZJcV4nyBoGo/h1KEY72vFQseVH9fNVWshVbHWvExDGRMrrtLAn3kKE4LK0WTvAG2zOA0+mLbJHk2ZeLG/IpJ2+eVS+qH9LZcz/B5PHhGleZqlJTgxcmnmWeRjyrkfK0l0RNK7ltHislDEqlJu8ULAeaJj5hfBpZmTEC9yP/T8ICiXnipkhiE0wdK/sa8TvTpJiwZnZI4ftATbxgglA7aaGnMwZ9rAuwSMn8GHLAXWBuYJo46lnxPJXu4XktcJayfCyhQ8Wuuc2C6/vPH80F9ebwAyGCDTODcN3HLt58sys+sd83I1A71QBv6KYMfEOln4Y3p5bF4m/LJHy85UFJMCjt2RHD6yfA3iKOM2WhEJFjNbnLo1zYpoqbWQybbrWKZLmRTlspqBWxMwUubplZGrEEpKUoqdHVKnIF1mUSytJviJH69UpA6r5Vwz9/NqdKt1TQS5n4TpkoWoEX6+/bvb746Pnr06dpfhN/Mxd/v/8Wh3f2L8/wFeyf+Pxwd/+f8/5flOvFBpDLsLxUtKAMRIJH6SPn97Kp7AY8dw/Vtbp5eREqqcFbmUAj/9RERJKDOJf5LCEX4ZRoU/i2U1dmbGuuJFIcJUKpGkhYiWWZoXoriUIpdZqqIizVffK/H+/fDt82dC3mQyj8iaxDxHHLpO8w8izYWfZfHKABFCJbIyx3CmpUgFTA2TR+pS+CLIZRgRJUew0fcw5WRR+gs5ZO9ekYUVhGIpfVXmUrxn23qvbUsbv1CZDOCL4TvmKUD8MIR/onVXXgDUFogfiGXu1tZ334mnaZ5jDPg4a/BNgv4IodSPQavhc0XEtc+sfDT5xzDzacGwakekiRxqImarQgom3BUsgiuZ04SiVGCoD9ePAZEfgzlLojKXPrhQRIF45QdP0vTDUAU+WKEjdQnvSpF3a2sosihJQMnPoOI9AvtxWIqLRhy/ECot80D+CFBIczgRT94e2zDMWET/oog+cMuFoyNmGfv5CtHGQcQc0Mjx/ith45kjxq8oGEchD3dYAGiieKMRQ8GgDSAKTLnQgf2CsOwSjkRBDEtiCcVh5aDVL8B+JoVCL5quo7C4pKjr0IILiIciNaE4AEk7zmT3kajYrAAOaATgUE8vELQlS3kb4TorC+NyFSHQusQjEBLSEnwKTb/gGOFwEIlIcymQZHrFo9GQIpWJWnrNAbQeoudRhHoZBXnKsRZr4hkeCY7xYCevzoZ6Z/x4YjkFM2koYpmBqVqlx/vuzqPdVxYuhcKIiQMna4B4NfPoRoZO04K1mLZr+WyzXDJQBkUrLn2yYbZgP7zyk0CyCVvZinevnxNeHkO5Q6kpTxMopq/Nfc4mQCTWkwxhBtpi6uyCCUzQV4W0sGOfNlaKdM6YJxsUhCwjjwLQTjSQ8UjYdb6Q61JmyBtyJPImiMsQ883zdEksjtIQpmQ8QVxN7Gq7DuXcL+PC+AubwgwphUEGEBcREmB0APEckoS9bMgqLgaueEnN15EyOsHu9cqPYnan2kHAL8ZREIH7s9iYD8kCeQxkBx7LhIBDSJQ5Uvle9ksvEnjHOLad8oY8BIT2NE1CH0OvojxNyOdubV1cXGDs5VbAXX5QRFe0mutrD5oQ7oxHW0Eo/nf7mVSw/Gyb3PO2cfj8F8n5sPbhpqL0Rl7lELNVcQnFGIK9UQaKNGVDKb53z6hCcCyPHdKl8++JIl7EW7+4VJrvQeVnSeLgrHGGKFEExWi4Lr1QP2cB5P71kB3pHOxSIvAT4t0MQgIjfLJ84NFebmgLjVDosqjiCFZFgev10es3gPReHv92/HLkvXvz5nS6XSyzavW6hh5ylbJ5zLOj0yM98N82YNumeW+b7fjk15enJ3cMzqWC4qmaawiyfq6NdYPHByezUtWLNMRnetSQSBH/jexyONQBRGzYKTAAprwTjUrR9MADzShS67c4XQxt8grzJ0VT8JOaYpKumTzn/CBHpA9T+ELSJsdEIu3aHCFjP1PkvaFsDpaYp+XiEhbtiOPTI+1rqQtWYv2VQvriL2Gw2rWFZWCdv+I0xXgmUhKk5zA0ssDKXpgdhDWXrHFR8SPCiUKwGg6hQYG8oFQkl7MyimGKBazvCDlTAC+k5mVsVqY9o5ldaQ2jULV1mxqxRmyzq3VnKKXvhsMSPgOKjOszwBAoffd3lSaVQr0ryavb3Ioqmlp73G0V5FEGs8/LxAOUS/kYFUpc9LACTH7697H4HwQKKXjqxpRDBh0STgJ3oSm1VkAeS836OfMejKbQdp1HHPdqISFyyGVWrARogNtPBSkbZwmcJUEEsQ9tiIqGbXcs6c1vx+/ev3txejwdi3vWVBOIbsHUqDpwgH2IQUpH/TrAORsjIAHpGMlR1Ix1OrHv6clv2o1p/b4gqkhXY6Q0LKkLUSKg55+jV8ZZbOsSnPju1WsyslaAp9BXy1vdyreT4+NnJ1NC4UxGkwNnd/T4MYTe5e7b0xevXvzX8buTqeYjqUWbzRw8aS4wW1PzCgU1ZLeEDTSDI/ltX1BOndc5NZtujlw7uUPEHSIcqtjvpQI8OaLw9E/JNtDW/s9w2fdJ4hYcr9+cHj958+YfNdWa6PvASR4sjq3fS8RdrmxmlDfIGSoDtT0ae3CNi1iy7N0oWyWzWqEtnMjiFF64mx+KOFVQQug6dO8mKpAAyxskDIjCN8WwEUw5l839gDJkmZAy2QJi4SMjtmmToLTJMRZAVU6pNppJ2zjIBjrJYZxdomhbtVJCneLpbOhJSgYSCrUEdWS2ernVq7ZUpeNFOi+uKX4iFlxyJvkjPAclaJwJCxVQwh/NYetahLdr3Kuj//ROTo/fnkwn6zbx5Oj06S/eCUQLl7PW+/zd0TPv6OnTX18ZDBtgjn87eum9eH16/A4/NgG8P37x/JfT9zQR9Ge0wS4bXm/dJG/xfJxiREsq+5BNw/Ik3MOvCPaXMg6HaEI1Dj0Zgkt5mq1YXrXKUHLLmUmOCB1lEHynWLc4XXHMutVWKJKDThJ0zQh5cD5B1ISRv0hSroRpTnWZlojJJu/THgKDEiQWUCSODLoapHIdgCgbOWUcNupaO637BXtr/woPBJ0H3/gM+IvO/yaTB6PxaG/81/nvn/Jo+etwVBVqX1kfvuT8f3dvTOd/e7uTv+T/Zzyb5a/rjWz1dea4e/9/f+dg96Da/9/fpfP/vQnU4K/9/z/h4c0uz5uXRZlLz7Nb9LxLwskXQrptyxcInEra90Bd2Z9cJZrfSLcu7e9U2V8ovArKx+y73t+zb+qyLKK4eltVw6iE32IaqSQNULtz7W0IUqizCt1NOVoczWwXbRjpjmJFu7G2/ShBJvoC2TBVENXKkAlknI8kWTUxSvrL1oubJAyClJERu2ZXxMDEKZ0icpPDG1HKgPHGgQGyWwye2ecIPd7w0YA66zGQyM14x+MpYzQQvOcrKj5/kF61Cwwxvfv1tXeCBO/VkYds7uTFm9diKiZbW1uhnItQXkWB9PyySPsDMfzJrEk307mjENHcNAZl6LuR8qpSqz/QEPTkEpqStIb3ezSiN2hjmfkB8rVQuctMfRk2DDDINk+WlejWy+K6FRlcXx+vosDmxb1Gxqon0XrmUjfDaLxJ5m7u0DMt/aT0Y6/Rp2drHVR7ftGnzXmPNtJveG6n2hM/FKSaZ6rIHdK5c6ZqDh0pNFnNg2xICWP7duhZr9nZO9eEVcf8a9BVjwVtUQlwnrYxoNVf4Ufnslx69w3WZ/J2FIRds0D8vX0+3xVyG/UPosE98VCMB2KbVtkfOy00myb6adq49tCdZuNKjC7AoWFNzYmH7clqGmqGDzfQY45xpmLk7tFSxu4IayDXB7eg+vwji9DDcw5a2ryZ1Q8tTmCrltQGGd6xNquiHnnIvGC/4i31HQ7jljRa2tXj3e2OhjpmXfpST7evY1Xs06YG+qxHr71z7qFNmQ9yRccN9TJ6zdtFPWe9XSPa1MP3iDZ12JtEzb7qolBrAN0VMg0NzwONsqxwF7Log+iB+NtUtBpqcK0/dEzzmx+X8jjP07zf6uXV96xj1yyaR6iExUeg+lv+qT4HZ7lwuWmPZun0hk7iUBf3Wli15tCpCm2AaHaQD/jYWDUZZ48dUJ9mPes1r0PBVBvMQABqQdb3o9pwtNHSRllfmaogP1nTbDGyxzRaWOZph/yGyd7J0Q4zebCgu0rKHkduZKTeJwjSMilUzczKQPQ5t4coou849SuNPqS4QDsIub/SK/xB/2nekiI3rxsbV8KqRr3JhShyaELJc9tgbKigw+Mz3XcKBqFDNN/ONXPKjM54p3AACQsAYxsT4mVsea8h/w5PdAdba05qJtJ1jkvaOsnpdDfpclAfotJMNqgnIR+nTg2tFDxJN3hyR/RrBjkDp2bCtPql0dxUCJB0Bx9qaZ+15K5BKOXxODNbtzPIyVcsJ62eRAv784HQSmteHjaYdu7oa4NTDAbI/u5gg6HZh9yYDhBRYldfAZxv1QNW32BJa330dNeJiNld67iz3s14WjxYh/kqXDGR7sYRK2t2ReqZBK42qcP7zUETaNLUViL4ZQZFpEBUPG2LRBcJsUFIx0ceMxChdXqalxLKvLobwCyPk3G+WOHxrYmv6lSqq5hVS5XtGibQtZ+zezlhWFEZZaW6lZMyC53qHHtDHtzgXK3gt7jUpua29axe+rThOdogFSOm9c82yAY3UwPUiktK65HCwmktZN8wU/efQ3z/Yaq71KM7PP0By9NcY5FNaeoLwigA3VdpWMaGGgY4rGrKz5DCvSpdJ1+cfZ/bkkF51ZWbqbkXZhtqAl0ivW+rmQLiS+KYE9WRyfD4/gS1GBhES4+jZd2Wrw5b/KskS3zUK26JwlhXw8irEY5ZaduTxOkiorNQOkeyi+kTljYY4htDIF7VyWfz0ZHuHYiPljbWNa4ghZHe9jcq6zO2XnsOu/SVSwcClnX2qVn40BZDhAMCg7u/7A9cLLQ/GCBrZzwdzJrVD3XJ1teLdv18QTVGGC2nQ1Q9U0w9cFW5BLIO1g4ltaAeThvT8alZ3JBYQzX6Ta0x5SHI94hFku3fLk8XPo1JHJRkTXtvZJzMxcMGpkbmWB/wAILrIGR/fRQu/QlU0GkMGjQTTnu+gkGWbRspagyZgZkeptOpZoseHoupwfH+xB0NujUCUW8q2xp9ldWaiGWv9HlJmi/79dXBprGbjSH3re09b2yuaIPXciEcpOm116wQujRRLXpN18DlWSdtP1aNYTusKGoWNW209rSSrMd4PANHrUzULZsxBRPfh8w2bMXEpHIL9wpiSnPNn2YSxHgRO9M8nE6qxNtc1P9ybjb2TdQfJdJYYiQZTM0b5lmNdY2b4ocfxMRaWce47mNsu3gntVJ/5EXfkFIX3kW6jAKPtkL7tBt5yJuQDrCtaG9w445Q7dQKSbt6dNI45b1Ml04kuV7uz3vuR26it09uscyMB6vGuHz3xKMLHzVPiBA3RLKp+oYEfeczKaYTR0C2hYe6VHEa4wh4kPQa8yXTn+kLgobFyCRI6cbktFcW8+EjW0nzv6lyzZ2afkWMw/QTW3i7Vngn5gblSeJnqDqKfhVBTX3N7PPgogrPQ54Rzx0ba5+/PW3U4KqEtUN+FWwtQ75cqZo8xiQvo0T6+TlVy58qSApMOUdQrW4L2ktIFh40t5ANjK0YaNWEJOCYy9GkKdrT2iui+osg1R+sx6kgzeiq8bQma3OWr1EjN/fm0qctebU5izdwaVncA0gfHGmR3lIONBK++4qBeiGutmXe23bRtPLuXI6FtkZZZdRmXlOUmCKJLHZnsj71fcTk8o8yAiPYcXt9rcdrY7SMzkiS57yFRShqX9CNpfQY19ZSFKNGNX7SW1cj12LWKv4MCtnXzfUOBHNFa4vHNWGeXvfx39q+Gysc1JrzezRrr4Ef1mncUObSEz339xQRtiamyAmf3cdyRK+HHKW5N4cYniz0dpzo2b+aoig0/sXuPHPCTLGL4Co7IDRMHkcT00AUnXf8JY0+G51zMieTvmnhxGcsJIQE6gxrGp85tXJuOg3RDodvbXthlBv32qyoaH+2USQ1voqapWm8llwThw/X811zvNL64ooOfq6vGUTeBDIrxAuG4oyTOtHa3X5pJaUtfeq1Ly3R3XNzg3tWFvpDGX01WoY/VrekaYumuvsOLud+Y29L31kHEVuGET5fi5ny+ZkLrznXKQ65Tw1hfDEl7Wv+mZlu4LSqajl3NcDRddUtTvB8nbXkAKKYEF1fuy0m6DmnlqyB6+urdm3GVRKdkkD71WtthYPuXHCltB/eR+bZI/3oOZyL089GCgBbIarsGCpjyG7TnPK/aQ/pKFKZZuHQwd6ictz2o72muXd87NntvsBp8V4bL5q5kAW155tKXb6oNm0YCfLgea/97SCffXxkUxkdhJ/cQF311pYFBqCd0xidNNxMO+50rSjQSyUOH7Ihdpmgb0Kjk/ONdictyOw284anpqJZHzAYkcPYc6as221uhHtKBoDaoPliaC2jHvmpadTH/IcKR2PSQnzXuWHYuEuGrFV1voSha8cNaXDW1dqs/2wubQjZPWnJ8/hI45DOtmUfrZQSkZp43q0joEPKX0jDPhrz9bhHTyv9bWG+Uxslecc1nSQkHUuxKewmrc9JcdqrOTMXbFky5x0ZzqFd8I7tg5Y5i2XKVHw6FB/XeUutaPjUoYwuZE/VSqH2CbGcTmdcqstpW+XXjMisbXMxT3cCrFu0CUR16u8t/SSaSwVXVV0EMNmCTlq42X1jO8/r3cJ2HDSBkAzxcGM/Of/6hK8mgJ2D0/gaDf5J8s1KypJqqgbtDaVFnpaZHcwvtwx0uRjzGER1z+NAretn9BHbevr5ca2FFaNDOVS9u5b7xnFVY42var7NBCsEjRVjcHP9d8FzZmbA9elao92B0urXjw18n3qDW1BWda02avIFzSK6+9TVtNkYW6+UedozjViZE+Ducxs1VMNFlC/k1WZMY5mNXsdOw/cObkMX51792d0mjG0Ah76/uxVZ8/8psAlXq9+hvdVNqD61WlrbCKS69YHklfTqL0S0QBo7CM0Udy0f/jyz35gf33bWT336BPmu00yCmtGhcH1P36MNOMOtzQANCrr7H7rGAitqheyETbqJrsCnpe+ZD6EhmvVLT524xhyjrUidpVLxppO7bgCsWQnos9oBNYe0nR9ZQD3qvBvLdZSvss1Ot7nycGjE0Ont8F8Xnb3Drlx498LsXHTx6wK1Hrlpt6MzZpM8K/Xf1HkfggYPNnU1hn+qf2aV0lcVszm+9kgFkEjwtuOqjze7yfaDU38Rpsu9jn6h+KTkjL8UQ3HC3wX4CeP4v/a+bbuNI0lwn/kVNejxEJBBCABJyWY3dFq2KbeO3ZJWktunl43FKQIFslq4NQoQyeFwzz7u8579kf2F/ZT5ko1LXiIvVQApWu6eYZ1uC8xL5C0yMjIyLi2oswSo9YarHoI8ngXqXUTwhvcCTv5jAliqDoImJabDSTpZZunoiuzTyZiJjLmArYBiN7/1GBKa0DU0s7dnjd3wLeZ8jgaVKdz5suJcmZ4UpHIYUx3Bj9UrW8spOnDAIStaBGOffsCR4+TOVloWSJM0mH9wXjhx6jWfW99IPxS5Ii0k7Bmxncm/CYKm+l2Sq1/HML2X1FBtUbFd3jJDrmKVacERwpGlYPxUptWUdJI6qIgiKNUjKGWULeuPHukaTDyUshu+gAbad5ioC1gega7JPWK1bVGTDeVbk/mFvYwr8yb1dkYXUamtmWv1TRSZ8Fyw3MTRi5TPbFZipaRpqAoGpx9upCGpTdVr53CE1ezIjBZlY8cunbpQ4qzWzVo2bL5W/IEycdXWugHTRLuufHw1OE+L86xQuIWQqlTWnFaaaiWNSqBWTzIlTmrWeRPMcvLMWeSW8OzkCWukrgzr4QrbH6O3NSSHCitrpzyyvqnUVNI7v6ugpVxhzRataTadpovwdZznGW9H11T4Bq1zvWuG0dtg3xhuJna5V5PSBfd5Q2n9oNyPtcWarArWVJpeXPpmR5MFveZykza4gy6S37Dl6DX+94bJTzmxFjS6Z34pUpReDHjeaSPW5bKRvNjbI96GozdkVx+6bkC6aOO/4kNNU1LjFUmX9MZljk95bIIZY0FIeFQAXqcr3PC495qJqRFVInRfrJXaNVcgtzAoy0YPUiwSzItkPTP600IMpafMAWBHrrayxzTcWu0DVYva6jGYHv+F2zCPIPJkea7FalYv+qQmkvtKAsf9I8h6HQNVGru3FB2vCX5L6JP4/ROFrEaJQxRi6iVKSaTn0nrpF81Rl8RJ6nXa7Y58GxO8zvZDw0359zawrjcwUgW9xWohgfn8Ywo82YUD2xcDU8cm6gaQTgE57ukJHWYXuoPFpsyjpKSzooQLx/fpp6HdWulfC4Jc1eBbX5lqsCTIlEKxmjIKQxv1wekC6Ll2LzX42JUK1qU3mRqTFSVz9Cm4o5MB5Ej0kMnTQKdImUpNGxJBMf2zpX/IC01E2mHoIgo0BvaV36mGKn7GS9DgFkAcKq+cCQ3IyVkuQRLjY49pT7OdTlo1YYYzaMExPJ/AzbgRFNeMD1RxWSU5uQap7dmtLmYewrvTjA5jvCoF6fQo1IuU1gg6MFo2/pZ6FK1vVbaQiqzmwzle2b37Px8TcHET10jEU0F9Btr7hsc8aX32GolXPRgDg5lo4x/U1EPCa7P24cYvNbbX8WcN2TFynoFjMur6dF5ojfwu/6G6iYRJXIjFTyGHZJEolK4WB8eUrGALjknAxhNIrhtgyRadAfpvsDueFauksr8j5kdG8XFS040qsb0hRYqosruGAZlCuBpQ6kEuQFWYK8esQz5NxBHby3EtnEQGWi+xqLS8CKDnhiKMiCQcCVKF+luQZ3TcghxPlU3k4wHmt4Rp8XYwJ9YKHYOVbbjeOQZnqbMwdEyh4pZMFPpcKpnv1TFBkbEkq+WzsWJgYxIhKLfHpg3opslVwRNKrO4jknqe3fRwQa9XA3IKVf4wjtsbGlY+RnzhBqsS8TVBCajZxWIvtA1sW66Xrk7bWOFViBcj7ydOAeehxL1yCPG5WQeVJsTs/QZxJ2JUO4rjd57X/Kc1mlWUs1jh09hSqZ5/X2TOi++LWvno2mMNbhxY9ojtXWvScrLrHcy7/aPmDftKxFLaNrDpgvIOo961l+CXN8caFQuKy8MMqoq9Ufom6L8HKhpJ7kcMPWXco5f5Rmu+yGb12gXQ1Vl2gcb5vRr89pXyGvhifQ7n2kSINuh+jdc9gNRCRaSfKaHO5ZpsnUbqBT2HSDc8CKxeiG42aZu4kgWcB6NcbxlWNOh0sVCZcKIMq53oml8kkZuI9hCHKOmU7kXtPvXHpFS73er5OvxKFiAum01f8hZqiwnyVAo4qIQftxbN8i+GJRp+bte20c3TLvGi1E0/3xMBDKqqE2/pHtDyc8iGtQAOipVTsNLyG4hZUC98Z4QTDwXYMIAYJUY1csKfvQSYrHZDEOYAkkv/A90N/ZXrcJgSkpnhgkAvShlup6qri6H+qm5Gc0FHSfmDbrwPSs9et5J0sr2v42tb+vTqHGVHYjVKKpSxYUdo4Fw3uFjWXhmLRneTzdUF+3bkEo0TtmwosZwLObygusirBiLMHTwQJqcaQGD84IHx8suAGfbyKELqNkyHx4bGIWwxIw7TGoeycVIiDG4c0i3mJcISV+2u8vkDuhNFs1ttMsuCH0V442o9BlXNscCwcgwpCNmoxYCf5ApQW9GlmSHtYH6jRbyP5B/0B5xBxeT9Lq5mEB1wydWjHPpWUNTFBP+Jlo/pLvhf/BjRn9FSsKtSrqkQVN54FpmS4UREt8kGKJWqC/K7Kc9SvK6IK4Kz3lqsIlYI+EXQhj2C5qTNwDJS6bMjulqUqc/K+WykVnW7IzCjd874Evt9eRw0Fa2meGKnS8+UEWbQPL74Gjsud24iXR0o44Bx7Rr+Omp1xjdFDRvEYQirMKWBv559mM0vZqEKQkS7Un/+VRC42bNlBjsuhIKf1LCE69LjsnuaW0ffveLMS2VNc87Dzc2hv7uYCFfG1sG4vLo+GnvXEfKxNYTFYhIHYA9GBNPdAAYOwN51p92GGYgB0+ejAvVFKSyF4r1r9YPxAnGid62RxldwNbWrFF1NoYjCq/68PewKcUZkjxoiGr3Fan0MbQBR65+4isSx6wiq+JSYJdmr50bIwW00CjF2Q92wTwNL52A+wjMuKvcqVYH1apQfTvapvIpel2fSvq4+fjzTjdi0myLOS538YjZhZTKGCkHB6TJLP7gV8ZEMtQFG5AOx5B5JIyy5O2oxYAhoS9Gf/rYSAcrCtxIF6u8WIkEWBfoAbGPwf+U6AfWNVvMBOmHXqmOyijRLD8csXDRsfCKNGVgaDwW9Tf4o9FeqTiA/8Vpd8QJcUvcWj9jO4hjHFp7mxjZCoEpXDBvlXYONLhnw2+iWAb+oootWytvGMQN+5GwBttBmlCB/gxfpcuTfMKCz/gY0Ed9YR6wdDsCE89G6MrNZC3UoixZWG5jLWInda/xyVXZilndu04Ij71bZec+JQcmlz4EQfXjRFIJTK/1U3IqASTXrbOEvnY5tMJJXMBQt+wvsFfSlypHohfo0D7sU18f3P76/jCXoa7cXbMpUeqFhg8cNJ6+dup79WSIvxjPY7UG8IBOhXtXdzSM7m8kQfrG7dy+WuF11Gk8ssZSvLF+48LJJegDu4pQuyIZFoI5GlBpK5nnrub3DfN5yDtW8sU6EJ/YueTwxql8lZwnDiknPtZQipv5RZnLgCpZOrLTS92lo3lOkOLIchhRKlkASAslyOFYs6ftONOMI5jYUqnIFt+uRanFJKleWvY1UjYlPuWIgM41Vj8tMyxVCHCTQWiFOosI8Qk27K81hi9aQVbs3FBVN0wVM+ZDWzndmoQL5kXaPdIXhaR5TqwMhdvN6F1gDKPS/n41jEuN7R+f6BjWye5RXhs/OvnIoaAAnIiTsB/LS7TfgXRqTuL6pyWCn3qVBu0OizcktHV34rXZ1Sc0tN3ZJ7Vvs7RIIt93eBozYkd5m97FZ7XfFnqCBvYPehm0J9OjuoMFqzWyOPIucptxQ1Bj+E1Wuq1J2dB92rcy54nW3zDDOn6cTb6f3K0D46ChPJqeMBMJUOEIXJHWuBuFiUWkvbG7YfnhCbqgm+jOcT0h0OVAV89m6MBqcwQPdpnHhQ8HmCSynCZ/UGzHWaD9EfrQHrjRQhTOmYIn4RpjN6lFxYTkE9IuAnqRihuEUPImcrhAU5elZuatQvoU4C6980YajY9ja7QTprWwiTUFAvFpTEB1twVKpz6arO4pgALdHRpqJsz+F5poRxTsFdvuuQlm4d2IgTGb4wDD2d1Dv2kWqoPLdtNLwv9pUHYOkKSe3KLlz7WopIghKS3V0kNbz5RlF7HxDOUrSx8Va6Wg0SFV+vaYDi6IDMI6m26tFYozWKkGgtvseac1XFlOWbFuUtOdHE21t0cdxD45ZjH2G9nIYK1DzgCUQ6HBpkoMIFBpWd59NJMQMkH1ndQ+17RxUwyi9yPJS \ No newline at end of file diff --git a/.level0-update/part01a b/.level0-update/part01a deleted file mode 100644 index 8bf27af..0000000 --- a/.level0-update/part01a +++ /dev/null @@ -1 +0,0 @@ -8OzBCpbQGCKeFeRLkEDQPwikqJuzWNm9isAldSzQ8owsKc1MipWXhFa1jokro4So6cAjszLxClkNlo1sNUSKvMgISmYR+IpLAVfqxqwZUwJ7ZQvdYr6xuegRaPNnQuSMwev4BraWNIvkijLFratyZHWl9sotetp91kKSZ0//KTcmUzAy497ZydF3InseIfPgwQC36WBQ4ynlPVsetC4e/0kJqu8pAFR1/KfOYfvw0MR/aneeYPyn/cMnD/GfPsd3i/hP6OdRBHa6ddilb9PJxAm7hMR9Z+e74xfPf/rxPQVJ1XbHtbJgqTXrEoIoGrEk5PKXbKbocMI063QXFbiIRFApOPqW+aLu2t+TAa8pCezLda1DptFITeHfK2CJ4B+g+zeBs1xjzRUB0sZaYxQ4kJPFOQEZjyNQSCqhxBGeOfq4ls+IIaVxZeksUcO9pn//aXlj5kQRQdfLIc6n8e5EDv9ohucFsnDZ7GO9FgklDj2Vy+LGr5H2hWythv8Ink4ZpG1ox0RCr3G8LOTfqKZj5abJpwLn8KQVYxCBdwV4DUz4CxRNDVPgVjf3+9vncPP0IXNd3XPjQppi3pAdd304Dl1lYMhS3GOEu030zal+qSWGQqEbE1VH+YlMAC4eOPRXQc4iZ/jmNz47UQUx7gU026hweTCu6eAf7DYdDuZpXmB4YNPatfrB+EZHqWjiBLre13tMB8sSfAV7M7JOPko9cZIhBJk8sM8I9GmwnbEDNkf7HKlKq0jH2YDEiFxObHRnnrjW5hnyJoiWfQpHBwaUTVH6iNTOTkzEbgipJ5z22QJ/qHYVKwZYBof2QlbiKAL0k/6jaecJeYFVpk19V4oSCciNN1DLSTWV3INcYErML4uPHVSXvDGS23IYJuJxAMOaLm/qhw2MHABxnAtUQ/EDKAe3cgHYTQ+emd1840jLJyReB34M59EzLWTBZdVsvnwVA6OCsW2u//Pzt3/86U3Jgjjh5jbNJkeSHnx3/O3zP4eQXLdom3rlhK0OgLmmQJv6RbAIYY7DETpuALaHZFBvO3QJvF7cCVveH797H4xmuw6ELg/u1gVo9dsf3ryGlXFWZ7tOxN7a77hvnv/5+C2gvUW27XrAvnQnS42HrgjOOqfb0Pwr7gAhE7/8IBOn3fhuQKJXgz8cP//Or4smdFtUPf7jN0HVbHq6ueo3P77+9gdLME11q3C05f5WkeLDOTf6et6cu8qYbqZg1bea+J9/rkC7rTpwG8xjYaJ1aV1XPA2xY81kmBarBvsmpUMancVNHRf/wvs5MI1QLF8CR+fOgMsHGr5PtFG3ddk3fUMxB8avlmKloL7DjcPfiuGKlNwQdIIMOBWDR86hiP9uJhZzJBU1U+8O/tP5URrNJ/OkBE4Hphy7USm1nyRKV8+4lCG9xI1DF3HG4brKtuqqO5qhtHEPhSswDHpYo/tqFU/JUfco2A1wrj/iLTdpJ1oIk+h4BtbH2FGi7sBB2xzkkttlB2H30jKBPUoURGujbqIIOJEm2ZuF+EsGjXSRRntusxEK2o2GF8WwBC2oGobUvDEcOHtZ+ZhpLHC6KDzC+aRQkHNLnTWxDTvMagK37zHXK+2y38KJ7gFs1y/cZOpin5a5MtyjclBDYEyToxwjpsIlIjm9SnQJghjrg+tI73fJYbt7+HRzm7Yat2td5gGF3+sG/vJGy/livl4ZFUM7xzWVpf3ryhscJODU68q/Q2e+m/umi1PHCiArxfgq8QFpHNILhGdJ4Vgm1KI8XtVtQd55bKLH4HoZIQtXwWBWsn4VLFmwo51RBxvA8fJ2uz1gqm6zDVxvct79hJqsRn9T3wmzrduczWez7CwtbzaIwI0Bp8v9b929J8UUg3yoGKwWpOmUqxqs3Fh7V0faGxvjvxoYbkTp8mWgXQbzHOuCuXaatj3Fm1JTB0a6rTrKjZgeIinH6HESXr96przb6F6ndSvUEbWrUSfWNixjPqHn51uuj6pYsTKx5qzO+K3HaapWDNK9UQVR4RVietcudrDu45MHCualsx3euvU8tOgIVJBOd7lnUmqFgZC08F+zrNfe6+7NVj2yOu5GEIhQkPklKA3tJ4gprnLhpzlIiaLhrSURM0dPHBXVrEtSsngz7W1COQMy3okYBv7aj1UP371/8fdfHWHvfl6Aq99/2+1Ot63ef588PTjs4PsvFn94//0MX61We5dNxnvKJT3cBo2bT3MvxHdY4N7N/bBVw4Btwj0numhvt7qtdu2BRPyDffH9z/FD7kn9Y5P+x5Mn7adm/3fhN+z/7uHhw/7/HN8t9D+WGat2oM4rC94KXVwpr6Fcr1T9Q4f5NfDI1gPoCEut3zx///747SvU3MqsJ/PafyeBTfGXVv0voy8bf2mRskM0ZDvaIS+QsRt8zNPBLLtYQQeH5+vJvx7yzURbLx454Zn5IsyW4gkboRyqYBJhFGdgx4wN5GyUT1EQ061isv4InCD1bI+6lqwXKKG1AjZ8O14t80vFvl5C66YBFbpYyylnBXBi9MJ92SrO00WGMTWfmT86JrKzLWt7dknV3ptGLpPHSf2SY0w30KIy23uq9AabyWkzGUKZ/dbBwcEhXCgOWk+fHgKXDQR+v3NoRAbCYpumT9zrYAxTauX3plXdcpo8gn+/TOqn8IPKfQmtPUrq9Pv3lASsLFTcdjxKEH6JYQKshSpKLm0QYlyHejQ2UiwGsWmGghGbvzjMVXXA6uTfol5ILZBH9udkqUxW2LNpV9oioq7neirzvz60+fSWsJx/NDFQXKsJeXd1mlBSH/cVQA0/pTAyQbBCCs/le3Kr6Q5SYCX+6ZXQXYQS+mf0ycYE2ZJ/2pLW+1QQ/ZmXw2jOaiOTgDjo5SULYw4vPZzMUVe+h5Mg0FaZvqtcjryi/ghd5jiueBxHDBQKuNwHw6YoaqFJcjSse7nlPfJy+WwdOm1w4ETol/xKaNkyG2Y56tymKCfYY+ploUZM+DV+DE7X4zGpitP8kKHdianYx2g1aiVLvA15gErsqnn1keQWg0n+IbNB4Tda00d628KgOdozhkH6foN0suvuooTAmNoDvi5Iwh0flwsEAZc7avF6F58B/NLJ4jztBf2OV4h7e6SzjuqbrRx3l0a7wevZFjOtjsJe5cEtZ7AMBK8ROvDqAIUT82lOycdBYgddmzwCkngY9W6hBu5QqMi+xM9Cpm50kj3HucqjOKywVQuHUIuH1lQLuRf1u6oOPqRaWuGQxWQ2zmEd35KVUqF67OL4e4k8HOhAoF+GzVGSvp73ssItdpQb7ql2uOKwcC1KppaNEIuSLBx+haNwlUouRgVaNMx6pxGMkbvziBwu19VA0J0r/F/AQrmZEThC8y2yHCpQqbCu3mdtIAYU94XhGWrSuYxs/NEj1eyONxFWH1k5pFbmTo7DYYAcvMUrHWU1mJ54YhyOz1oqXQpg7yh9tboNzqmjXgAhYeTEwggCG1nM1GwrnazudgjOnCt0Je0zEyqcqWuMSTn2BDmyDXplNKfgnWHP4AzzVYC8YKDxfWG2RFNOsAxKqBeXJLvqMN+4rtUhdgkIsLJci/U8STWGl7XflHE8HdayT4HhzEWrjuV4hcmH9yaIqDhaDsK139uMzQJN8OkQDjdhHXhS02RO9cGkuwvTFxtdzQtpEmuPZrqWRQ5B/WjQTnGU/OsqhGlGicY69drKF5fi2rYIhFxfA2BmWRM7sIbYagVxhHOMm133B6uViZpyLnuTdHo6ShPMO0rq+A+cYXBiNRP9u62t/J0rF/Y56gsu9D2qeU2xlyJuWZj313P3SAwwUtYJ57tlHX+bVhb2rgtxJslfXu/URZJmVowZFx1SQn4RTtGNx1zSeO16lyDvRtrZnc35z90bJNPXdqxHrSfjm9BhY6wTdlsS9itSWpyUoKHH79m7lKNBprysaxtPN4qeE0GvnNTFww5vG4nzroEzYZb9qhggkx7zglPcdWsvO/wcKvwcLmkl1Y+szmTZ20Rdwkqn2SotenW/ \ No newline at end of file diff --git a/.level0-update/part01b b/.level0-update/part01b deleted file mode 100644 index 2d725bb..0000000 --- a/.level0-update/part01b +++ /dev/null @@ -1 +0,0 @@ -IiZ3BP10c7rxONgY/MIvrh+i/bYtHe2XTCWqktFDZqXhyHoG9/IFkTcL4Cjxw37o11AEOHCPQNbIvcOpl64v80meLq8Gv8SZ+gkHokBi7zhyRUkZR+eInHJWEGQ2szm7yo5SUwdpAc5zcOP0+CbYK10nH5+PkeG7K+uMhg1TpVHW8GiNwjHdN5d5lFhxYgdlDnZ7hTfVQs93ATqclLEkkkJGmrCIWs42UHaUd7BjcWNpeJzwxgOYD1+nG+WnqXv6bltp2+PXP3o3HLKBFY04OMdq5iIHYEzmNyZlCtOrOFclsUdzVLwWfY9ObFhPKKhofnRlPQS7j+UNu7btGt+q5l0XOhLR/JP5q0r0iIVQx1W5L76qUYZkd+Tj8QvoTsjV43dLzp56K5A4YGLoRaOEinq4BoyJJAluphbcBZwDVREiT7ealgn2nFjJqpYVGHq1JLoEDZaSEfFzA9vmb9FgHoJt45a4L25sW05M/ez/emoT8fd/0rG/r+f/De//B137/v+0/bTdxff/dvfpw/v/5/hu8f6vPfzov8+G+td5WqAXCP0nOl4SjiLUr+LK/FydL4FJhF1hEjAm2a0dSuh3WPULvacmb1T4WNPr2Xq6uEKD8dlihyEpBzcalLBTZ3t+GDH5JnA9zCXILf/08tX7zhM0boY/Z4tWns/G8zr8YAsZDJ1+ad6c32tTh7rulBb5apm8EgCTzQSJ5SkhmyujGU6iNLyakxk8XM6XMHNAwdRTJkZssL4v6J4DtfpHSavV0uJLjI2cAmVVtlR1jjRS6Kfhf6O3RKoPcKSjASwVvjWqs8iNGaJLk+d91MlFcbZKVOzY+XwNbJCKlTKiK80o/wj3KF2umew/aSsRPrpSW2UF2coVtqSp3EyeWGE/QQ46OK5dc8bo5vxawTtqd0c3U7Qc459FTcPQBSJQdFaspi1FOZSunwRUPJTBFP6TnqnoQEoBQGnRmOiSR9Y6UYnZIzmj+ZC8QMk04UFNLKpWa0EDgzDPXWsPglpCPr+8vIYMjrPk9zt/JMljH6Ia1dTIgRC+N8pkLwBknnM48o4F8JibhkWjf58lbe9JfpEth+wAvdNGW6FHsU5GOgGjc0RjETdxi2UGVDAjv2PRkDfjmlml3rVdsGgo0t51gAYYGMdHAD8sKY+ud61+HB1iFJcvnDIm0Euw+f0V9TzULSg+K87rUbPVHt8op5aP3SFSwJgQ9Cr1wAFzPssu1DiwfKSWh6QNxbNb5Z3BGzXLbzOSNCm0rdVqx9McTh+4rK1Os3RVJLBEs+TiPJ9kZBtZZKtER0TnEC1w7kzpGCn43RC4dVJm1TTWKJcwcY3uRKNi720rIQEiDQcfwfmFz0ttuHV80Oaq6md49UhaA0BJ0Ye8KE7ncILPZ/mw7hUl//6GNFEFD4RbPtg5MgYOlTBIHmahU+YP2CV94rd+hAS/S4NiNV84pY4/oq87vxjnOwXf0y/3FgADOctWPa4TBK7B07dXU7uZvAfqyfA0SkZpBpPoxTJqWFyhCVOIMl+vFmvA6Xx5ZB3cuKdmJJBWCVGJhFTm3eThDvmODPAMqQV3p3dtu+UHdaoM5hQL4hRfC8aaupgVVlxQ0+KdV/FDLzJXwFZU4TF5CbL4FcQZDlD2WRyVQ42Osh0CHdoJykm8J4Nm/XcjLBvZRRx+yk0WEzkoZkCoz+eMYqyQSpJtTYPUbVQLue9n9oKDLz7i8PEhOsqwGPZtzyc4lcX81fDfMeyEwVa3c+UiFJ8JKNy2BKd1keY8uQHZ9ZwT2GEnmk8wsQLV6aWpqF01FwlKwuiV8Ij+589rr2x68fOpRIxIxGtaxqViofHzeIheZYhi73zvqb8j2nhh0saYc2Xx5hw6PV/ceg8JNAEuouQkav11ns/quNGA0PaMBlr8JP9SqOTwMtFOC7BGu447T7uHTwY4Acp9mjlXDOM+ys+yAsmTuoC3uJLqLe1zrKpiyC9P457TUP5JzBBKOClAPMspj3TgVjphO+3uAfDR+A9cAk5rNX+PUF9aivwTPIeVVvnn2SX/siOlC7jxC2A11mis5hKt35KvhH6SFh58oNp8VbgcZotV8pKy6JEShwypR0nyG9iF6dkUBjabK8cEe5AFTHSOivWnV8m3P75E+fKyaNllojfPd1fFKpseX+b+Cf5yRujMjkfUU2hCYgY62gdWqHbkHee1xdXqfD5L9qbJIke9YQa0lyW7rRNiBHZthQbDhHHwNOhZITrPw0c9sIF2ledq35niLSVsINW5Z4kVZ1S88Xo8iG36Gtv4p+UNYl/y8rsCzdhXQEOIzI5zMq1VnkM8LkKhhAFl5BQwTRfZKUWhLLhhQg7Fyzfl/VmlDayCo5OsFRaCjGIxyVeR9GWG3jG0L0bKAgQ5nRcZK9mrS7OWNJE6lRGV6JJ3YPbYUzC5taHbCSCivrr47N+1HPNNcu2OVSTQGP2Aosg+8hB71/6g74k71B3vOctmp0H2XxgzIEPujsVm0lB6zsBsphmQPx5RX1/5Avfmn7ZseqgMH40mR1e/1RhNEe70CXo/U+sQPtJNAA4JNpjqh0uLVehfKKI8E8HfnmY+uryw7pmwAHt0JCUAkikGRy1rdGLeDg9kBvRK9ApdLKnumLSzoeYJLb1H4y64+Z9OsoG+nPOU678CP6mKsi+IBQpzPa+/H7Irz2ELufX2oog03WyNlrF07aU+zCF8jGVoRHR8tBjXNyJReicSYMidk4et6eyqrqdIe2AhV0B6Zmyi8OiCsxGqNAuHw5zgAK7RuIznJtQUcdrw8n3/Y2qnMAHgZZD3Y+sG1iHoZeurxNOwoviPocz5+GqAbA8qviojKBXbqdS5LHfCeEA2XWo42DdAlgkKqdKPkxrmcDQJvRR4wjnlW3nBrFrg3OQFpL6ar17M17ORcQilHJnpiSKaItD/2oGtHe4aSVaPXlgoXlVRd7uBdIiOz3rgNNfq4jtLHdsbhFWRN5BtuYSaHhizRcp0ynU6Ru6g9r55c2zMzFku+NuATWJYSQosUFacG8qbY9hQ4BdgqRHkfHnlsxnhWG/nJc0dBfnDAOZQ8TTSnxVyP9YBiUZkx2RM61ZFqZ/+0TR1K33kuP0azTNuh60xcJbxskfeBpN8BEcQRpZS9yG5cVEOX7XrgYZcs5YE77OwPN02nJLkURDBCPdwyi+cdggnLw7+VsNXFOJdWqe5iEVvDBHiG+12my1RHOC13FuSAA1Or1aZFtDIOaFRMwtCXqge4RMg4ZR8AiTtD6DnsvceaHTNkpAVIAwIDeHqDfiHvaYJes6lN7jWCm/ENXekiZpSYtOvI02iwJBa8rceQzPofO31q+kr18gROxSasD+40dI+5AJ2Zje7hfRGhwDQMyRbGOnB1pyLhkZbdVBRQA1eUz7B1V2Drh3CvJdcbW97fAknjebOqnKEj0TxbmddJYpE4TFRPvspriLGINlOOtcVexxi1mR+Nih5taA3MrQMvuOxyX+3ph8gBbUoSXxEHCzgcQ6jmX9QwcCxNG8nxDovwiFTiiPrZk0LRCWvBESES9i58yNyaRB2Hh0/r+GbDEa6Er1qkQt7o0GIT4xelY2uvBwCzOcDRbMqdVe5VM9aDtMaPHqFb5WxVfU3IWKEpXDOI6kYoWo/GpBH5ypRP6+3c5zMl/yYX0bvRdHh+Xr2oZD4SxoLQD5nI4rHzk7t+VJ/0je2a6R8Zxu7kSC1bNRWbG9TT0rvb1NPWy3yS9e6CB/GWFEjSB7lxTBdYsYpnk+ohOaXCB7StFzwqFpu6fLI8QsjXQphWEzl3MsgPTRaKZGvaYJVGn4NrTlN281W1corjfC+OeKDAOWk8LtRIoCq2FmK1yIhXwbQFCc2TBfpkJkcCYBwjfV00oKQC5ttslPbnj2xnUre2n2pnmkQVAsJhFvaLhgU7ARx5yXCPOtRSD6BVI1wzKfAv39wUvmQ60lkPBFgXYtsqWIhaBpXoBiD7o5RGa7sIP2AeDTNZ3UDrulPQrC2VOtZ \ No newline at end of file diff --git a/.level0-update/part01c b/.level0-update/part01c deleted file mode 100644 index b999684..0000000 --- a/.level0-update/part01c +++ /dev/null @@ -1 +0,0 @@ -LJy63PeqPY05lHhyhHUjBtHxvuJEY4WS4mZBRPFOUFZu2hg8PaDf+eMOh/eb5FWGguQJ3EhwBxbz9XKYGcRIhkv0KZEmxRDdkuTjfMg9bSXJ+/NwFL9J1jPsHRxL43F+iRR5lE3yUxQ6ZpMrS0OSlC9UmpS0AlDl5CZEakCN2CSULEEvhl1lK8/YH90e1eeO/sz5o54WxGOcWMcAIz9h7wWSLfQBgpsCuNR0haZpSELheKN0OOdIfU4evEMVrqVOJbzGKK3ulN88E/ogID0ocyDPF/rpPVwt5K7FMvksy1vgSPJp9PYOl/TFGhAv5ysrvmoA6uHjhOVrmC5pTtWXNJtO9K69Xt00LRCdSf27idzeFf9Cp4aZgOgUbS+4VUcVO5WPBPvkAZSpcAmM612LP1wVLYZRroKlx3LS7Tdufkv3EXoAoLvIp8uL1U0Cn8Bw1+Ndy2G5KCQYcj3ydu/fKWy+c6cXe8jOvtuWe6FviRt9azVd2FmCExhY1iHs6RlpjIQHRaO1mtN10W2gUdKwIfo9L8OU56HaYs6dNNbI5rHPMfjjYpIOfQC6mWaZeENbmpVIvd37UUxMdxST0QlPU4/8e2NUGH0knwfdKNlCOK2uVf5DohPKV7N+QWHLFAYy7iMjiRNZjiDqSFKxoJB9ZjnyWQDZlkqDa2vxV3KErvZaEG/c1EapP9wNKYpzDKRSaVT8IhRyOcewJFILKCaIhijjHdEDc3hj62UHuZwMFsUdKQwXOUM41lBms57llxRDWYdZxv/UncuxxV8harZi8JaRg9t9HFZpKekKyqBNL6jSaD1dFELWiWfzbNXrNsmsaoAPFSwpsL33hdjyRSS68Sz0mPjenpWf8PJXema4Snf+efBlUoMpJIWRiJBN0QXUB4ZzGUVrlvQ0lfjAUCAWVCgrM1cEdy9vjXGpmZH+GbF13ClAMyoKK5Uu6YgtyjOACNoShpu8AxF0H/eOqLkTN7EfKa2e/PzyKjlWg58C/QqcGitvXgj9Kiaj/8uSa01/Vev0V7+cCHsGrlJexxCcUDCBFbCV3XFpESgmKCvkeAq0iCMjS99o6vV3HJ+cqcHeKF9Wl1PLL9qyt4qSKhNY0smeeNwvKUcrs2fC8mwVl/wjQr5VDVykW1YBEjesDGJe1jmm3neoOZmf7WmR6Z4SmereKomann6UiivD7/JQ6pitY5rHxLGetFiByFieFe+OEROf0cmtooOgkzx9fm0Xur2csOoO65dUFQahTu/rSU1pYe1lo7X7Vif7jrKHvJhPkLewgaOUAlyRzGdwYzRInTggDWOtF6ZpgjWZxng4kqToYGTmfbLpFbYUxY1DFRQU1MSPUaX4C8cFmu5l+d2ciJSKgEsxHlQNI97nWPAUHL5I3GD17uMKx3y3LyxmqWwaC/FdmM7qUijigYr17JfjOMV+wW1fb87XZ2fAjY2B59INiNaA1RIFappJU1HtpH/P2h9eDP7w+o/HHEW27oNtNDbUxRP/3fH7dxyLuQSKjicNdYvaRpB/+OmbjdDO16fVgOzh+Yefvv/+5avvXzz/9liCNgUqm/C43FiP37/84f3rH45fqWjU3718q/rtrofWDdXd1qGtdaTBGBy4dW+JDlIL1dfiVQe3YVuUgNc8JvdK+UnBPzrRY2gf0IlB8qB6lMGPqQc4LyH4VSkqyY9bCHXOdc97+kdYxHn+jqqnW2CksFwPVBea4u3DkyAu0qKwAAJPOCW2BuOKCw1rpPCZkU5Im5FXvEBfTnyhiTjc/QT1/LDD+mVR5vAdwFzb1Dk7ROru6CgqjUdLRG2hRpVSlJVJIwMJ0wJHLhct0/bVH1uU4MgqNSWdHvrKBSzMtU3azniCm9Att3etqQlesBkvaa40W5TVl5ktiop7jCNNiy5F4H/FRd5gp5LSuybijsG+r2h673r/+FXr/tM03F3/n6rfxgYAP88O4E64Un0FjuPMpmtwFHs2XIXL0GjzdfjG3VV4/ERMB/QXmhD4U2c00KunxR1+b5s5cQbf22JC/GH3Ns8GfupG1GNOmP8IngTK9J7MRNoa/tFnjuSe+SUyBV/e20IgYJnz3kaBgGDPe5sFAv5e6PkJohOVMxa7y/VKb3mSYXPPsp2dHI3KEZswTFIvqQ0GKKYYDJTmKcss/t78/7Bv6M8U/+fgyRMb/+dgn/z/dPc7D/5/Psd3C/8/U+OLB3cSuWbIbAQgneSF95F/tGYzcsQz81Nb4/WMBDnpBAu8wBA/FiA7gfj+zftvWd5M+8bKOk3UHgxQTHk2xrPOA66EcqQLfEg+UInostJPw9DJOq3T/YpFKhwm2Ampwi3mqWeGwBekPBuwV7PCidRiPFt8m66LdIIB2J6vVqgfMp/VYTr+OB+tJ1ksMo0KX4KR3c2ESL8TfowUVyuB3MpTTOgv1B84ci+aNH6h7lVVLGknijR1A21aFfCeaMkvQCB7ol9uARsbGkuov9wif0MD6r+iktes9SNQrnRZt9CaifyNa9TDBPzh9fXD/YD5eD9gYJT3A2iZFTmwPmYaAdp3/EddzGhQ6SxH930qkof3JjZNiw/eVYB38WqZT3SEpVlWUAN2I3Kf5d+shccVcGs0Gq2PeXYRstidJv6vGpzLI7vdW+BDUYG7qyc0oqmgtREfz5cX6XKk9telGyKsJBgYfjYaN5p9/20NF8xsMMlmZ6tz6OM5kFG44SQmZJeph7uB9f17ttjjx3LnmLJ/074FGNnrl7GZquyIgNq0Tdt5aJkQW3WY6q5FiA+66Q+fvemPuumPn61pSSvP04JUvV40k1oB91cUEs1pW47Ww9Ug1eTaN4PHUAgvWlU1Qgz/Wyg3+RARKYVJAHM2wB3Zs6Zx8lMbfLDoOfSUfLvB38Yxu3bUGkLIi8GQzqhqGRbe4kvu2chKopsD4yX8zdV73Ep43y5YhRGWAf5OT6GGe/82c4bK5X9Lfp98EGu218VY140GefJCo5a/LVd1s8yNUkDmdwtnDlZpnE8m4aL8D5oiLHJy1EzwfwGC+SmR8ErsvKm2l8/GtUbZ9Pk9BPyZj1eoJm1SgV7m095ep6qSWt5wKBZKkOViRpivUaTnIEzVQK7kFMOSfbTbCrOu/F3XosgxZ+v5uqjfeotr0rkT9kW613DOwbpzxtavGiJK4I9vfin2ixodD0uPdPStse2xHvAGfuVtAW3mDD75gJTL4CwATf6L1lk2WdfV5ACNbybpAnIuc9jSWa+2SmfntYZYoW/wyP9F12gy66i5xZvCKwyQue10IjmGujGmHmoE7XTv2s50gs7SEFkJ7CevEQcE/dIOoq5nAlYk2FGmJHRDF+xSwR17VftFlwgq8p3ATfaCFPDsHus/aXbtrVFuEn97BZENYqB8ZjQKCjF+W76bA91yaZ66H4FtrZ8wyuNK+1FXZZiwfoBeg/Fd8UsFcai8fsiZJFjEWwseipEJNccZSWgtJTIBSV6jCG014ot01znIu4akiTBo+DneisTEuWwYdBzbowC30Aj7IyLcNpS/xTfzZjLN0lmvTY6bVqOe7FhjC5iwC1q3Bqcu41I+4DQll0EB1uywh+UqV4U+RcPefDjNVufzkd122GsMVwIodWSxK+IlzXUKwjWaSd1gQdPZBo1G9Zxz/ZJ5wQVvbGjYtMteSRQ8xLZyXQzdCQ4AWhd1IoTSQVbL1OSjy1iMaPzYtWThZsc8VAhfYw6kWEXxsjgImBzUKRn510fi4N1izwxtFKRpk1RHAzEWdbaqEOdogshGgtj9VIV99vmxUfYxH2Y97DD/tEAuNQIj9auH2O7hdR1gNPRJE1LkuumTbeG2NAJ7ROlweEnBf04vSnIL1g1JdQ5EFS7YCQRMpmOEI+U4quq9aJFx2ABYBJiSq5Bv574g94qLX9/rNHUSLlAdLz8Rjp1bF7VK7x0mfijCbKo4opiBe2S2nsqwkHxkP0I9kOFkPcoG4Zq4AlDaAPlMvNGzFjNbXItQROtpNqk3wqjINoqyCkzpkIuK \ No newline at end of file diff --git a/.level0-update/part02a b/.level0-update/part02a deleted file mode 100644 index 6f03eb8..0000000 --- a/.level0-update/part02a +++ /dev/null @@ -1 +0,0 @@ -bjij5Ub3emWIpMiT7oY/N1RduBJEfY8lxmDBQ6mwXgU5vhRtd461qklXX2xurnRUVTgx0QU1HqvApgadM+zpUrtYVYjt64csMzQe7CXjGm/oa4JCbsvd92TuUiu7XAVRA/E7CVLwq49r19zEzeDnwX+tqd7x8cpCqghihjV/cGt+2L7mn9yaH7ev+dqtqZmBreoCxz14+coAwJN/PNy65uuf3jtVS5rtb9istGD3917YelwMl/liVTy+N5DBh698Tw8PS97/+Ld5/3ty8F/anc5B5/C/JIe/XJfs95/8/c+u/3I9G8BRBZzGfbexYf07+/sdb/33DzoP8V8+y/ebf3q8LpaPT/PZ42z2ETW5z3dQY3svW89RJykbp/lkZ+fdt29fvnmP+qG92j/Xh6ME/jvKl+Qqo/bP1988f/eHwbvXP7399vik3b+pNWrJv/xLsrgYNWo7b4/fvB68ff36valpgT1utURJyjTFazs7qtb1q+evXn8PNX48/tPxj23KPdp7vJouHs/S2fxssdpTio3wswsn3Os371/+8eV/O8bOXneO9iioG6S/Oz7+DpO6R3uAdE8h5bvjP7389jjSBmdA1fVqjuXQxqikOybvaO+f8R+KnnSDA3/304/v35VVk9m6JjBt68kKbaF3nr/9/l0PT2RthZJETE8o24TahNkzI69RVgFMPU44jJsTmCuHJB6fSoT+7pHKMKTrwXCW6pDOlX2u7TRQueXkJAnH9vpPx29/fvvyPUxg+6ZGei+dWtLv/xYF/uioF0f3Za8Onf+YLUkvqbEzznd2sksO+/Pn9394/eqnV9/89OLF8VtYtM6O1pRbo7Kcp6/CgmnsB8I9+T1g4K+nUfOP9bn0n0LfIdLc6ymw6fx/erjv0X/4b/uB/n+O7++a/iPdehchnZTONLzZbXefNg/aX38t6X6sks20B8LLF+96u81dct+b7C2TvTRB2IPnb98+/3Pyu9/9ThHPd9GiBqJT3rYDI8AbnKXPTKO8akSufpuM5jvK3QDSbC5p+yILJUmMaYPyNk42/IFgkIaPUDpB/4muf+vx4gpvInCvba3m08kvgWPV+n/A7bW7dv8fdmD/d5/sdx/2/+f4Tk7X+WS0V5COe39H+Zij2LY1oAPrxWo+nxTPek++Qn9uF+dZNqn1d7jSaTr8kM1QfC+KtiiPFG4B/08UbvV3VFDsmsex6SO8tqMMurFMu9VttWs7o4zRXKXiixryQis466MOZBE0pFpzRNxOFCWU5McYMbVmRrinGAoA/Ky33+pQe+iWKpsN80wH962RvPNZr9vqktZTjcIJPut1Wt0DTlgA6LTAEm1OmKarxWS+muSnCPepKnWF7BpMI5Xq24mhWKao+LgnW+/vKE1+o/lNLXS+Jqd6yqjrWa8N4Ps7KdS+KnJeMxZnXeCjebakEq1DrPTXNYwXUzrYAxjHKcwkTPnqWe8ppPR3VhxJ4gRNEuDns95XlGw7qkhOfydiRISz6PNk5NME1ZtrugZzaZGilKHK7pwgGrUEQi0AzdKzrGiN89movwM4uMwYP5fD2q8YOvU/xNd6jKv9S0p/bif/eXKwj/Kf9pOnD/Kfz/Hp9SeLCrMh7y30L33V53+3c7h/YOL/duj8P+jCleDh/P8M3y30/2VYXwzmWxqvNxZ6VyXxaYlpi5FJowPHNRvYefP82x+ef39MNwFt2Dkgy8LBAC06yay93mgpI+WTTn8H7VDZRfisgIOt3maLaAfS44RODVRWsUZ64iQqDQ0cL87u7bkwv54EAb2Ms+eIyTHnhWZZzZ1GvD2yy9ENAqvTtPoz8Qp0JUiMAceHbGAuCUW8Bh/S7pjeqWevd8pvHfebwI3zSxgMDPpUjRRvJFZP0CoFvYDSXjAnzRKi1I6fpXR4ol6y36UE4zmMPOJWKDnEXNbGwiL77yknneTLpA7V6sPzdJkOV+iq6Itkv8MvkiZRe87tK18zq3x2ZcapfXAYJ0fcTNyBUehLx7flXc/yLax4q0oZ+13lkDleSthcRqFJp0O8SD6gUhduXF8aAB6hEU7oFMhmd7pfRfwAleTfCEdFtCnCWXW8Ie13PdhC0eEo8RtWSkfYaJBDpi6xDNSNwgx/kErbC7ICDesa6qVAhmeXwIO3ukGBlzo5eo2D4QRYzVmcfA/82RJIWzocrqcDuPktikiRaXpZmpfh0mkzyMh0UD51ICutXtk/WvsNUCB5+GFB7vwqujLJ0iVtUnw051Vo+2WmcDUcRAoGU4Ll0vXlgARIsRohaNgCk6XKDHIv0uV0vTDTHGQTAgxG2TC9YhB+iVO4Z3co6+tITpdzDv3pXxT5hPZ9J9vzkZ8wYzjJF5gdYCxtDBiQ6VNYxIqhgDLQRPmEgaZxOkcfOetpvItUZIZuYpfzj8EGoCIk4TpKnvpYMZ8ugE0It5XcNfrOHO4a5wJdsjedMlW4twR2Zz5lDI/3RvsuU3zBgM4VckfiBX20akRVZtwUvvGklp4OAejZef7XD5PpbL7427JYrT9eXF79a62fPEqeHAjFg4WJZIGfNfR2Duy6NMmW5t4OUReG3Q61lkbcTkZgr/33fzLeaGNvxQfg0JRbHtWFQV4AgBR5jnw4wHNzcLrIDIelXdNEvXj5zKp686vh79DlHFrgApfmuvhyvO7Q05vlr0oqKOd/VFhGbPSK80FLxQRP45yzZIQr9plzzDq4Ig7ZAydVHbBuoj5cJfKUnawlp2rpiXoTG6s5VmF6xEFFgz/YVCE4XKnaV5uqebSVKnWUhbFXzdAvqGaJTB91Dtn8WGAnLRt1fwAlBy7xKobzBXqzywYzYrWvKLhPgKx88eiRar+5cmiVPNU3dhrg6Q1S8LinA7jPD7oHzgygR3auonTYgqm1JXy9uwYV7h6YawQJP+k+oSN4NFlTvqR+X7ajsLTdJm02rEUgY0X2hTKXW1A4aA/uSzyKhj82XaXFvZJj0s16WrE17avPqyqXm1kTtK5DPJqjmuGIMYqWP7sECjjMV4hnZ0vILgKCFL/bVODBo0ceidBe0szLV8+/fPKUNFWjjZO2syJCGdxU0arTlNAisb4zo9dsa0djOnE5pz5f5igLZ8/AZP1SNQ83LmFzaIrlvBxqkU4mdadZmFR6Q4c5Jqdnagtv07xzovhShAE7/S+0f2gdgBvOTVpV8htkNivXQUwsDy8ehBQ/FMP1Q4h32iLT+sYXNNqPCN4RFfwA4PvSX4rGdAGVdpbqcLgbsKePoUOlpRbZcojW3nAYtdpflBbT7vk77aK8DLAkVfmzOfCoF3pRYJF7+25hsaTECAiuTYmCSMiGLEO6AiI+pNUkaVF9NV2QI/kmBrgpTBRP4edpa06QuL5OWzJxGvidOb+vonzfQZzrO/iPwPPRUJXbIPc+EnUX9KQt8RzWEEWs5O+ClpPc0gH7ki21E2KNVXE3fRR+i84AwC8NDd2xhlvED29RWUWvwxZ8oy26gWMUBcs87xOvQs5QovXKHeL3Mc6QrGQ2Cnm8xCXm8Ap+ZEUkxU9KK8LKlVb7qrw5jGm/VTU8uPFKYKq3zibz03rtETnTVwemFhMjlpR4qbRUIfQx6aCRgXXiuvJm/t048P6qqZ1zHzSN6+2DG0m55hMO+Ugd4NvNX+noMJ1RpMmZGumVc0NMgGtX1Lhejb/awwbxodqRMX5Vu2mIPXWRQ1P8fNAi46aibq2bmhwQFV12BzFnpdeGDfPsHMtC2g0oOSeeOR19REbFMkzL2Zk+iRWdxghkbDGFV6B4ADJT/SybsdNJY231vU4B9JqmM7ImzGD2O919FVtvhf0/zeAAwlM/BIXxNQdUCkAMJ/OZjj80zpf07O/L8t2Y7ZbQ2atQT8jG7C1PngpKcCcLYrd7B/oaqDpP5PLX7YPaMjzdwPukk3r1JDadOXeApNKbhISYwgZgG8BT9cu1WEtZ1InZ9AOJ9r/mizqtUVPNk1/FB6kr+cD0thHIvLqY0/XUYi5t02Kgj42BDVo2 \ No newline at end of file diff --git a/.level0-update/part02b b/.level0-update/part02b deleted file mode 100644 index 98161a4..0000000 --- a/.level0-update/part02b +++ /dev/null @@ -1 +0,0 @@ -wLNnnA6Bw/D2PNEE5Qxabn+SLVAJpUMcK6SyuFwoAjOwVVyKNT4DyQcmMdHRO4vLdRConvkl2V3bw578Q9Rny8bacLE2UUjUrYkmi3WURFzuOvYWhwjd1ROqyaF1SNvQe8CEsIyA0LkDIfAG6rrMh0UpRIphFgdHWRsBqHz0ez/i3GHxUYPQlSFJBydPZ/mYVYgiTercaGuufEPP5klNz9oozivIouZGJeUt3bLSiqUAQjDjkk8Oyoq6wpJplhbAQnG8IarpMCIkr+CZoYvfvpOJZ7/KhRMZuqmuhydtcnDVdS7A5PB6dlWHW9p6OuPQqwUed3V+Gamp50nK5guHWhFKKRyKpPHnxGISt48Vr1Xzzr0Wzqa8GKMhNZkZqtrUMlqJam/ekdKEXOVFNSKc0BPQmi8/QO1Xc+g4yS6RAaGpFd7UGWPNzYhrmbFgrIFaSMw3NSUEg+LkHuigBgN8HR/YqtxCFBM1lovteYrIvlhJh+QbatAY/SqCWs/g0J4uVlcDJChaidFKc4wdxa9In7cRIDnUWzt8jxHnCqLsEONSThAdqh+TC3OXHRQ2J5IN/AU69mvr9mzztR4DqclO5/MPv5wO4G30/w6fkv3n0/39B/2/z/HJ9W93hQFQvriand5PG9X6f539jrD/4fU/7D49fND/+xzf9Q6cQtlkgk9ieCSSMI5SBvriPU2XH0bzCyU4q+X4EFcbfp12R1lX6fzUtOAG39NZWlbjSMgKLPz9m+SljhekFfb//X/+n4Qwjg0VR3l6NpvjqymfOMgREbBIp1DbS7WdXWbDNZ3u5K0BcmfryUR29qvDUed0tP9VeWfZXzdNQj/efaEE+RctQ9SJ88ImlWlF2hJBdWsx0Fpc4S/Sj5ysgoJCoTLIk5qVsbakIuW8wLtsNvtY360wSt1tJrtlRrbaUnW30bBNLOcX9A7Xt0nFejpFKV4kix1+rOiaW5BryDq2ymKx3Uck4hg8eiyuGtDYka2vDm0KzLU6V4qgbrbl5HVgWua7doNL2a5b0fCWbrVNFzEPigpc4/XCsncJPlXOV25rIoKNC01xWMBbrTM3RzyR0aSpYCjQaYoQ2FqS/K++y1MKy9ppuADGS9bElDc9YiEjxU52TXO76K7E/BUtiy3u6rhYpnMeXEQOHROe6jXiq+HeLd1ZkzfKMiRw6/vLUg5BIrHu53W4OGJijsSaRArSrBx5cxIvKO5rUMPcxbycfmVlcWnCK5kDx8+rhmSudQ4Mm7pF7UW2hGm/zFdXIQyRtwUkVLBYpsOrgXrwA3iddjt55EPV5eIw+W4pBybu/0GuD+JGoIra7oglPpkix1tBhKf6ePfV3DzSjHDDFglkZsvkGknhza6Ank6MzIZ3KsfOrmNzzSSHgxOuZuTmh58CAhrMtb6Dc+8F7rG6xOpGi0Lw4mUXbk5yizcVtvZDgPd7RrfTbJim3fannNF4WT5FMQEeokp20kyuJulpNvHJKZyvLbjjrZcoszgjuXG9024mT/zd75jwNhWthDNLLEiLXvBPr+pi4mLkOz07W2Zn6n2HANmaQii12zjhvvdbUAOWA13n0UqscCHQ+iFb8VrXG2ErxWLJ3httcydctYWujmdpHSNbhvVQ36eJWAIzQxMo68vu9ZsOaOodpNE098wMRFrgOYd5O81WFwCpHhZxp8lvd3N57kyyp2Zh+xpfbqgxWZynvXarW5IPCzZf9siC4owepif4XBMp3AjR8JJmDlBA++PmQe9GijIu1xVKh/mrfAWsw3j3mhHo5ojcLib/7/8m6KAQVWpG6XJEcgoi+rE2zpb5qG7GexgpMcnO8AyM5BTn8wuZbn/ZfYlnDlNTQGl7/iTkDW9PecMjb3Syd159cVS4UERGeW1zJEDd7DIdAmsNRzBHr010ZoLMCIUlaVSA4uehiXpyH5ylC69D5O3v3//X/zbPcmpo90s+T7962h6ddg4+hXxeXJTx6cj/VrLpIWsOp6FmOLXSmmZIPyNjG94xsDL33VOExJewR/FrBoHajkc2Rbfkk0X5rXhl/NQ6lbHMig9Rpfxjj8JTId9RrIfDrCjG60nyM83EzzwTiXzlgBkEhqQl8Z+c/btALy4clkS1XM2VcL2TXSI0NGwAsJoPyL1hPgQgSEjrKr/RTCjwcNHbHc6R3fMJF+vOqveQHm3net1RCKUAw7uM86wHiht1Mp+dYS7+1v8yrHy0S/F3dd2LC+gExeR2W6atjspYFyfyIUSMrdGHmVlc+fQSO0ZNNVXvDWtBIA1rIIcWw8wyTubQ52R0ozFuRvRgW24GvyqOhm4mDTMLd+Jl8LsrP0NoUcXTqKvTHZkZO/UbGRp3pkzD2xXejqMpqbWBq6FaGzgb/LbmbvArYfzSy3PSKO+iC2j8VayuJllvd29vl/++yEer855PwnV1zSRZnWLGr3hpxSftupRN0ZJ4FWad0GKYPY6XFKvmjnSpOIekc5lLsgzAToInsnNoQ3LtQ7bE4BaLbKjNlGqjHI689MooR75hJyb76tifpLOzdXpGWezfRGXo8py4X9Nsh6kBW3A8N824xRUM5aAF09lfCvUcwdRmp2N0+r1Sdhvm78E0hxMA7UJ2bv4hnsP+033O+09nUAB/OskG9/sCtOH950l7/6n//nP45MnD+8/n+O76/pN2s4P09Onok99/GOOCB6DYFZLUNxP0+ETWH4/RoZN19oR3v3yGnqFewJ8/Z6d7x6N1635vWVnn4HA4TPfLR/3wkHQPD0nGQx7em0qbePX6/fE3r1//YD37YUNk/iTPeHSipy5VW8DC0ggGfQs6fXr70yuAouwyx7vWi98N3z7JWd/NbmzGtDUNXqx3HVU4bEk+cDVjT1NNLaR2npqaWzxHiVv8NC8KjmRzEtyKTP/UzbGOY31M2UIZSQCDcgre1oLuP6r2UZJgdElJ/wwau4H9owA6Mm/R/bh6IXfUm9NGyVNOqaamghLOfBkkZzniwCIrVgbNLGMlwI2LXQY+rsipZ06+rsYm3nkyOPFevXDKdsnLCsLz7wO71gwUChl1PJvMZDa4CO162pzek4yfG1SXImO3apUweRdV9hz9Qn5yk/WjRTYCijwrRYsEgKLPbhJMpEAA5NPetzwhGTqDXQPF9qCUFJLAbvr3LPBMxyMg9fvDTzmKkQjyDtCvREiRTtwpqO9qWXVMQN30C7uCaSmNDopuKYX+ouHU7W/5kBWIf8i5LApMWRJG4mglq47Kd/g1EA0zlNYynHpU/2ZwzUk38oTR1fBwoFxU4rVjLBEgmUb46Zb/jN+aSX6j++I/SHFFLbehbn6mlxbLCLDrXmYE8ERTc/SLPLHc72bqpk++/uq0c/gpm2krPNy4jia95G2l7wHTUqW2liopKZIvYvKqbYEIUoy0xXOOW3EjdrhDS3BoHohyHKl+aLvjOmhhqElwnBtZkeguXrgSnZlQ5u0Ao0ckEhIPwiZ8sSA3CTXy6Xpq40PtsS39j29v23R6uUXTR6Ll9HLLljfLJyVCVc7fRuTRtfewdlLAqYtR7m6HQCGB+aWIS6czOj192v4k7UvB9CLjkS7zgth4lz11iWVMRcvCARZGbOs9zUrtRvWOXO2rgUWwzVpY5cpXA0nrNithxXSvLIgNOljlqlceiE0qWNtpXnlASzWwbpq3XTJiWKsWybl4VTy1RBbGq3rrVfHq33pJYvU/bT1iELdaDHnMRvfePfMfo9F4eHj4SXK1iwsSVFDsRqWn8NOrzU/9sXdzgvMrPJyfBMoFjgaD7ln/s76sIyjeRRFI9oz9e32ehyJO43nBrvTCvcTCqx+yK5ZZeW+HjIDJOQpN5wwyyUdsHp1pS0x/2HfWDYgycd1m8jR2qVNKBHdUH9CKLfx6L/VAd6teWInT0porCgfUn2poaHG3/JAte7vzXY835wwa1r65s5W9vN7m8dgwbtQHGFU3dv3a7i35lu/IW3FucGmYFfg2CpzkNAWyfMnAMAN9asyXeRa9jm59YxzP \ No newline at end of file diff --git a/.level0-update/part02c b/.level0-update/part02c deleted file mode 100644 index a2f52e9..0000000 --- a/.level0-update/part02c +++ /dev/null @@ -1 +0,0 @@ -gfrgxD5tJjNY9R5gzenp/HIA+zadDc/nS8Ak9GyadBoY83XY210v4NwApnK8io8JHWnBymD48lup/OF/9IO1sktG+bPaiK79skK2prNZUZxPUw0/SCp/OWWRBTqZQ9PcgmmABUOo3492oU55J16H+p7mc6QX/UZrleaT+n678fBm//A9fA/fw/fwPXwP38P38D18D9/D9/A9fA/fw/fw/WN9/x87zv+zALgBAA== \ No newline at end of file diff --git a/level_0_baseline/README.md b/level_0_baseline/README.md index cacde29..2795c09 100644 --- a/level_0_baseline/README.md +++ b/level_0_baseline/README.md @@ -1,76 +1,121 @@ -# Level 0 Baseline +# Isolated Level 0 nanoGPT Baseline -A deliberately self-contained nanoGPT baseline. It does not import the repository's existing experiment framework or WW-PGD code. +This subtree is an independent, auditable nanoGPT baseline. It does not import the repository's WW-PGD experiment framework or apply WW-PGD. Its purpose is to establish a credible AdamW language-model baseline and measure WeightWatcher layer spectra before adding an optimizer extension. -## Scope +## Corrected baseline -- one transformer block, one ordinary Q/K/V attention head, width 64, context 256 -- byte-level next-token language modeling on a fixed FineWeb-Edu subset -- AdamW or Muon with a global warmup/cosine schedule -- Muon applies only to hidden 2-D matrices; AdamW handles embeddings, tied LM head, LayerNorm parameters, and other non-matrix parameters -- deterministic seeds for initialization and sampled training windows -- immutable train, validation, and test splits -- CSV logging of loss, next-token accuracy, perplexity, validation/test generalization gaps, gradient norm, weight norm, tokens, and elapsed time -- optional checkpoint-time WeightWatcher layer analysis -- single-seed and multi-seed notebooks; multi-seed plots use mean ± one standard deviation shaded bands +The original isolated baseline was an 82K-parameter, one-layer byte model. This version uses a materially more realistic MacBook-scale configuration: -## Install +- pinned FineWeb-Edu `sample-10BT` source; +- GPT-2 BPE tokenization (`tiktoken`, vocabulary 50,257); +- 16M training, 1M validation, and 1M test tokens stored as `uint16`; +- 4 transformer blocks, 4 attention heads, width 128, context 256; +- 7,253,248 parameters with tied token embedding/output weights; +- AdamW with decoupled weight decay, gradient clipping, 100-step warmup, and cosine decay; +- microbatch 4 with 8 accumulation steps: 8,192 tokens per optimizer update and 16.384M tokens over 2,000 updates; +- fixed, independent train/validation/test probes that do not advance the training RNG; +- test evaluation only at the final and validation-selected checkpoints; +- non-randomized WeightWatcher analysis of the 24 transformer block matrices only. The large embedding/output matrix is excluded from periodic spectral analysis. -```bash -cd level_0_baseline -python -m venv .venv -source .venv/bin/activate -pip install -e '.[data,analysis,test]' -``` - -## Paths +The default layer learning-rate multiplier is flat (`layer_lr_decay: 1.0`). Layerwise decay is available as an explicit ablation, not silently enabled in the baseline. -Defaults are under `/tmp/nanogpt-level0`. Override them without editing code: +## Install in the existing Conda environment ```bash -export NANOGPT_LEVEL0_DATA_ROOT=/tmp/my-level0/data -export NANOGPT_LEVEL0_RESULTS_ROOT=/tmp/my-level0/results -export NANOGPT_LEVEL0_CACHE_ROOT=/tmp/my-level0/cache +conda activate ww_prod310 +cd ~/Desktop/work/nanoGPT/nanogpt-experiments/level_0_baseline +python -m pip install -e '.[data,analysis,test]' ``` -## Prepare the real corpus +## Paths -This prepares fixed 50 MB training, 2 MB validation, and 2 MB test byte-token splits from streamed FineWeb-Edu: +The corrected format uses a new root so the earlier raw-byte files cannot be mistaken for GPT-2-tokenized data: ```bash -level0-prepare-data --dataset fineweb-edu +export NANOGPT_LEVEL0_ROOT=/tmp/nanogpt-level0-gpt2 +export NANOGPT_LEVEL0_DATA_ROOT=$NANOGPT_LEVEL0_ROOT/data +export NANOGPT_LEVEL0_RESULTS_ROOT=$NANOGPT_LEVEL0_ROOT/results ``` -To monitor the streamed download and preparation, enable heartbeat logging: +## Prepare the pinned FineWeb-Edu corpus ```bash level0-prepare-data \ + --config configs/level0.yaml \ --dataset fineweb-edu \ --verbose \ --log-interval-seconds 10 ``` -Verbose output reports documents processed, bytes collected, completion percentage, elapsed time, average throughput, estimated time remaining, and how long the stream has produced no new bytes. The heartbeat continues while the streaming iterator is blocked, making a network or dataset stall visible. +The preparer reports documents, GPT-2 tokens, elapsed time, throughput, ETA, and time since the stream last produced tokens. It validates compatible existing data and reuses it; pass `--force` to rebuild it. + +A successful preparation produces: -## Run one seed +```text +/tmp/nanogpt-level0-gpt2/data/train.bin +/tmp/nanogpt-level0-gpt2/data/val.bin +/tmp/nanogpt-level0-gpt2/data/test.bin +/tmp/nanogpt-level0-gpt2/data/meta.json +``` + +## Run one AdamW seed ```bash -./scripts/run_one.sh adamw 1337 -./scripts/run_one.sh muon 1337 +./scripts/run_one.sh adamw 1337 \ + 2>&1 | tee /tmp/level0-gpt2-adamw-seed1337.log ``` -## Run multiple seeds +The command refuses to overwrite an existing nonempty run. To intentionally replace it: ```bash -NANOGPT_LEVEL0_SEEDS=1337,2027,4099 ./scripts/run_multiseed.sh +NANOGPT_LEVEL0_OVERWRITE=1 ./scripts/run_one.sh adamw 1337 ``` -The notebooks read `NANOGPT_LEVEL0_RESULTS_ROOT`. Select the single-seed run with `NANOGPT_LEVEL0_NOTEBOOK_OPTIMIZER` and `NANOGPT_LEVEL0_NOTEBOOK_SEED`. +The run writes periodic metrics and checkpoints, validation-selected and final test metrics, WeightWatcher CSV files, and `run_complete.json` under: + +```text +/tmp/nanogpt-level0-gpt2/results/adamw_seed_1337 +``` -For a bounded infrastructure smoke test, override the run length and batch size: +## Run several AdamW seeds ```bash -NANOGPT_LEVEL0_MAX_STEPS=2 NANOGPT_LEVEL0_BATCH_SIZE=2 NANOGPT_LEVEL0_EVAL_INTERVAL=1 ./scripts/run_one.sh adamw 1337 +NANOGPT_LEVEL0_SEEDS=1337,2027,4099 \ +NANOGPT_LEVEL0_OPTIMIZERS=adamw \ + ./scripts/run_multiseed.sh ``` -Next-token error is `1 - next-token accuracy`; the notebooks derive and plot it explicitly. +Muon remains available for a later baseline comparison: + +```bash +NANOGPT_LEVEL0_OPTIMIZERS=adamw,muon ./scripts/run_multiseed.sh +``` + +## Analyze one seed + +```bash +export NANOGPT_LEVEL0_RESULTS_ROOT=/tmp/nanogpt-level0-gpt2/results +export NANOGPT_LEVEL0_NOTEBOOK_OPTIMIZER=adamw +export NANOGPT_LEVEL0_NOTEBOOK_SEED=1337 +jupyter lab notebooks/01_single_seed.ipynb +``` + +The notebook plots train/validation loss, perplexity, exact next-GPT-2-token accuracy, generalization gap, learning rate, final versus validation-selected test metrics, and WeightWatcher alpha by transformer matrix. + +## Bounded smoke run + +The smoke run checks the software path only; it is not a scientific result: + +```bash +NANOGPT_LEVEL0_MAX_STEPS=2 \ +NANOGPT_LEVEL0_BATCH_SIZE=1 \ +NANOGPT_LEVEL0_GRAD_ACCUM_STEPS=1 \ +NANOGPT_LEVEL0_EVAL_INTERVAL=1 \ +NANOGPT_LEVEL0_WEIGHTWATCHER=0 \ +NANOGPT_LEVEL0_OVERWRITE=1 \ + ./scripts/run_one.sh adamw 1337 +``` + +## Primary outcomes + +Use held-out cross-entropy and perplexity as the principal language-model outcomes. Exact token accuracy is reported as a secondary diagnostic and should not be compared numerically with the old 256-byte-vocabulary accuracy. diff --git a/level_0_baseline/notebooks/01_single_seed.ipynb b/level_0_baseline/notebooks/01_single_seed.ipynb index d887603..ad5ca4a 100644 --- a/level_0_baseline/notebooks/01_single_seed.ipynb +++ b/level_0_baseline/notebooks/01_single_seed.ipynb @@ -1,10 +1,188 @@ { "cells": [ - {"cell_type":"markdown","metadata":{},"source":["# Level 0 single-seed diagnostics\n"]}, - {"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["import os\n","from pathlib import Path\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","ROOT=Path(os.getenv('NANOGPT_LEVEL0_RESULTS_ROOT','/tmp/nanogpt-level0/results'))\n","OPT=os.getenv('NANOGPT_LEVEL0_NOTEBOOK_OPTIMIZER','adamw')\n","SEED=int(os.getenv('NANOGPT_LEVEL0_NOTEBOOK_SEED','1337'))\n","RUN=ROOT/f'{OPT}_seed_{SEED}'\n","df=pd.read_csv(RUN/'metrics.csv')\n","for split in ['train','val','test']: df[f'{split}_error']=1-df[f'{split}_accuracy']\n","df.head()\n"]}, - {"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["for metric in ['loss','accuracy','error','perplexity']:\n"," plt.figure(figsize=(9,5))\n"," for split in ['train','val','test']: plt.plot(df.step,df[f'{split}_{metric}'],label=split)\n"," plt.xlabel('step'); plt.ylabel(metric); plt.title(f'{OPT} seed {SEED}: {metric}'); plt.legend(); plt.grid(alpha=.25); plt.show()\n"]}, - {"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["files=sorted(RUN.glob('weightwatcher_step_*.csv'))\n","if files:\n"," ww=pd.concat([pd.read_csv(f) for f in files],ignore_index=True)\n"," layer_col='layer_id' if 'layer_id' in ww else 'layer'\n"," for layer,g in ww.groupby(layer_col): plt.plot(g.step,g.alpha,label=str(layer))\n"," plt.xlabel('step'); plt.ylabel('alpha'); plt.legend(bbox_to_anchor=(1.02,1)); plt.show()\n","else: print('No WeightWatcher files found.')\n"]} + { + "cell_type": "markdown", + "id": "a2e4ab7d", + "metadata": {}, + "source": [ + "# Isolated Level 0 — single-seed diagnostics\n", + "\n", + "GPT-2 BPE AdamW/Muon baseline on pinned FineWeb-Edu." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e145cca3", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "ROOT = Path(os.getenv('NANOGPT_LEVEL0_RESULTS_ROOT', '/tmp/nanogpt-level0-gpt2/results'))\n", + "OPTIMIZER = os.getenv('NANOGPT_LEVEL0_NOTEBOOK_OPTIMIZER', 'adamw')\n", + "SEED = int(os.getenv('NANOGPT_LEVEL0_NOTEBOOK_SEED', '1337'))\n", + "RUN = ROOT / f'{OPTIMIZER}_seed_{SEED}'\n", + "\n", + "required = ['manifest.json', 'metrics.csv', 'run_complete.json', 'final_metrics.json', 'selected_checkpoint_metrics.json']\n", + "missing = [name for name in required if not (RUN / name).is_file()]\n", + "if missing:\n", + " raise FileNotFoundError(f'Missing run artifacts in {RUN}: {missing}')\n", + "\n", + "manifest = json.loads((RUN / 'manifest.json').read_text())\n", + "completion = json.loads((RUN / 'run_complete.json').read_text())\n", + "final_metrics = json.loads((RUN / 'final_metrics.json').read_text())\n", + "selected_metrics = json.loads((RUN / 'selected_checkpoint_metrics.json').read_text())\n", + "metrics = pd.read_csv(RUN / 'metrics.csv')\n", + "\n", + "pd.DataFrame([{\n", + " 'run': str(RUN),\n", + " 'parameters': manifest['parameter_count'],\n", + " 'optimizer_steps': completion['optimizer_steps'],\n", + " 'tokens_seen': completion['tokens_seen'],\n", + " 'best_validation_step': completion['best_validation_step'],\n", + " 'best_validation_loss': completion['best_validation_loss'],\n", + " 'selected_test_loss': completion['selected_test_loss'],\n", + " 'final_test_loss': completion['final_test_loss'],\n", + " 'weightwatcher_failures': completion['weightwatcher_failures'],\n", + "}])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afdada3c", + "metadata": {}, + "outputs": [], + "source": [ + "for metric, ylabel in [\n", + " ('loss', 'cross-entropy loss'),\n", + " ('perplexity', 'perplexity'),\n", + " ('accuracy', 'exact next-token accuracy (%)'),\n", + "]:\n", + " plt.figure(figsize=(10, 5))\n", + " for split in ('train', 'val'):\n", + " values = metrics[f'{split}_{metric}']\n", + " if metric == 'accuracy':\n", + " values = 100 * values\n", + " plt.plot(metrics['tokens_seen'], values, label=split)\n", + " plt.xlabel('training tokens')\n", + " plt.ylabel(ylabel)\n", + " plt.title(f'{OPTIMIZER} seed {SEED}: {metric}')\n", + " plt.grid(alpha=0.25)\n", + " plt.legend()\n", + " plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2a698b15", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(10, 5))\n", + "plt.plot(metrics['tokens_seen'], metrics['val_generalization_gap'])\n", + "plt.axhline(0.0, linewidth=1, linestyle='--')\n", + "plt.xlabel('training tokens')\n", + "plt.ylabel('validation loss − training loss')\n", + "plt.title(f'{OPTIMIZER} seed {SEED}: generalization gap')\n", + "plt.grid(alpha=0.25)\n", + "plt.show()\n", + "\n", + "plt.figure(figsize=(10, 5))\n", + "plt.plot(metrics['step'], metrics['learning_rate'], label='base learning rate')\n", + "plt.plot(metrics['step'], metrics['min_group_learning_rate'], linestyle='--', label='minimum parameter-group LR')\n", + "plt.plot(metrics['step'], metrics['max_group_learning_rate'], linestyle=':', label='maximum parameter-group LR')\n", + "plt.xlabel('optimizer step')\n", + "plt.ylabel('learning rate')\n", + "plt.title(f'{OPTIMIZER} seed {SEED}: learning-rate schedule')\n", + "plt.grid(alpha=0.25)\n", + "plt.legend()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11dbb708", + "metadata": {}, + "outputs": [], + "source": [ + "checkpoint_comparison = pd.DataFrame([\n", + " {\n", + " 'checkpoint': 'validation-selected',\n", + " 'step': selected_metrics['selected_step'],\n", + " 'validation_loss': selected_metrics['validation_loss'],\n", + " 'test_loss': selected_metrics['test_loss'],\n", + " 'test_perplexity': selected_metrics['test_perplexity'],\n", + " 'test_accuracy_percent': 100 * selected_metrics['test_accuracy'],\n", + " },\n", + " {\n", + " 'checkpoint': 'final',\n", + " 'step': final_metrics['step'],\n", + " 'validation_loss': final_metrics['validation_loss'],\n", + " 'test_loss': final_metrics['test_loss'],\n", + " 'test_perplexity': final_metrics['test_perplexity'],\n", + " 'test_accuracy_percent': 100 * final_metrics['test_accuracy'],\n", + " },\n", + "])\n", + "checkpoint_comparison" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2ddfc553", + "metadata": {}, + "outputs": [], + "source": [ + "ww_files = sorted(RUN.glob('weightwatcher_step_*.csv'))\n", + "if not ww_files:\n", + " print('No successful WeightWatcher measurements found.')\n", + "else:\n", + " ww = pd.concat([pd.read_csv(path) for path in ww_files], ignore_index=True)\n", + " ww['alpha'] = pd.to_numeric(ww.get('alpha'), errors='coerce')\n", + " ww['step'] = pd.to_numeric(ww['step'], errors='coerce')\n", + " layer_column = next((name for name in ('source_layer', 'longname', 'name', 'layer_id') if name in ww), None)\n", + " if layer_column is None:\n", + " raise KeyError('WeightWatcher output has no layer identifier column')\n", + " valid = ww[np.isfinite(ww['alpha'])].copy()\n", + " plt.figure(figsize=(12, 7))\n", + " for layer, frame in valid.groupby(layer_column):\n", + " frame = frame.sort_values('step')\n", + " plt.plot(frame['step'], frame['alpha'], marker='o', linewidth=1, markersize=3, label=str(layer))\n", + " plt.axhline(2.0, linestyle='--', linewidth=1, label='alpha = 2')\n", + " plt.xlabel('optimizer step')\n", + " plt.ylabel('WeightWatcher alpha')\n", + " plt.title(f'{OPTIMIZER} seed {SEED}: transformer-matrix alpha trajectories')\n", + " plt.grid(alpha=0.25)\n", + " plt.legend(fontsize=7, ncol=2, bbox_to_anchor=(1.02, 1), loc='upper left')\n", + " plt.tight_layout()\n", + " plt.show()\n", + "\n", + " display_columns = [column for column in ('step', layer_column, 'alpha', 'D', 'xmin', 'num_evals') if column in valid]\n", + " display(valid[display_columns].sort_values(['step', layer_column]).tail(30))" + ] + } ], - "metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10"}}, - "nbformat":4,"nbformat_minor":5 + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/level_0_baseline/notebooks/02_multiseed.ipynb b/level_0_baseline/notebooks/02_multiseed.ipynb index d2ac6d9..3490f18 100644 --- a/level_0_baseline/notebooks/02_multiseed.ipynb +++ b/level_0_baseline/notebooks/02_multiseed.ipynb @@ -1,10 +1,152 @@ { "cells": [ - {"cell_type":"markdown","metadata":{},"source":["# Level 0 multi-seed uncertainty bands\n"]}, - {"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["import os\n","from pathlib import Path\n","import pandas as pd\n","import matplotlib.pyplot as plt\n","ROOT=Path(os.getenv('NANOGPT_LEVEL0_RESULTS_ROOT','/tmp/nanogpt-level0/results'))\n","rows=[]\n","for path in ROOT.glob('*_seed_*/metrics.csv'):\n"," optimizer,seed=path.parent.name.split('_seed_')\n"," d=pd.read_csv(path); d['optimizer']=optimizer; d['seed']=int(seed)\n"," for split in ['train','val','test']: d[f'{split}_error']=1-d[f'{split}_accuracy']\n"," rows.append(d)\n","all_df=pd.concat(rows,ignore_index=True)\n","all_df.groupby('optimizer').seed.nunique()\n"]}, - {"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["def band_plot(metric):\n"," plt.figure(figsize=(10,6))\n"," for optimizer,d in all_df.groupby('optimizer'):\n"," a=d.groupby('step')[metric].agg(['mean','std']).reset_index(); s=a['std'].fillna(0)\n"," line,=plt.plot(a.step,a['mean'],label=optimizer)\n"," plt.fill_between(a.step,a['mean']-s,a['mean']+s,alpha=.2,color=line.get_color())\n"," plt.xlabel('step'); plt.ylabel(metric); plt.title(f'{metric}: mean ± 1 standard deviation'); plt.legend(); plt.grid(alpha=.25); plt.show()\n","for metric in ['test_loss','test_accuracy','test_error','test_perplexity','test_generalization_gap']: band_plot(metric)\n"]}, - {"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["ww=[]\n","for run in ROOT.glob('*_seed_*'):\n"," optimizer,seed=run.name.split('_seed_')\n"," for f in run.glob('weightwatcher_step_*.csv'):\n"," d=pd.read_csv(f); d['optimizer']=optimizer; d['seed']=int(seed); ww.append(d)\n","if ww:\n"," ww=pd.concat(ww,ignore_index=True); layer_col='layer_id' if 'layer_id' in ww else 'layer'\n"," for layer,ld in ww.groupby(layer_col):\n"," plt.figure(figsize=(10,5))\n"," for optimizer,d in ld.groupby('optimizer'):\n"," a=d.groupby('step').alpha.agg(['mean','std']).reset_index(); s=a['std'].fillna(0)\n"," line,=plt.plot(a.step,a['mean'],label=optimizer); plt.fill_between(a.step,a['mean']-s,a['mean']+s,alpha=.2,color=line.get_color())\n"," plt.title(f'Layer {layer} alpha'); plt.legend(); plt.show()\n"]} + { + "cell_type": "markdown", + "id": "c9a2de26", + "metadata": {}, + "source": [ + "# Isolated Level 0 — multi-seed diagnostics" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85d1bd38", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "ROOT = Path(os.getenv('NANOGPT_LEVEL0_RESULTS_ROOT', '/tmp/nanogpt-level0-gpt2/results'))\n", + "rows = []\n", + "summary_rows = []\n", + "for path in sorted(ROOT.glob('*_seed_*/metrics.csv')):\n", + " run = path.parent\n", + " completion_path = run / 'run_complete.json'\n", + " selected_path = run / 'selected_checkpoint_metrics.json'\n", + " if not completion_path.is_file() or not selected_path.is_file():\n", + " continue\n", + " optimizer, seed_text = run.name.rsplit('_seed_', 1)\n", + " frame = pd.read_csv(path)\n", + " frame['optimizer'] = optimizer\n", + " frame['seed'] = int(seed_text)\n", + " rows.append(frame)\n", + " selected = json.loads(selected_path.read_text())\n", + " completion = json.loads(completion_path.read_text())\n", + " summary_rows.append({\n", + " 'optimizer': optimizer,\n", + " 'seed': int(seed_text),\n", + " 'selected_step': selected['selected_step'],\n", + " 'selected_validation_loss': selected['validation_loss'],\n", + " 'selected_test_loss': selected['test_loss'],\n", + " 'selected_test_perplexity': selected['test_perplexity'],\n", + " 'selected_test_accuracy_percent': 100 * selected['test_accuracy'],\n", + " 'final_test_loss': completion['final_test_loss'],\n", + " })\n", + "if not rows:\n", + " raise FileNotFoundError(f'No completed runs under {ROOT}')\n", + "all_metrics = pd.concat(rows, ignore_index=True)\n", + "summary = pd.DataFrame(summary_rows).sort_values(['optimizer', 'seed'])\n", + "summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aecaa20", + "metadata": {}, + "outputs": [], + "source": [ + "def band_plot(column, ylabel):\n", + " plt.figure(figsize=(10, 6))\n", + " for optimizer, frame in all_metrics.groupby('optimizer'):\n", + " aggregated = frame.groupby('tokens_seen')[column].agg(['mean', 'std']).reset_index()\n", + " spread = aggregated['std'].fillna(0.0)\n", + " line, = plt.plot(aggregated['tokens_seen'], aggregated['mean'], label=optimizer)\n", + " plt.fill_between(\n", + " aggregated['tokens_seen'],\n", + " aggregated['mean'] - spread,\n", + " aggregated['mean'] + spread,\n", + " alpha=0.2,\n", + " color=line.get_color(),\n", + " )\n", + " plt.xlabel('training tokens')\n", + " plt.ylabel(ylabel)\n", + " plt.title(f'{column}: mean ± one standard deviation')\n", + " plt.grid(alpha=0.25)\n", + " plt.legend()\n", + " plt.show()\n", + "\n", + "band_plot('val_loss', 'validation cross-entropy loss')\n", + "band_plot('val_perplexity', 'validation perplexity')\n", + "band_plot('val_accuracy', 'exact next-token accuracy (fraction)')\n", + "band_plot('val_generalization_gap', 'validation loss − training loss')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b870db14", + "metadata": {}, + "outputs": [], + "source": [ + "ww_rows = []\n", + "for run in sorted(ROOT.glob('*_seed_*')):\n", + " if '_seed_' not in run.name:\n", + " continue\n", + " optimizer, seed_text = run.name.rsplit('_seed_', 1)\n", + " for path in sorted(run.glob('weightwatcher_step_*.csv')):\n", + " frame = pd.read_csv(path)\n", + " frame['optimizer'] = optimizer\n", + " frame['seed'] = int(seed_text)\n", + " ww_rows.append(frame)\n", + "if not ww_rows:\n", + " print('No successful WeightWatcher measurements found.')\n", + "else:\n", + " ww = pd.concat(ww_rows, ignore_index=True)\n", + " ww['alpha'] = pd.to_numeric(ww.get('alpha'), errors='coerce')\n", + " layer_column = next((name for name in ('source_layer', 'longname', 'name', 'layer_id') if name in ww), None)\n", + " valid = ww[np.isfinite(ww['alpha'])].copy()\n", + " for layer, layer_frame in valid.groupby(layer_column):\n", + " plt.figure(figsize=(10, 5))\n", + " for optimizer, frame in layer_frame.groupby('optimizer'):\n", + " aggregated = frame.groupby('step')['alpha'].agg(['mean', 'std']).reset_index()\n", + " spread = aggregated['std'].fillna(0.0)\n", + " line, = plt.plot(aggregated['step'], aggregated['mean'], label=optimizer)\n", + " plt.fill_between(\n", + " aggregated['step'],\n", + " aggregated['mean'] - spread,\n", + " aggregated['mean'] + spread,\n", + " alpha=0.2,\n", + " color=line.get_color(),\n", + " )\n", + " plt.axhline(2.0, linestyle='--', linewidth=1)\n", + " plt.xlabel('optimizer step')\n", + " plt.ylabel('WeightWatcher alpha')\n", + " plt.title(str(layer))\n", + " plt.grid(alpha=0.25)\n", + " plt.legend()\n", + " plt.show()" + ] + } ], - "metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.10"}}, - "nbformat":4,"nbformat_minor":5 + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/level_0_baseline/scripts/run_multiseed.sh b/level_0_baseline/scripts/run_multiseed.sh old mode 100644 new mode 100755 diff --git a/level_0_baseline/scripts/run_one.sh b/level_0_baseline/scripts/run_one.sh old mode 100644 new mode 100755 diff --git a/level_0_baseline/src/level0_baseline/data.py b/level_0_baseline/src/level0_baseline/data.py index 219849b..4fe63b5 100644 --- a/level_0_baseline/src/level0_baseline/data.py +++ b/level_0_baseline/src/level0_baseline/data.py @@ -149,7 +149,9 @@ def _load_tokenizer(name: str) -> Tokenizer: ) from exc tokenizer = tiktoken.get_encoding(name) if tokenizer.n_vocab - 1 > UINT16_MAX: - raise ValueError(f"tokenizer {name!r} has IDs that do not fit in uint16") + raise ValueError( + f"tokenizer {name!r} has IDs that do not fit in uint16" + ) return tokenizer @@ -312,7 +314,7 @@ def write_token_splits( split_documents[split] += 1 used_tokens += take if take < int(chunk.size): - # Never let one source document cross a scientific split. The + # Never let one source document cross a scientific split. The # unused suffix is deliberately discarded at the boundary. discarded_boundary_tokens += int(chunk.size) - take if split_collected[split] == split_sizes[split]: diff --git a/level_0_baseline/src/level0_baseline/model.py b/level_0_baseline/src/level0_baseline/model.py index 73f8af6..5032e08 100644 --- a/level_0_baseline/src/level0_baseline/model.py +++ b/level_0_baseline/src/level0_baseline/model.py @@ -35,9 +35,9 @@ def __init__(self, cfg: GPTConfig): self.resid_dropout = nn.Dropout(cfg.dropout) self.register_buffer( "mask", - torch.tril( - torch.ones(cfg.block_size, cfg.block_size, dtype=torch.bool) - ).view(1, 1, cfg.block_size, cfg.block_size), + torch.tril(torch.ones(cfg.block_size, cfg.block_size, dtype=torch.bool)).view( + 1, 1, cfg.block_size, cfg.block_size + ), persistent=False, ) diff --git a/level_0_baseline/src/level0_baseline/optim.py b/level_0_baseline/src/level0_baseline/optim.py index 34a741c..e676826 100644 --- a/level_0_baseline/src/level0_baseline/optim.py +++ b/level_0_baseline/src/level0_baseline/optim.py @@ -71,9 +71,7 @@ def step(self, closure=None): else momentum_buffer ) update = zeropower_via_newtonschulz5(update_input) - update.mul_( - max(1.0, parameter.shape[0] / parameter.shape[1]) ** 0.5 - ) + update.mul_(max(1.0, parameter.shape[0] / parameter.shape[1]) ** 0.5) if group["weight_decay"]: parameter.mul_(1 - group["lr"] * group["weight_decay"]) parameter.add_(update, alpha=-group["lr"]) @@ -154,15 +152,14 @@ def make_optimizers(model, config: dict[str, Any]) -> list[torch.optim.Optimizer raise ValueError(f"unsupported optimizer: {optimizer_name}") muon_grouped: dict[float, list[torch.nn.Parameter]] = defaultdict(list) - auxiliary_grouped: dict[ - tuple[bool, float], list[torch.nn.Parameter] - ] = defaultdict(list) + auxiliary_grouped: dict[tuple[bool, float], list[torch.nn.Parameter]] = defaultdict(list) for record in _parameter_records(model, training): name = str(record["name"]) parameter = record["parameter"] multiplier = float(record["lr_multiplier"]) - use_muon = parameter.ndim == 2 and not name.startswith( - ("token_embedding", "position_embedding", "lm_head") + use_muon = ( + parameter.ndim == 2 + and not name.startswith(("token_embedding", "position_embedding", "lm_head")) ) if use_muon: muon_grouped[multiplier].append(parameter) diff --git a/level_0_baseline/src/level0_baseline/train.py b/level_0_baseline/src/level0_baseline/train.py index 2123405..a3b1aac 100644 --- a/level_0_baseline/src/level0_baseline/train.py +++ b/level_0_baseline/src/level0_baseline/train.py @@ -1,126 +1,756 @@ from __future__ import annotations -import argparse, csv, json, math, platform, random, time + +import argparse +import csv +import json +import math +import os +import platform +import random +import shutil +import sys +import time +from dataclasses import asdict from pathlib import Path +from typing import Any, Iterable + import numpy as np import torch +import torch.nn as nn + from .config import load_config, roots +from .data import validate_prepared_data from .model import GPT, GPTConfig from .optim import make_optimizers -def device_auto(): +RUN_SCHEMA_VERSION = 2 + + +def device_auto() -> torch.device: if torch.cuda.is_available(): return torch.device("cuda") if torch.backends.mps.is_available(): return torch.device("mps") return torch.device("cpu") -def seed_all(seed): + +def seed_all(seed: int) -> None: random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) -def lr_at(step, t): - if step < t["warmup_steps"]: - return t["learning_rate"] * (step + 1) / max(1, t["warmup_steps"]) - if step >= t["max_steps"]: - return t["min_lr"] - ratio = (step - t["warmup_steps"]) / max(1, t["max_steps"] - t["warmup_steps"]) - return t["min_lr"] + 0.5 * (1 + math.cos(math.pi * ratio)) * (t["learning_rate"] - t["min_lr"]) -def batch(data, batch_size, block_size, device, generator): - ix = torch.randint(len(data) - block_size - 1, (batch_size,), generator=generator) - x = torch.stack([torch.from_numpy(np.array(data[i:i+block_size], dtype=np.int64)) for i in ix]) - y = torch.stack([torch.from_numpy(np.array(data[i+1:i+1+block_size], dtype=np.int64)) for i in ix]) - return x.to(device), y.to(device) +def learning_rate_at(step_index: int, training: dict[str, Any]) -> float: + warmup_steps = int(training["warmup_steps"]) + max_steps = int(training["max_steps"]) + learning_rate = float(training["learning_rate"]) + minimum_learning_rate = float(training["min_lr"]) + if step_index < warmup_steps: + return learning_rate * (step_index + 1) / max(1, warmup_steps) + if step_index >= max_steps: + return minimum_learning_rate + ratio = (step_index - warmup_steps) / max(1, max_steps - warmup_steps) + cosine = 0.5 * (1.0 + math.cos(math.pi * ratio)) + return minimum_learning_rate + cosine * ( + learning_rate - minimum_learning_rate + ) + + +def _assert_data_matches_config( + metadata: dict[str, Any], + config: dict[str, Any], +) -> None: + data = config["data"] + for key in ( + "dataset_name", + "dataset_config", + "dataset_split", + "dataset_revision", + "tokenizer", + "dtype", + ): + if metadata.get(key) != data.get(key): + raise ValueError( + f"prepared data field {key!r} does not match the configured identity" + ) + expected_splits = { + "train": int(data["train_tokens"]), + "val": int(data["val_tokens"]), + "test": int(data["test_tokens"]), + } + if metadata.get("split_tokens") != expected_splits: + raise ValueError( + "prepared data split sizes do not match the configured token counts" + ) + + +def _sample_cpu_batch( + data: np.ndarray, + *, + batch_size: int, + block_size: int, + generator: torch.Generator, +) -> tuple[torch.Tensor, torch.Tensor]: + upper = len(data) - block_size - 1 + if upper <= 0: + raise ValueError("prepared split is shorter than the configured block size") + indices = torch.randint(upper, (batch_size,), generator=generator) + x = torch.stack( + [ + torch.from_numpy( + np.asarray(data[int(index) : int(index) + block_size], dtype=np.int64) + ) + for index in indices + ] + ) + y = torch.stack( + [ + torch.from_numpy( + np.asarray( + data[int(index) + 1 : int(index) + 1 + block_size], + dtype=np.int64, + ) + ) + for index in indices + ] + ) + return x, y + + +def _to_device( + batch: tuple[torch.Tensor, torch.Tensor], + device: torch.device, +) -> tuple[torch.Tensor, torch.Tensor]: + x, y = batch + return x.to(device, non_blocking=True), y.to(device, non_blocking=True) + + +def make_fixed_probe( + data: np.ndarray, + *, + batch_size: int, + block_size: int, + batches: int, + seed: int, +) -> list[tuple[torch.Tensor, torch.Tensor]]: + generator = torch.Generator(device="cpu").manual_seed(seed) + return [ + _sample_cpu_batch( + data, + batch_size=batch_size, + block_size=block_size, + generator=generator, + ) + for _ in range(batches) + ] + @torch.no_grad() -def evaluate(model, data, batch_size, block_size, n_batches, device, generator): +def evaluate_probe( + model: nn.Module, + probe: Iterable[tuple[torch.Tensor, torch.Tensor]], + device: torch.device, +) -> dict[str, float]: + was_training = model.training model.eval() - losses, correct, total = [], 0, 0 - for _ in range(n_batches): - x, y = batch(data, batch_size, block_size, device, generator) - logits, loss = model(x, y) - losses.append(loss.item()) - correct += (logits.argmax(-1) == y).sum().item() - total += y.numel() - model.train() - loss = float(np.mean(losses)) - return loss, math.exp(min(20, loss)), correct / total - -def weightwatch(model, out, step, randomize): + total_nll = 0.0 + correct = 0 + token_count = 0 + try: + for cpu_batch in probe: + x, y = _to_device(cpu_batch, device) + logits, loss = model(x, y) + if loss is None: + raise RuntimeError("evaluation did not return a loss") + count = y.numel() + total_nll += float(loss.detach().cpu()) * count + correct += int((logits.argmax(dim=-1) == y).sum().detach().cpu()) + token_count += count + finally: + model.train(was_training) + loss_value = total_nll / max(token_count, 1) + return { + "loss": loss_value, + "perplexity": math.exp(min(20.0, loss_value)), + "accuracy": correct / max(token_count, 1), + "bits_per_token": loss_value / math.log(2.0), + "tokens": float(token_count), + } + + +def _gradient_norm(parameters: Iterable[torch.nn.Parameter]) -> torch.Tensor: + norms = [ + parameter.grad.detach().float().norm(2) + for parameter in parameters + if parameter.grad is not None + ] + if not norms: + return torch.tensor(0.0) + return torch.linalg.vector_norm(torch.stack(norms), ord=2) + + +def _weight_norm(parameters: Iterable[torch.nn.Parameter]) -> float: + squares = sum( + float((parameter.detach().float() ** 2).sum().cpu()) + for parameter in parameters + ) + return math.sqrt(squares) + + +def _atomic_json(path: Path, payload: dict[str, Any]) -> None: + temporary = path.with_name(f".{path.name}.tmp") + temporary.write_text( + json.dumps(payload, indent=2, sort_keys=True, allow_nan=False), + encoding="utf-8", + ) + os.replace(temporary, path) + + +class _SpectralSnapshot(nn.Module): + def __init__(self, model: GPT): + super().__init__() + layers: dict[str, nn.Linear] = {} + cpu_rng = torch.get_rng_state() + try: + for name, source in model.spectral_layers(): + copied = nn.Linear( + source.in_features, + source.out_features, + bias=False, + device="cpu", + ) + copied.weight.data.copy_( + source.weight.detach().to(device="cpu", dtype=torch.float32) + ) + copied.weight.requires_grad_(False) + layers[name] = copied + finally: + torch.set_rng_state(cpu_rng) + self.layers = nn.ModuleDict(layers) + + +def _source_layer_from_row(row: dict[str, Any], names: list[str]) -> str: + text = " ".join( + str(row.get(key, "")) for key in ("longname", "name", "layer_id") + ) + matches = [name for name in names if name in text] + return matches[0] if len(matches) == 1 else "" + + +def weightwatch( + model: GPT, + output_dir: Path, + *, + step: int, + randomize: bool, +) -> dict[str, Any]: try: import weightwatcher as ww - except ImportError: - return - df = ww.WeightWatcher(model=model).analyze(randomize=randomize) - df.insert(0, "step", step) - df.to_csv(out / f"weightwatcher_step_{step:07d}.csv", index=False) - -def main(): - p = argparse.ArgumentParser() - p.add_argument("--config", default="configs/level0.yaml") - p.add_argument("--data-root") - p.add_argument("--results-root") - p.add_argument("--optimizer", choices=["adamw", "muon"]) - p.add_argument("--seed", type=int) - p.add_argument("--device", default="auto") - a = p.parse_args() - cfg = load_config(a.config) - t = cfg["training"] - if a.optimizer: - t["optimizer"] = a.optimizer - if a.seed is not None: - t["seed"] = a.seed - resolved = roots() - data_root = Path(a.data_root or resolved["data"]) - base = Path(a.results_root or resolved["results"]) - run = base / f"{t['optimizer']}_seed_{t['seed']}" + except ImportError as exc: + raise RuntimeError( + "WeightWatcher is enabled but not installed; install the analysis extra" + ) from exc + + started = time.perf_counter() + snapshot = _SpectralSnapshot(model) + source_names = [name for name, _ in model.spectral_layers()] + try: + details = ww.WeightWatcher(model=snapshot).analyze( + randomize=bool(randomize) + ) + details.insert(0, "step", int(step)) + rows = details.to_dict(orient="records") + details.insert( + 1, + "source_layer", + [_source_layer_from_row(row, source_names) for row in rows], + ) + path = output_dir / f"weightwatcher_step_{step:07d}.csv" + details.to_csv(path, index=False) + return { + "step": step, + "success": True, + "rows": int(len(details)), + "path": str(path), + "elapsed_sec": time.perf_counter() - started, + } + except Exception as exc: # WeightWatcher should not erase the training run. + payload = { + "step": step, + "success": False, + "exception_type": type(exc).__name__, + "exception_message": str(exc), + "elapsed_sec": time.perf_counter() - started, + } + _atomic_json( + output_dir / f"weightwatcher_error_step_{step:07d}.json", + payload, + ) + print( + "[level0-train] WeightWatcher failed " + f"step={step}: {type(exc).__name__}: {exc}", + file=sys.stderr, + flush=True, + ) + return payload + finally: + del snapshot + + +def _optimizer_manifest(optimizers: list[torch.optim.Optimizer]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for optimizer_index, optimizer in enumerate(optimizers): + for group_index, group in enumerate(optimizer.param_groups): + rows.append( + { + "optimizer_index": optimizer_index, + "optimizer_class": type(optimizer).__name__, + "group_index": group_index, + "group_name": group.get("group_name", f"group_{group_index}"), + "parameter_count": sum( + parameter.numel() for parameter in group["params"] + ), + "initial_lr": float(group.get("initial_lr", group["lr"])), + "lr_multiplier": float(group.get("lr_multiplier", 1.0)), + "weight_decay": float(group.get("weight_decay", 0.0)), + } + ) + return rows + + +def _save_checkpoint( + path: Path, + *, + model: GPT, + optimizers: list[torch.optim.Optimizer], + step: int, + config: dict[str, Any], + train_generator: torch.Generator, + best_validation_loss: float, + best_validation_step: int, +) -> None: + torch.save( + { + "run_schema_version": RUN_SCHEMA_VERSION, + "model": model.state_dict(), + "optimizers": [optimizer.state_dict() for optimizer in optimizers], + "step": int(step), + "config": config, + "train_generator_state": train_generator.get_state(), + "cpu_rng_state": torch.get_rng_state(), + "best_validation_loss": float(best_validation_loss), + "best_validation_step": int(best_validation_step), + }, + path, + ) + + +def _prepare_run_directory(run: Path, *, overwrite: bool) -> None: + if run.exists() and any(run.iterdir()): + if not overwrite: + raise FileExistsError( + f"run directory already contains files: {run}; " + "use --overwrite or choose a fresh results root" + ) + shutil.rmtree(run) run.mkdir(parents=True, exist_ok=True) - device = device_auto() if a.device == "auto" else torch.device(a.device) - seed_all(t["seed"]) - generator = torch.Generator().manual_seed(t["seed"]) - arrays = {s: np.memmap(data_root / f"{s}.bin", dtype=np.uint8, mode="r") for s in ("train", "val", "test")} - model = GPT(GPTConfig(**cfg["model"])).to(device) - optimizers = make_optimizers(model, cfg) - base_lrs = [group["lr"] for opt in optimizers for group in opt.param_groups] - manifest = {"config": cfg, "device": str(device), "torch": torch.__version__, "platform": platform.platform(), "parameter_count": sum(p.numel() for p in model.parameters()), "data_root": str(data_root.resolve())} - (run / "manifest.json").write_text(json.dumps(manifest, indent=2)) - fields = ["step", "tokens_seen", "elapsed_sec", "learning_rate", "train_loss", "train_perplexity", "train_accuracy", "val_loss", "val_perplexity", "val_accuracy", "test_loss", "test_perplexity", "test_accuracy", "val_generalization_gap", "test_generalization_gap", "grad_norm", "weight_norm"] - with open(run / "metrics.csv", "w", newline="") as f: - writer = csv.DictWriter(f, fieldnames=fields) + + +def run_training( + config: dict[str, Any], + *, + data_root: str | Path, + results_root: str | Path, + device: str = "auto", + overwrite: bool = False, +) -> Path: + training = config["training"] + model_config = GPTConfig(**config["model"]) + seed = int(training["seed"]) + optimizer_name = str(training["optimizer"]).lower() + selected_device = device_auto() if device == "auto" else torch.device(device) + torch.set_float32_matmul_precision("high") + seed_all(seed) + + data_path = Path(data_root) + data_metadata = validate_prepared_data(data_path, verify_hashes=True) + _assert_data_matches_config(data_metadata, config) + if int(data_metadata["vocab_size"]) > model_config.vocab_size: + raise ValueError("model vocabulary does not cover the prepared tokenizer") + arrays = { + split: np.memmap( + data_path / f"{split}.bin", + dtype=np.uint16, + mode="r", + ) + for split in ("train", "val", "test") + } + + run = Path(results_root) / f"{optimizer_name}_seed_{seed}" + _prepare_run_directory(run, overwrite=overwrite) + raw_model = GPT(model_config).to(selected_device) + optimizers = make_optimizers(raw_model, config) + model: nn.Module = raw_model + if bool(training.get("compile", False)): + if not hasattr(torch, "compile"): + raise RuntimeError("torch.compile was requested but is unavailable") + model = torch.compile(raw_model) + + train_generator = torch.Generator(device="cpu").manual_seed(seed + 101) + probe_batch_size = int(training.get("eval_batch_size", training["batch_size"])) + train_probe = make_fixed_probe( + arrays["train"], + batch_size=probe_batch_size, + block_size=model_config.block_size, + batches=int(training["eval_batches"]), + seed=1001, + ) + validation_probe = make_fixed_probe( + arrays["val"], + batch_size=probe_batch_size, + block_size=model_config.block_size, + batches=int(training["eval_batches"]), + seed=2001, + ) + test_probe = make_fixed_probe( + arrays["test"], + batch_size=probe_batch_size, + block_size=model_config.block_size, + batches=int(training["test_eval_batches"]), + seed=3001, + ) + + tokens_per_step = ( + int(training["batch_size"]) + * model_config.block_size + * int(training["grad_accum_steps"]) + ) + max_steps = int(training["max_steps"]) + manifest = { + "run_schema_version": RUN_SCHEMA_VERSION, + "profile": "level0_gpt2_bpe_realistic_v2", + "config": config, + "device": str(selected_device), + "torch_version": torch.__version__, + "platform": platform.platform(), + "parameter_count": raw_model.num_parameters(), + "non_embedding_parameter_count": raw_model.num_parameters( + exclude_position_embedding=True + ), + "data_root": str(data_path.resolve()), + "data_metadata": data_metadata, + "tokens_per_optimizer_step": tokens_per_step, + "planned_optimizer_steps": max_steps, + "planned_training_tokens": tokens_per_step * max_steps, + "evaluation_protocol": { + "train_and_validation": "fixed_probe_periodic", + "test": "final_and_validation_selected_only", + "training_rng_isolated_from_evaluation": True, + "fixed_probe_seeds": {"train": 1001, "val": 2001, "test": 3001}, + }, + "optimizer_groups": _optimizer_manifest(optimizers), + "accuracy_definition": "exact_top1_next_gpt2_bpe_token", + } + _atomic_json(run / "manifest.json", manifest) + + metric_fields = [ + "step", + "tokens_seen", + "elapsed_sec", + "tokens_per_sec", + "learning_rate", + "min_group_learning_rate", + "max_group_learning_rate", + "train_loss", + "train_perplexity", + "train_accuracy", + "train_bits_per_token", + "val_loss", + "val_perplexity", + "val_accuracy", + "val_bits_per_token", + "val_generalization_gap", + "grad_norm", + "weight_norm", + ] + best_validation_loss = float("inf") + best_validation_step = -1 + last_gradient_norm = 0.0 + weightwatcher_records: list[dict[str, Any]] = [] + start_time = time.perf_counter() + latest_metrics: dict[str, Any] = {} + + initial_scale = learning_rate_at(0, training) / float(training["learning_rate"]) + for optimizer in optimizers: + for group in optimizer.param_groups: + group["lr"] = float(group["initial_lr"]) * initial_scale + + print( + "[level0-train] starting " + f"optimizer={optimizer_name} seed={seed} device={selected_device} " + f"parameters={manifest['parameter_count']:,} steps={max_steps:,} " + f"tokens_per_step={tokens_per_step:,} " + f"planned_tokens={tokens_per_step * max_steps:,}", + file=sys.stderr, + flush=True, + ) + + with (run / "metrics.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=metric_fields) writer.writeheader() - start = time.time() - last_grad_norm = float("nan") - for step in range(t["max_steps"] + 1): - lr = lr_at(step, t) - scale = lr / t["learning_rate"] - j = 0 - for opt in optimizers: - for group in opt.param_groups: - group["lr"] = base_lrs[j] * scale - j += 1 - if step % t["eval_interval"] == 0 or step == t["max_steps"]: - values = {s: evaluate(model, arrays[s], t["batch_size"], cfg["model"]["block_size"], t["eval_batches"], device, generator) for s in ("train", "val", "test")} - weight_norm = math.sqrt(sum(float((p.detach().float() ** 2).sum()) for p in model.parameters())) - writer.writerow({"step": step, "tokens_seen": step * t["batch_size"] * cfg["model"]["block_size"] * t["grad_accum_steps"], "elapsed_sec": time.time() - start, "learning_rate": lr, "train_loss": values["train"][0], "train_perplexity": values["train"][1], "train_accuracy": values["train"][2], "val_loss": values["val"][0], "val_perplexity": values["val"][1], "val_accuracy": values["val"][2], "test_loss": values["test"][0], "test_perplexity": values["test"][1], "test_accuracy": values["test"][2], "val_generalization_gap": values["val"][0] - values["train"][0], "test_generalization_gap": values["test"][0] - values["train"][0], "grad_norm": last_grad_norm, "weight_norm": weight_norm}) - f.flush() - print(step, {k: round(v[0], 4) for k, v in values.items()}) - if cfg["analysis"]["weightwatcher"] and step % cfg["analysis"]["weightwatcher_interval"] == 0: - weightwatch(model, run, step, cfg["analysis"]["randomize"]) - if step == t["max_steps"]: + + for step in range(max_steps + 1): + if step == 0 or step % int(training["eval_interval"]) == 0 or step == max_steps: + train_metrics = evaluate_probe(model, train_probe, selected_device) + validation_metrics = evaluate_probe( + model, + validation_probe, + selected_device, + ) + elapsed = time.perf_counter() - start_time + group_lrs = [ + float(group["lr"]) + for optimizer in optimizers + for group in optimizer.param_groups + ] + current_lr = learning_rate_at(max(step - 1, 0), training) + latest_metrics = { + "step": step, + "tokens_seen": step * tokens_per_step, + "elapsed_sec": elapsed, + "tokens_per_sec": ( + step * tokens_per_step / max(elapsed, 1e-9) + ), + "learning_rate": current_lr, + "min_group_learning_rate": min(group_lrs), + "max_group_learning_rate": max(group_lrs), + "train_loss": train_metrics["loss"], + "train_perplexity": train_metrics["perplexity"], + "train_accuracy": train_metrics["accuracy"], + "train_bits_per_token": train_metrics["bits_per_token"], + "val_loss": validation_metrics["loss"], + "val_perplexity": validation_metrics["perplexity"], + "val_accuracy": validation_metrics["accuracy"], + "val_bits_per_token": validation_metrics["bits_per_token"], + "val_generalization_gap": ( + validation_metrics["loss"] - train_metrics["loss"] + ), + "grad_norm": last_gradient_norm, + "weight_norm": _weight_norm(raw_model.parameters()), + } + writer.writerow(latest_metrics) + handle.flush() + + if validation_metrics["loss"] < best_validation_loss: + best_validation_loss = validation_metrics["loss"] + best_validation_step = step + torch.save( + { + "model": raw_model.state_dict(), + "step": step, + "validation_metrics": validation_metrics, + "config": config, + }, + run / "checkpoint_best.pt", + ) + + remaining_steps = max_steps - step + steps_per_second = step / max(elapsed, 1e-9) + eta = ( + remaining_steps / steps_per_second + if steps_per_second > 0 + else None + ) + eta_text = f"{eta:.1f}s" if eta is not None else "unknown" + print( + "[level0-train] progress " + f"step={step:,}/{max_steps:,} " + f"tokens={step * tokens_per_step:,} " + f"train_loss={train_metrics['loss']:.4f} " + f"val_loss={validation_metrics['loss']:.4f} " + f"val_ppl={validation_metrics['perplexity']:.2f} " + f"val_acc={100 * validation_metrics['accuracy']:.2f}% " + f"elapsed={elapsed:.1f}s eta={eta_text}", + file=sys.stderr, + flush=True, + ) + + weightwatcher_due = ( + bool(config["analysis"]["weightwatcher"]) + and ( + step % int(config["analysis"]["weightwatcher_interval"]) == 0 + or step == max_steps + ) + ) + if weightwatcher_due: + weightwatcher_records.append( + weightwatch( + raw_model, + run, + step=step, + randomize=bool(config["analysis"]["randomize"]), + ) + ) + + if step == max_steps: break - for opt in optimizers: - opt.zero_grad(set_to_none=True) - for _ in range(t["grad_accum_steps"]): - x, y = batch(arrays["train"], t["batch_size"], cfg["model"]["block_size"], device, generator) + + scheduled_base_lr = learning_rate_at(step, training) + scale = scheduled_base_lr / float(training["learning_rate"]) + for optimizer in optimizers: + for group in optimizer.param_groups: + group["lr"] = float(group["initial_lr"]) * scale + optimizer.zero_grad(set_to_none=True) + + model.train() + for _ in range(int(training["grad_accum_steps"])): + cpu_batch = _sample_cpu_batch( + arrays["train"], + batch_size=int(training["batch_size"]), + block_size=model_config.block_size, + generator=train_generator, + ) + x, y = _to_device(cpu_batch, selected_device) _, loss = model(x, y) - (loss / t["grad_accum_steps"]).backward() - last_grad_norm = float(torch.nn.utils.clip_grad_norm_(model.parameters(), t["grad_clip"])) - for opt in optimizers: - opt.step() - if (step + 1) % t["checkpoint_interval"] == 0: - torch.save({"model": model.state_dict(), "step": step + 1, "config": cfg}, run / f"checkpoint_{step+1:07d}.pt") - torch.save({"model": model.state_dict(), "step": t["max_steps"], "config": cfg}, run / "checkpoint_final.pt") + if loss is None: + raise RuntimeError("training did not return a loss") + (loss / int(training["grad_accum_steps"])).backward() + + if float(training["grad_clip"]) > 0: + gradient = torch.nn.utils.clip_grad_norm_( + raw_model.parameters(), + float(training["grad_clip"]), + ) + else: + gradient = _gradient_norm(raw_model.parameters()) + last_gradient_norm = float(gradient.detach().cpu()) + for optimizer in optimizers: + optimizer.step() + + completed_step = step + 1 + if completed_step % int(training["checkpoint_interval"]) == 0: + _save_checkpoint( + run / f"checkpoint_{completed_step:07d}.pt", + model=raw_model, + optimizers=optimizers, + step=completed_step, + config=config, + train_generator=train_generator, + best_validation_loss=best_validation_loss, + best_validation_step=best_validation_step, + ) + + _save_checkpoint( + run / "checkpoint_final.pt", + model=raw_model, + optimizers=optimizers, + step=max_steps, + config=config, + train_generator=train_generator, + best_validation_loss=best_validation_loss, + best_validation_step=best_validation_step, + ) + + final_test_metrics = evaluate_probe(model, test_probe, selected_device) + final_metrics = { + "step": max_steps, + "validation_loss": float(latest_metrics["val_loss"]), + "validation_perplexity": float(latest_metrics["val_perplexity"]), + "validation_accuracy": float(latest_metrics["val_accuracy"]), + "test_loss": final_test_metrics["loss"], + "test_perplexity": final_test_metrics["perplexity"], + "test_accuracy": final_test_metrics["accuracy"], + "test_bits_per_token": final_test_metrics["bits_per_token"], + } + _atomic_json(run / "final_metrics.json", final_metrics) + + best_checkpoint = torch.load( + run / "checkpoint_best.pt", + map_location="cpu", + weights_only=False, + ) + raw_model.load_state_dict(best_checkpoint["model"]) + selected_test_metrics = evaluate_probe(model, test_probe, selected_device) + selected_metrics = { + "selected_step": int(best_checkpoint["step"]), + "validation_loss": float( + best_checkpoint["validation_metrics"]["loss"] + ), + "validation_perplexity": float( + best_checkpoint["validation_metrics"]["perplexity"] + ), + "validation_accuracy": float( + best_checkpoint["validation_metrics"]["accuracy"] + ), + "test_loss": selected_test_metrics["loss"], + "test_perplexity": selected_test_metrics["perplexity"], + "test_accuracy": selected_test_metrics["accuracy"], + "test_bits_per_token": selected_test_metrics["bits_per_token"], + } + _atomic_json(run / "selected_checkpoint_metrics.json", selected_metrics) + + completion = { + "completed": True, + "run_schema_version": RUN_SCHEMA_VERSION, + "optimizer": optimizer_name, + "seed": seed, + "optimizer_steps": max_steps, + "tokens_seen": max_steps * tokens_per_step, + "best_validation_step": selected_metrics["selected_step"], + "best_validation_loss": selected_metrics["validation_loss"], + "final_validation_loss": final_metrics["validation_loss"], + "selected_test_loss": selected_metrics["test_loss"], + "final_test_loss": final_metrics["test_loss"], + "validation_collapse_final_minus_selected": ( + final_metrics["validation_loss"] - selected_metrics["validation_loss"] + ), + "test_collapse_final_minus_selected": ( + final_metrics["test_loss"] - selected_metrics["test_loss"] + ), + "weightwatcher_measurements": len(weightwatcher_records), + "weightwatcher_failures": sum( + not bool(record.get("success")) for record in weightwatcher_records + ), + "elapsed_sec": time.perf_counter() - start_time, + } + _atomic_json(run / "run_complete.json", completion) + print( + "[level0-train] complete " + f"run={run} selected_step={selected_metrics['selected_step']} " + f"selected_test_loss={selected_metrics['test_loss']:.4f} " + f"final_test_loss={final_metrics['test_loss']:.4f}", + file=sys.stderr, + flush=True, + ) + return run + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", default="configs/level0.yaml") + parser.add_argument("--data-root") + parser.add_argument("--results-root") + parser.add_argument("--optimizer", choices=["adamw", "muon"]) + parser.add_argument("--seed", type=int) + parser.add_argument("--device", default="auto") + parser.add_argument("--overwrite", action="store_true") + args = parser.parse_args() + + config = load_config(args.config) + if args.optimizer: + config["training"]["optimizer"] = args.optimizer + if args.seed is not None: + config["training"]["seed"] = args.seed + resolved = roots() + run = run_training( + config, + data_root=args.data_root or resolved["data"], + results_root=args.results_root or resolved["results"], + device=args.device, + overwrite=args.overwrite, + ) + print(run) + if __name__ == "__main__": main() diff --git a/level_0_baseline/tests/test_baseline.py b/level_0_baseline/tests/test_baseline.py index 76d8995..abc2c19 100644 --- a/level_0_baseline/tests/test_baseline.py +++ b/level_0_baseline/tests/test_baseline.py @@ -1,97 +1,285 @@ +from __future__ import annotations + import json import sys from pathlib import Path +import numpy as np +import pandas as pd +import pytest import torch -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +PACKAGE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PACKAGE_ROOT / "src")) -from level0_baseline.data import _progress_message, write_splits +from level0_baseline.config import load_config +from level0_baseline.data import ( + _progress_message, + validate_prepared_data, + write_token_splits, +) from level0_baseline.model import GPT, GPTConfig from level0_baseline.optim import make_optimizers +from level0_baseline.train import ( + _SpectralSnapshot, + make_fixed_probe, + run_training, +) + + +class FakeTokenizer: + name = "gpt2" + n_vocab = 32 + eot_token = 0 + @staticmethod + def encode_ordinary(text: str) -> list[int]: + return [1 + (ord(character) % 31) for character in text] -def cfg(opt): + +def tiny_training_config() -> dict: return { + "data": { + "dataset_name": "unit", + "dataset_config": "unit", + "dataset_split": "train", + "dataset_revision": "unit", + "tokenizer": "gpt2", + "dtype": "uint16", + "train_tokens": 256, + "val_tokens": 128, + "test_tokens": 128, + }, + "model": { + "vocab_size": 32, + "block_size": 8, + "n_layer": 1, + "n_head": 1, + "n_embd": 16, + "dropout": 0.0, + "bias": False, + "tie_weights": True, + }, "training": { - "optimizer": opt, + "batch_size": 2, + "grad_accum_steps": 2, + "max_steps": 2, + "eval_interval": 1, + "eval_batches": 2, + "eval_batch_size": 2, + "test_eval_batches": 2, + "checkpoint_interval": 1, "learning_rate": 0.001, - "weight_decay": 0.1, + "muon_learning_rate": 0.02, + "muon_aux_adamw_learning_rate": 0.001, + "min_lr": 0.0001, + "warmup_steps": 1, + "weight_decay": 0.01, "beta1": 0.9, "beta2": 0.95, + "epsilon": 1e-8, + "grad_clip": 1.0, + "layer_lr_decay": 1.0, + "optimizer": "adamw", "muon_momentum": 0.95, "muon_nesterov": True, - "muon_learning_rate": 0.02, - "muon_aux_adamw_learning_rate": 0.001, - } + "seed": 7, + "compile": False, + }, + "analysis": { + "weightwatcher": False, + "weightwatcher_interval": 1, + "randomize": False, + }, } -def test_forward_and_accuracy_shape(): - model = GPT(GPTConfig(block_size=8, n_embd=16, n_head=1, n_layer=1)) - x = torch.randint(0, 256, (2, 8)) - logits, loss = model(x, x) - assert logits.shape == (2, 8, 256) - assert torch.isfinite(loss) +def prepare_tiny_data(path: Path) -> None: + write_token_splits( + iter(["abcdefghijklmnopqrstuvwxyz"] * 64), + path, + tokenizer=FakeTokenizer(), + train_tokens=256, + val_tokens=128, + test_tokens=128, + dataset_metadata={ + "dataset_name": "unit", + "dataset_config": "unit", + "dataset_split": "train", + "dataset_revision": "unit", + }, + ) -def test_adamw_step(): - model = GPT(GPTConfig(block_size=8, n_embd=16)) - optimizers = make_optimizers(model, cfg("adamw")) - _, loss = model( - torch.randint(0, 256, (2, 8)), - torch.randint(0, 256, (2, 8)), - ) - loss.backward() - for optimizer in optimizers: - optimizer.step() - - -def test_muon_partition_and_step(): - model = GPT(GPTConfig(block_size=8, n_embd=16)) - optimizers = make_optimizers(model, cfg("muon")) - assert len(optimizers) == 2 - _, loss = model( - torch.randint(0, 256, (2, 8)), - torch.randint(0, 256, (2, 8)), - ) - loss.backward() - for optimizer in optimizers: - optimizer.step() +def test_default_config_is_realistic_gpt2_bpe_baseline(): + config = load_config(PACKAGE_ROOT / "configs" / "level0.yaml") + assert config["data"]["tokenizer"] == "gpt2" + assert config["data"]["dtype"] == "uint16" + assert config["model"] == { + "vocab_size": 50257, + "block_size": 256, + "n_layer": 4, + "n_head": 4, + "n_embd": 128, + "dropout": 0.0, + "bias": False, + "tie_weights": True, + } + assert config["training"]["batch_size"] == 4 + assert config["training"]["grad_accum_steps"] == 8 + assert config["training"]["layer_lr_decay"] == 1.0 + assert config["analysis"]["randomize"] is False + + +def test_model_size_and_weightwatcher_scope_are_not_toy_byte_baseline(): + model = GPT(GPTConfig()) + assert model.num_parameters() == 7_253_248 + assert len(model.blocks) == 4 + assert len(model.spectral_layers()) == 24 + names = [name for name, _ in model.spectral_layers()] + assert "block_00_W_Q" in names + assert "block_03_W_MLP_OUT" in names + snapshot = _SpectralSnapshot(model) + assert len(snapshot.layers) == 24 + assert "token_embedding" not in snapshot.layers + + +def test_adamw_uses_decoupled_decay_and_explicit_lr_groups(): + config = tiny_training_config() + model = GPT(GPTConfig(**config["model"])) + optimizer = make_optimizers(model, config)[0] + assert isinstance(optimizer, torch.optim.AdamW) + assert {float(group["weight_decay"]) for group in optimizer.param_groups} == { + 0.0, + 0.01, + } + assert all(float(group["lr_multiplier"]) == 1.0 for group in optimizer.param_groups) -def test_progress_message_reports_elapsed_eta_and_stall(): +def test_progress_message_reports_tokens_elapsed_eta_and_stall(): message = _progress_message( - collected_bytes=50, - required_bytes=100, + collected_tokens=50, + required_tokens=100, documents=4, elapsed_seconds=10, stalled_seconds=3, ) assert "documents=4" in message + assert "tokens=50/100" in message assert "percent= 50.0%" in message assert "elapsed=10s" in message assert "eta=10s" in message - assert "no_new_bytes_for=3s" in message + assert "no_new_tokens_for=3s" in message -def test_verbose_split_preparation_logs_and_writes_files(tmp_path, capsys): - write_splits( - iter(["abcdef"]), +def test_gpt2_token_split_preparation_is_atomic_and_valid(tmp_path, capsys): + metadata = write_token_splits( + iter(["abcdef"] * 10), tmp_path, - train_bytes=2, - val_bytes=2, - test_bytes=2, + tokenizer=FakeTokenizer(), + train_tokens=8, + val_tokens=4, + test_tokens=4, + dataset_metadata={ + "dataset_name": "unit", + "dataset_config": "unit", + "dataset_split": "train", + "dataset_revision": "unit", + }, verbose=True, log_interval_seconds=60, ) - captured = capsys.readouterr() assert "[level0-prepare-data] starting" in captured.err - assert "[level0-prepare-data] collection complete" in captured.err - assert "[level0-prepare-data] complete" in captured.err - assert (tmp_path / "train.bin").stat().st_size == 2 - assert (tmp_path / "val.bin").stat().st_size == 2 - assert (tmp_path / "test.bin").stat().st_size == 2 - metadata = json.loads((tmp_path / "meta.json").read_text()) - assert metadata["sizes"] == {"train": 2, "val": 2, "test": 2} + assert "tokenization complete" in captured.err + assert metadata["tokenizer"] == "gpt2" + assert metadata["dtype"] == "uint16" + assert metadata["document_disjoint_splits"] is True + assert metadata["boundary_tokens_discarded"] > 0 + assert (tmp_path / "train.bin").stat().st_size == 16 + assert (tmp_path / "val.bin").stat().st_size == 8 + assert (tmp_path / "test.bin").stat().st_size == 8 + assert not list(tmp_path.glob("*.tmp")) + validated = validate_prepared_data(tmp_path, verify_hashes=True) + assert validated["split_tokens"] == {"train": 8, "val": 4, "test": 4} + + +def test_old_byte_data_is_rejected(tmp_path): + (tmp_path / "meta.json").write_text( + json.dumps({"tokenizer": "utf8-byte", "dtype": "uint8"}) + ) + with pytest.raises(ValueError, match="corrected GPT-2-BPE"): + validate_prepared_data(tmp_path) + + +def test_fixed_probes_do_not_advance_training_rng(): + data = np.arange(256, dtype=np.uint16) + training_generator = torch.Generator().manual_seed(123) + state_before = training_generator.get_state().clone() + first = make_fixed_probe( + data, + batch_size=2, + block_size=8, + batches=2, + seed=456, + ) + second = make_fixed_probe( + data, + batch_size=2, + block_size=8, + batches=2, + seed=456, + ) + assert torch.equal(training_generator.get_state(), state_before) + assert all( + torch.equal(a_tensor, b_tensor) + for a_batch, b_batch in zip(first, second) + for a_tensor, b_tensor in zip(a_batch, b_batch) + ) + + +def test_two_step_training_writes_complete_scientific_artifacts(tmp_path): + data_root = tmp_path / "data" + results_root = tmp_path / "results" + prepare_tiny_data(data_root) + run = run_training( + tiny_training_config(), + data_root=data_root, + results_root=results_root, + device="cpu", + ) + + completion = json.loads((run / "run_complete.json").read_text()) + selected = json.loads((run / "selected_checkpoint_metrics.json").read_text()) + final = json.loads((run / "final_metrics.json").read_text()) + metrics = pd.read_csv(run / "metrics.csv") + manifest = json.loads((run / "manifest.json").read_text()) + + assert completion["completed"] is True + assert completion["optimizer_steps"] == 2 + assert completion["tokens_seen"] == 64 + assert completion["weightwatcher_measurements"] == 0 + assert len(metrics) == 3 + assert list(metrics["step"]) == [0, 1, 2] + assert not any(column.startswith("test_") for column in metrics.columns) + assert selected["selected_step"] in {0, 1, 2} + assert np.isfinite(selected["test_loss"]) + assert np.isfinite(final["test_loss"]) + assert manifest["evaluation_protocol"]["test"] == ( + "final_and_validation_selected_only" + ) + assert manifest["evaluation_protocol"][ + "training_rng_isolated_from_evaluation" + ] is True + assert (run / "checkpoint_best.pt").is_file() + assert (run / "checkpoint_final.pt").is_file() + + +def test_nonempty_run_requires_explicit_overwrite(tmp_path): + data_root = tmp_path / "data" + results_root = tmp_path / "results" + prepare_tiny_data(data_root) + config = tiny_training_config() + run_training(config, data_root=data_root, results_root=results_root, device="cpu") + with pytest.raises(FileExistsError, match="--overwrite"): + run_training(config, data_root=data_root, results_root=results_root, device="cpu") From aa53cdc0ecf4eaaf526ea06bfb495d30e335091d Mon Sep 17 00:00:00 2001 From: Charles Martin Date: Wed, 29 Jul 2026 14:51:35 -0700 Subject: [PATCH 20/20] Remove temporary Level 0 source applicator --- .github/workflows/apply-realistic-level0.yml | 68 -------------------- 1 file changed, 68 deletions(-) delete mode 100644 .github/workflows/apply-realistic-level0.yml diff --git a/.github/workflows/apply-realistic-level0.yml b/.github/workflows/apply-realistic-level0.yml deleted file mode 100644 index 818f478..0000000 --- a/.github/workflows/apply-realistic-level0.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Apply corrected realistic isolated Level 0 baseline - -on: - push: - branches: - - agent/correct-realistic-level0-baseline - pull_request: - branches: - - main - types: - - opened - - synchronize - - reopened - -permissions: - contents: write - -jobs: - apply-level0: - if: >- - (github.event_name == 'push' && github.actor != 'github-actions[bot]') || - (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository && - github.event.pull_request.head.ref == 'agent/correct-realistic-level0-baseline') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: agent/correct-realistic-level0-baseline - fetch-depth: 0 - - - name: Reconstruct exact locally validated Level 0 tree - shell: bash - run: | - set -euo pipefail - cat .level0-update/part* | base64 --decode > /tmp/level0.tar.gz - echo '393ca945e35a43ec3a42092af6b8a5ed1fab1bc7c72ace6f5f80d9fb3ad1ac92 /tmp/level0.tar.gz' \ - | sha256sum --check --strict - rm -rf level_0_baseline - mkdir -p level_0_baseline - tar -xzf /tmp/level0.tar.gz -C level_0_baseline - chmod +x level_0_baseline/scripts/*.sh - rm -rf .level0-update - git add -A - git diff --cached --check - python -m compileall -q level_0_baseline/src level_0_baseline/tests - bash -n level_0_baseline/scripts/*.sh - - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install isolated Level 0 package and test tooling - run: | - python -m pip install --upgrade pip - python -m pip install --no-cache-dir -e 'level_0_baseline[test]' - - - name: Run isolated Level 0 regression suite - run: pytest -q level_0_baseline/tests/test_baseline.py - - - name: Commit exact validated implementation - shell: bash - run: | - set -euo pipefail - git config user.name github-actions[bot] - git config user.email 41898282+github-actions[bot]@users.noreply.github.com - git commit -m 'Correct isolated Level 0 GPT-2-BPE baseline' - git push origin HEAD:agent/correct-realistic-level0-baseline