diff --git a/.github/workflows/rom-tests.yml b/.github/workflows/rom-tests.yml index 2651fdb..e77586c 100644 --- a/.github/workflows/rom-tests.yml +++ b/.github/workflows/rom-tests.yml @@ -12,6 +12,8 @@ on: jobs: rom-tests: + # Every step below is a make target, so `make -C test-framework check` + # runs this same job locally, with the same flags, before pushing. name: run-roms runs-on: ubuntu-24.04 @@ -35,16 +37,35 @@ jobs: # The verdict engine and header decoder are pure Python; these run even # when the ROM list is empty, so the job still checks something useful. - name: Unit tests - run: python3 -m unittest discover -s test-framework/tests -v + run: make -C test-framework unit-tests - name: Run ROM tests run: | - python3 test-framework/run_tests.py \ - --rom-list test-framework/roms-ci.txt \ - --baseline test-framework/baseline.json \ - --json test-framework/results.json \ - --junit test-framework/results.xml \ - --github --no-color + make -C test-framework smoke SMOKE_EXTRA="\ + --json results.json \ + --junit results.xml \ + --github --no-color" + + # The test ROMs (blargg's suites, Bisqwit's ppu_read_buffer, nestest) are + # not stored in this repository (none carries a licence -- see + # test-framework/roms/README.md). They are downloaded from a pinned + # christopherpow/nes-test-roms commit and checked against the + # SHA-256 of every file in testroms.json, so the bytes CI runs are the + # bytes the baseline was recorded from. + - name: Cache test ROMs + uses: actions/cache@v6.1.0 + with: + path: test-framework/roms/external + key: testroms-${{ hashFiles('test-framework/testroms.json') }} + + # Unlike the smoke sweep above, these ROMs report their own verdict; + # `make testroms` carries the flags that reads it (see the Makefile). + - name: Run test ROMs (nes-test-roms) + run: | + make -C test-framework testroms TESTROM_EXTRA="\ + --json testroms-results.json \ + --junit testroms-results.xml \ + --github --no-color" - name: Upload results if: always() @@ -54,4 +75,6 @@ jobs: path: | test-framework/results.json test-framework/results.xml + test-framework/testroms-results.json + test-framework/testroms-results.xml if-no-files-found: ignore diff --git a/README.md b/README.md index 4eda5d4..a20bc46 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,43 @@ Note that no commercial ROM is (or should ever be) committed to the repository, so the list used by CI only contains license-clean ROMs. Point the framework at your own local collection when testing a change. +## Test ROMs + +Alongside the game sweep, CI runs the NESdev test suites — blargg's CPU, PPU, +APU and MMC3 tests, Bisqwit's PPU read-buffer test and kevtris's **nestest**, +112 ROMs from [nes-test-roms](https://github.com/christopherpow/nes-test-roms). +These do not need any heuristic: they exercise documented hardware behaviour +and report whether they passed, which the framework reads straight out of the +emulated machine (blargg's status byte at `$6000`, the result byte of his +2005-era runtime at `$F8` or `$F0`, nestest's result bytes at `$02`/`$03`). + +```sh +make -C test-framework testroms +``` + +That fetches the ROMs if they are not already there and runs them: + +``` + [PASS] m0 16-special.nes + [FAIL] m0 09-even_odd_frames.nes + -> blargg test failed (code 2): FD Pattern ----- should not skip + any clocks 09-even_odd_frames Failed #2 +``` + +The test ROMs are **not** stored in this repository. None of them ships a +licence, and rNES is BSD-3-Clause, so they are downloaded +from a pinned commit on GitHub and verified against a recorded SHA-256 rather +than redistributed here. See +[test-framework/roms/README.md](./test-framework/roms/README.md). + +Everything CI checks is available as a single target, so a change can be +verified before it is pushed: + +```sh +make -C test-framework check # strict build, unit tests, both ROM suites +make -C test-framework help # list every target +``` + See the [test framework documentation](./test-framework/README.md) for the complete list of options, the failure heuristics and the CI setup. diff --git a/src/apu.c b/src/apu.c index bb48f0a..9321367 100644 --- a/src/apu.c +++ b/src/apu.c @@ -506,6 +506,8 @@ static void frm_seq_clock(void) /* PS: We do nothing in Step 3 */ } else if (apu.frm_seq.cycles == frm_cycles[apu.type][4]) { /* Step 4 */ + frm_QuarterFrame(); + frm_HalfFrame(); apu.frm_seq.cycles = 0; return; } @@ -901,6 +903,44 @@ void apu_reset(void) audio_play(); } +/** + * Soft reset: what the APU does when the console's reset button is pressed. + * + * Unlike apu_reset() (power-on), a reset leaves most of the APU alone: + * - $4015 is cleared, silencing every channel; + * - $4017 is rewritten with the last value written to it, so the frame + * counter keeps its mode and restarts, and the frame IRQ flag is cleared; + * - the pulse and noise length counters are zeroed and un-halted, while the + * triangle is left as it was. + * + * https://www.nesdev.org/wiki/CPU_power_up_state + * Verified by blargg's apu_reset tests (4015_cleared, 4017_written, + * irq_flag_cleared, len_ctrs_enabled). + */ +void apu_soft_reset(void) +{ + int ch; + + for (ch = 0; ch < 2; ch++) { + apu.PWM[ch].lc.counter = 0; + apu.PWM[ch].lc.halt = 0; + apu.PWM[ch].lc.enable = 0; + } + apu.noise.lc.counter = 0; + apu.noise.lc.halt = 0; + apu.noise.lc.enable = 0; + /* The triangle keeps its length counter; it is only disabled */ + apu.triangle.lc.enable = 0; + + apu.status.reg.raw = 0; + apu.dmc.dma.bytes_remaining = 0; + apu.dmc.irq_flag = 0; + + apu.frm_seq.irq_flag = 0; + cpu_clear_irq(); + apu_reg_write(REG_FRM_SEQ, apu.frm_seq.last_write); +} + /** * Read one byte (8 bits) from the specified register * @param [in] reg Register address @@ -1043,6 +1083,7 @@ void apu_reg_write(uint16_t reg, uint8_t value) break; case REG_FRM_SEQ: + apu.frm_seq.last_write = value; if ((value & 0x80) == 0) apu.frm_seq.mode = 0; else diff --git a/src/include/apu.h b/src/include/apu.h index d370be9..3949302 100644 --- a/src/include/apu.h +++ b/src/include/apu.h @@ -83,6 +83,8 @@ struct _apu_frm_seq { uint8_t irq_inhibit; /** IRQ internal flag */ uint8_t irq_flag; + /** Last value written to $4017, rewritten on reset */ + uint8_t last_write; /** APU cycles */ uint32_t cycles; }; @@ -417,6 +419,7 @@ struct _apu_t { int apu_init(void); int apu_destroy(void); void apu_reset(void); +void apu_soft_reset(void); uint8_t apu_reg_read(uint16_t reg); void apu_reg_write(uint16_t reg, uint8_t value); void apu_clock(void); diff --git a/src/include/ppu.h b/src/include/ppu.h index dbd3c78..ab93349 100644 --- a/src/include/ppu.h +++ b/src/include/ppu.h @@ -201,8 +201,12 @@ struct _ppu { struct _OAM_sprite sprites[TOTAL_SPRITES_AUX]; uint8_t mem[TOTAL_SPRITES_AUX * sizeof(struct _OAM_sprite)]; } OAM_aux; - /** Dynamic latch */ + /** Open-bus decay register (the "dynamic latch") */ uint8_t gen_latch; + /** Frames left before each decay group falls back to zero */ + uint8_t decay_timer[3]; + /** Read buffer behind $2007 */ + uint8_t vram_buffer; /** Game cartridge */ cartridge_t *cartridge; /** Video standard */ diff --git a/src/ppu.c b/src/ppu.c index 3084651..a59c349 100644 --- a/src/ppu.c +++ b/src/ppu.c @@ -91,6 +91,16 @@ static inline uint8_t get_sprite_height(void) return 8; } +/** + * Check if a sprite whose top is at Y covers the scanline being evaluated + * @param [in] y Sprite's Y coordinate + * @return 1 if the sprite is in range, 0 otherwise + */ +static inline uint8_t sprite_in_range(uint8_t y) +{ + return ppu.scanline >= y && ppu.scanline < y + get_sprite_height(); +} + /** * Check if PPU rendering is enabled * @return uint8_t @@ -241,6 +251,63 @@ int ppu_destroy(void) return 0; } +/* + * Open-bus decay register. + * + * The PPU data bus has no driver of its own: reading a write-only port + * returns whatever charge the bus is still holding from the last time + * something drove it, and that charge leaks away in about 600 ms. It does + * not leak as one byte, though -- the register refreshes in three + * independent groups, and which groups a read refreshes depends on how many + * bits the PPU actually drove: + * + * $2002 drives bits 7-5 from the status register and leaves 4-0 to + * the bus, so it refreshes bit 5 and bits 7-6 only. + * $2007 drives all eight bits from VRAM, but only 5-0 when the + * address is in palette memory (which is 6 bits wide), so a + * palette read leaves bits 7-6 to decay. + * $2004 drives all eight. + * write-only drives nothing and refreshes nothing. + * + * Any write to a port drives the whole bus and refreshes everything. + * https://www.nesdev.org/wiki/PPU_registers#Ports + */ +#define DECAY_GROUPS 3 +/* ~600 ms at 60 fps. The bus is measured in frames here because that is the + * coarsest tick the tests can see: they sample 10 ms apart at the finest. */ +#define DECAY_FRAMES 36 +#define DECAY_LOW (1u << 0) /* bits 4-0 */ +#define DECAY_BIT5 (1u << 1) /* bit 5 */ +#define DECAY_HIGH (1u << 2) /* bits 7-6 */ +#define DECAY_ALL (DECAY_LOW | DECAY_BIT5 | DECAY_HIGH) + +static const uint8_t decay_mask[DECAY_GROUPS] = {0x1f, 0x20, 0xc0}; + +/* Drive `groups` of the bus with `value` and restart their decay. */ +static void decay_refresh(uint8_t value, uint8_t groups) +{ + int i; + + for (i = 0; i < DECAY_GROUPS; i++) { + if ((groups & (1u << i)) == 0) + continue; + ppu.gen_latch = (uint8_t)((ppu.gen_latch & ~decay_mask[i]) | + (value & decay_mask[i])); + ppu.decay_timer[i] = DECAY_FRAMES; + } +} + +/* One frame of leakage: a group nobody refreshed reaches zero. */ +static void decay_tick(void) +{ + int i; + + for (i = 0; i < DECAY_GROUPS; i++) { + if (ppu.decay_timer[i] > 0 && --ppu.decay_timer[i] == 0) + ppu.gen_latch &= (uint8_t)~decay_mask[i]; + } +} + /** * Resets PPU’s to initial state */ @@ -262,6 +329,10 @@ void ppu_reset(void) ppu.PPUSCROLL = 0; ppu.PPUDATA = 0; ppu.gen_latch = 0; + ppu.decay_timer[0] = 0; + ppu.decay_timer[1] = 0; + ppu.decay_timer[2] = 0; + ppu.vram_buffer = 0; ppu.age = 0; ppu.sprite_cnt = 0; @@ -301,22 +372,24 @@ uint8_t ppu_reg_read(uint16_t reg) switch (reg) { case REG_PPUCTRL: /* PPUCTRL: Write-Only */ - value = ppu.gen_latch | ppu.PPUSTATUS.reg.open_bus; + value = ppu.gen_latch; break; case REG_PPUMASK: /* PPUMASK: Write-Only */ - value = ppu.gen_latch | ppu.PPUSTATUS.reg.open_bus; + value = ppu.gen_latch; break; case REG_PPUSTATUS: /* PPUSTATUS: Read-Only */ value = (ppu.PPUSTATUS.raw & 0xe0) | (ppu.gen_latch & 0x1f); - ppu.gen_latch = value; + /* Bits 7-5 come from the status register, 4-0 from the bus: only + * the ones the PPU drove are refreshed. */ + decay_refresh(value, DECAY_BIT5 | DECAY_HIGH); ppu.iregs.w = 0; ppu.PPUSTATUS.reg.vblank = 0; break; case REG_OAMADDR: /* OAMADDR: Write-Only */ - value = ppu.gen_latch | ppu.PPUSTATUS.reg.open_bus; + value = ppu.gen_latch; break; case REG_OAMDATA: /* OAMDATA: Read/Write */ @@ -324,27 +397,35 @@ uint8_t ppu_reg_read(uint16_t reg) /* If we have byte 2 of a sprite, reserved bits must be zero */ if ((ppu.OAMADDR & 0x03) == 0x02) value &= 0xe3; - ppu.gen_latch = value; + decay_refresh(value, DECAY_ALL); break; case REG_PPUSCROLL: /* PPUSCROLL: Write twice */ - value = ppu.gen_latch | ppu.PPUSTATUS.reg.open_bus; + value = ppu.gen_latch; break; case REG_PPUADDR: /* PPUADDR: Write twice */ - value = ppu.gen_latch | ppu.PPUSTATUS.reg.open_bus; + value = ppu.gen_latch; break; case REG_PPUDATA: /* PPUDATA: Read/Write */ - value = ppu.gen_latch; - ppu.gen_latch = - cartridge_mem_read(ppu.cartridge, ppu.iregs.v.reg.raw, MEM_PPU); - - /* For palette memory, return the actual value */ if (ppu.iregs.v.reg.raw >= 0x3f00) { - value = ppu.gen_latch; - ppu.gen_latch = cartridge_mem_read( + /* Palette memory answers at once and is only 6 bits wide, so + * bits 7-6 are left to the bus and keep decaying. The read + * buffer is still refilled, from the nametable underneath. */ + value = (uint8_t)(cartridge_mem_read(ppu.cartridge, + ppu.iregs.v.reg.raw, MEM_PPU) & + 0x3f); + value |= (uint8_t)(ppu.gen_latch & 0xc0); + ppu.vram_buffer = cartridge_mem_read( ppu.cartridge, ppu.iregs.v.reg.raw & 0x2fff, MEM_PPU); + decay_refresh(value, DECAY_LOW | DECAY_BIT5); + } else { + /* Everything else is one read behind. */ + value = ppu.vram_buffer; + ppu.vram_buffer = + cartridge_mem_read(ppu.cartridge, ppu.iregs.v.reg.raw, MEM_PPU); + decay_refresh(value, DECAY_ALL); } if (ppu.PPUCTRL.reg.vram_inc == 0) @@ -354,7 +435,7 @@ uint8_t ppu_reg_read(uint16_t reg) break; case REG_OAMDMA: /* OAMDMA: Write-Only */ - value = ppu.gen_latch | ppu.PPUSTATUS.reg.open_bus; + value = ppu.gen_latch; break; } @@ -373,7 +454,7 @@ void ppu_reg_write(uint16_t reg, uint8_t value) /* Write to any register should set the latch */ if (reg >= REG_PPUCTRL && reg <= REG_PPUDATA) - ppu.gen_latch = value; + decay_refresh(value, DECAY_ALL); switch (reg) { case REG_PPUCTRL: /* PPUCTRL: Write-Only */ @@ -515,6 +596,7 @@ void ppu_clock(void) uint8_t attr, bpal, spal, p[2], a[2], sp[2]; uint8_t sp_lsb, sp_msb, sp_row, sp_prio, draw_sp, is_sp0; int nf; + uint8_t m; RGB_t bg_pixel, sp_pixel, draw_pixel; palette_t *sys_pal = get_palette(); @@ -649,29 +731,37 @@ void ppu_clock(void) ppu.OAM_aux.mem[i] = 0xff; ppu.sprite_cnt = 0; ppu.draw_sp0 = 0; - for (i = 0; i < TOTAL_SPRITES && ppu.sprite_cnt <= 8; i++) { + for (i = 0; i < TOTAL_SPRITES && ppu.sprite_cnt < 8; i++) { /* Check Y coordinate. Sprite should fit in the screen. * Y == 0 is a valid position, so it should not be skipped. * Otherwise sprite 0 placed at the top of the screen * won't trigger a sprite 0 hit. */ - if ((ppu.scanline >= ppu.OAM.sprites[i].Y) && - (ppu.scanline < - (ppu.OAM.sprites[i].Y + get_sprite_height()))) { - if (ppu.sprite_cnt < 8) { - /* Copy sprite to secondary OAM */ - ppu.OAM_aux.sprites[ppu.sprite_cnt] = - ppu.OAM.sprites[i]; - ppu.sprite_cnt++; - /* Is this the sprite zero? If so, it might have sprite - * collision */ - if (i == 0) - ppu.draw_sp0 = 1; - } else if (ppu.sprite_cnt++ == 8) { - if (is_rendering()) - ppu.PPUSTATUS.reg.sprite_overflow = 1; - } + if (sprite_in_range(ppu.OAM.sprites[i].Y)) { + /* Copy sprite to secondary OAM */ + ppu.OAM_aux.sprites[ppu.sprite_cnt] = ppu.OAM.sprites[i]; + ppu.sprite_cnt++; + /* Is this the sprite zero? If so, it might have sprite + * collision */ + if (i == 0) + ppu.draw_sp0 = 1; + } + } + + /* Sprite overflow, hardware bug included: once 8 sprites are + * found, the PPU keeps looking for a 9th, but each sprite that + * is not in range advances both the sprite index n and the byte + * index m. It therefore takes the tile, attribute or X byte of + * later sprites as their Y coordinate, giving both false + * positives and false negatives. + */ + for (m = 0; i < TOTAL_SPRITES; i++) { + if (sprite_in_range(ppu.OAM.mem[i * 4 + m])) { + if (is_rendering()) + ppu.PPUSTATUS.reg.sprite_overflow = 1; + break; } + m = (m + 1) & 0x3; } } @@ -880,6 +970,7 @@ void ppu_clock(void) if (ppu.scanline > 261) { ppu.scanline = 0; ppu.odd_frame = !ppu.odd_frame; + decay_tick(); } } } diff --git a/test-framework/.gitignore b/test-framework/.gitignore index d1301c6..ca3d5ef 100644 --- a/test-framework/.gitignore +++ b/test-framework/.gitignore @@ -1,9 +1,14 @@ # Built harness binary /rnes_headless +# Test ROMs downloaded by fetch-testroms.py. Not redistributable from here: +# blargg's suites and nestest carry no licence (see roms/README.md). +/roms/external/ + # Test outputs (regenerated by every run) /results.xml /results*.json +/testroms-results.* /failed-roms.txt *.junit.xml diff --git a/test-framework/Makefile b/test-framework/Makefile index 3fe10b2..d5605d3 100644 --- a/test-framework/Makefile +++ b/test-framework/Makefile @@ -56,7 +56,28 @@ STRICT_CFLAGS := -Wall -Wextra -Werror TARGET := rnes_headless -.PHONY: all clean sanitize strict +# --- Test ROM suite (blargg's NES tests + nestest) ------------------------- +# The ROMs are fetched rather than vendored -- neither suite carries a licence +# (see roms/README.md) -- so `testroms` downloads them first, verifying each +# against the SHA-256 in testroms.json. +# +# Self-checking ROMs need flags a game sweep does not: --test-mode auto reads +# the verdict blargg's ROMs publish at $6000 instead of guessing from the +# screen (a finished test parks the CPU in a `jmp *`, which the heuristics +# read as a hang), and --no-input keeps the menu masher away from ROMs that +# drive themselves. The frame budget only bounds a ROM that never reports: +# a run stops the moment one publishes its verdict. +TESTROM_FLAGS := --rom-list testroms-ci.txt \ + --test-mode auto --no-input \ + --frames 5400 --warmup 30 --interval 30 --timeout 180 + +# --- Game smoke sweep ------------------------------------------------------ +# The other half: real games, judged by the screen/CPU heuristics. roms/ ships +# empty, so this is a no-op until license-clean ROMs are added to roms-ci.txt. +SMOKE_FLAGS := --rom-list roms-ci.txt --baseline baseline.json + +.PHONY: all clean sanitize strict help unit-tests smoke \ + fetch-testroms testroms testroms-baseline check all: $(TARGET) @@ -82,5 +103,64 @@ sanitize: strict: $(CC) -fsyntax-only $(STRICT_CFLAGS) $(CFLAGS) $(HARNESS_SRCS) +# Download the test ROMs (idempotent: already-correct files are left alone). +fetch-testroms: + ./fetch-testroms.py + +# Run the test ROM suite, gated on testroms-baseline.json. TESTROM_EXTRA is +# how CI adds its reporting flags without restating the run's flags: +# make testroms TESTROM_EXTRA="--json r.json --junit r.xml --github" +testroms: $(TARGET) fetch-testroms + python3 run_tests.py $(TESTROM_FLAGS) --baseline testroms-baseline.json \ + $(TESTROM_EXTRA) + +# Re-record the expected verdicts. Run this after a change moves a test ROM +# from FAIL to PASS, and commit the updated baseline with the fix. +testroms-baseline: $(TARGET) fetch-testroms + python3 run_tests.py $(TESTROM_FLAGS) --baseline testroms-baseline.json \ + --update-baseline + +# The Python side: verdict engine, header decoder, ROM-list parsing, and the +# consistency of testroms.json against the ROM list and the baseline. +unit-tests: + python3 -m unittest discover -s tests + +# Run the game ROMs, gated on baseline.json. SMOKE_EXTRA mirrors TESTROM_EXTRA. +smoke: $(TARGET) + python3 run_tests.py $(SMOKE_FLAGS) $(SMOKE_EXTRA) + +# Everything .github/workflows/rom-tests.yml runs, in the same order, so a +# change can be checked before it is pushed rather than after. +# +# Sub-makes rather than prerequisites: these are ordered steps, and `make -j` +# is free to start prerequisites in any order (or at once). Each one aborts +# the target if it fails. +check: + @echo "== harness compiles clean ==" + @$(MAKE) --no-print-directory strict + @echo "== build headless harness ==" + @$(MAKE) --no-print-directory all + @echo "== unit tests ==" + @$(MAKE) --no-print-directory unit-tests + @echo "== game smoke sweep ==" + @$(MAKE) --no-print-directory smoke + @echo "== test ROMs (blargg + nestest) ==" + @$(MAKE) --no-print-directory testroms + @echo + @echo "all checks passed" + +help: + @echo "all : Build the headless harness (no SDL)" + @echo "check : Run everything CI runs, in order" + @echo "unit-tests : Run the Python unit tests" + @echo "smoke : Run the game ROMs against baseline.json" + @echo "testroms : Run blargg's suites + nestest (fetches them)" + @echo "testroms-baseline : Re-record the expected test ROM verdicts" + @echo "fetch-testroms : Download the test ROMs and verify checksums" + @echo "strict : Compile harness/ with warnings as errors" + @echo "sanitize : Build the harness under ASan+UBSan" + @echo "clean : Remove the built harness" + @echo "help : Show this help" + clean: @rm -f $(TARGET) diff --git a/test-framework/README.md b/test-framework/README.md index e81156f..6f99020 100644 --- a/test-framework/README.md +++ b/test-framework/README.md @@ -1,10 +1,15 @@ # rNES ROM Test Framework -Automated smoke-testing for [rNES](../README.md): run a batch of NES ROMs and -flag the obviously-broken ones — crashes, hangs, blank (black/gray/white) -screens, frozen screens, and CPU jams / infinite loops — without a display, -sound card, or human. Designed to run locally over a large ROM set **and** in -GitHub CI. +Automated testing for [rNES](../README.md), in two flavours, both without a +display, sound card, or human: + +- **A smoke sweep over real games** — run a batch of NES ROMs and flag the + obviously-broken ones: crashes, hangs, blank (black/gray/white) screens, + frozen screens, and CPU jams / infinite loops. Designed to run locally over + a large ROM set **and** in GitHub CI. +- **A test ROM suite** — [blargg's NES tests and nestest](#test-roms-blargg--nestest), + which are written to exercise documented hardware behaviour and report + whether they passed. No heuristics involved: the ROM says. ## How it works @@ -30,7 +35,11 @@ Two pieces: rnes_headless ──► JSON metrics │ ▼ - heuristics ─► PASS / FAIL / SKIP + did the ROM report a result? + │ yes ─► its own verdict + │ no ─► heuristics + ▼ + PASS / FAIL / SKIP │ ▼ compare vs baseline.json ─► regressions? ─► exit 1 @@ -40,6 +49,36 @@ Two pieces: ```sh make -C test-framework # produces test-framework/rnes_headless +make -C test-framework help # list every target +``` + +## Running everything CI runs + +`check` is the whole CI job in one target, in the same order and with the same +flags, so a change can be checked before it is pushed rather than after: + +```sh +make -C test-framework check +``` + +It runs `strict`, builds the harness, runs the unit tests, runs the game smoke +sweep, then fetches and runs the test ROMs — about a minute from scratch, most +of it the test ROMs. Each step is also a target on its own: + +| Target | What it runs | +|--------|--------------| +| `unit-tests` | the Python unit tests | +| `smoke` | the game ROMs (`roms-ci.txt`) against `baseline.json` | +| `testroms` | blargg's suites + nestest, fetching them first | +| `testroms-baseline` | re-record the expected test ROM verdicts | +| `strict` | compile `harness/` with warnings as errors | +| `sanitize` | rebuild the harness under ASan+UBSan | + +`SMOKE_EXTRA` and `TESTROM_EXTRA` add flags to the two ROM runs without +restating the ones the run needs — which is how CI attaches its reporting: + +```sh +make -C test-framework smoke SMOKE_EXTRA="--json results.json --github" ``` ### Extra checking @@ -120,6 +159,71 @@ animates → **PASS**. A genuinely hung game ignores the input and stays static - `--frames 600` — give games with unusually long intros more time to reach animating content. +## Test ROMs (blargg + nestest) + +A *game* can only be judged by how it behaves; a *test ROM* states its own +result. The suite understands three conventions, selected with `--test-mode`: + +| Mode | ROMs | How the result is read | +|------|------|------------------------| +| `off` (default) | games | not used — screen/CPU heuristics only | +| `auto` | blargg's suites | the ROM writes `$DE $B0 $61` to `$6001-$6003`, a status byte to `$6000` (`$80` running, `$81` wants a reset, otherwise the result code, `0` = passed) and an ASCII summary at `$6004` | +| `blargg2005` | blargg's 2005-era suites (`sprite_hit_tests`, `vbl_nmi_timing`, `blargg_apu_2005.07.30`, ...) | a zero-page result byte holds the number of the test running, then `1` if all passed; the run is over once the CPU parks in a `jmp *`. The byte is at `$F8` in the October 2005 runtime and at `$F0` in the July/September one -- pass `result-addr=0xf0` for those | +| `nestest` | nestest | entered at `$C000` instead of its reset vector, it runs every test unattended and leaves the result in zero page at `$02`/`$03`, both `0` when all passed | + +A ROM's own verdict wins outright — the screen heuristics never see it. They +would in fact get it backwards: a blargg test that has *finished* parks the +CPU in a `jmp *` on a screen that stops changing, which "CPU stuck in tight +loop" reads as a hang. The one check that still comes first is `cpu_jammed`: +a CPU that hit a JAM/KIL opcode stopped in the middle, so whatever the ROM had +written down is not the answer it meant to give. + +A run stops the moment a ROM publishes its result, so the frame budget only +bounds ROMs that never report at all. + +```sh +make -C test-framework testroms # fetch (if needed) + run + gate +make -C test-framework testroms-baseline # re-record the expected verdicts +``` + +The ROMs themselves are **not** in this repository — neither blargg's suites +nor nestest carries a licence, so rNES fetches them from a pinned upstream +commit and verifies every SHA-256 instead of redistributing them. See +[`roms/README.md`](roms/README.md) for the reasoning and for how to add more; +[`testroms.json`](testroms.json) is the manifest and +[`testroms-ci.txt`](testroms-ci.txt) the list CI runs. + +Failure reasons quote the ROM verbatim, which is usually enough to fix the bug +without opening an emulator: + +``` +[FAIL] m0 09-even_odd_frames.nes + -> blargg test failed (code 2): FD Pattern ----- should not skip any + clocks 09-even_odd_frames Failed #2 +[PASS] m0 16-special.nes + -> All 16 tests passed +``` + +### Per-ROM options + +A line in a ROM list may carry overrides after `" | "` (a pipe with spaces — +not plain whitespace, because ROM filenames are full of spaces): + +``` +roms/external/nestest/nestest.nes | test=nestest frames=120 input=none +``` + +`test=`, `frames=`, `warmup=`, `interval=`, `timeout=` and `ram-fill=` map to +the command-line options of the same name, and `result-addr=` to the harness's +`--result-addr`; `input=none` turns the menu masher +off and `input=start,select` sets the buttons. An unknown option is an error +rather than a silent default — a ROM run under flags nobody asked for would +have the resulting verdict recorded as its baseline. + +A ROM's baseline key is its filename, so two ROMs in one list must not share +one; the list is refused if they do. Give one of them a different `dest` name +in `testroms.json`. + ## What counts as a failure Applied to the *post-warmup* samples; a signal only fails a ROM if it holds for @@ -132,11 +236,16 @@ escapes a loop, passes). | `hang: wall-clock timeout` | ROM didn't finish within `--timeout` | | `emulator hang (no video output)` | PPU never produced a vblank | | `CPU jammed (JAM/KIL opcode)` | the CPU executed an illegal jam opcode | +| ` test failed (code N): …` | a [test ROM](#test-roms-blargg--nestest) reported a non-zero result, quoted verbatim | +| ` test did not finish` | a test ROM never reported: the frame budget ran out, or the run stopped early | | `blank/solid screen (#RRGGBB, N% drawn)` | ≤ `--blank-colors` colors, or less than `--min-content-frac` of the screen drawn, every sample — after running on to `--blank-frames` to confirm (this is the black/gray/white "freeze") | | `CPU stuck in tight loop` | ≤ `--loop-max-pcs` distinct executed PCs/frame, every sample | | `frozen screen (no change for N frames)` | **opt-in** (`--detect-freeze`): a *colourful* framebuffer that never changes | -`SKIP` covers unsupported mappers, non-NES files, and missing files. +`SKIP` covers unsupported mappers, non-NES files, and missing files. The two +test-ROM rows are the only ones that are not heuristics, and they apply only +under `--test-mode`; everything else in the table is judged from the screen and +the CPU. > **Why `frozen screen` is opt-in.** A *blank* screen (black/gray/white) that > is still blank once it has been given time to draw (see below) is a reliable @@ -218,13 +327,20 @@ Use `--fail-on-new` to also fail on ROMs that fail but aren't in the baseline. ## CI -[`.github/workflows/rom-tests.yml`](../.github/workflows/rom-tests.yml) builds -the harness and runs [`roms-ci.txt`](roms-ci.txt) against `baseline.json` on -every push/PR, emitting GitHub annotations + a step summary and uploading -`results.json` / `results.xml` (JUnit) artifacts. +[`.github/workflows/rom-tests.yml`](../.github/workflows/rom-tests.yml) runs on +every push/PR. Every step in it is one of the make targets above — `strict`, +the harness build, `unit-tests`, `smoke`, then `testroms` (with the ROMs cached +between runs, keyed on `testroms.json`) — so +[`make check`](#running-everything-ci-runs) reproduces the job locally and +there is no second copy of the flags to drift out of step. + +CI adds its reporting through `SMOKE_EXTRA` / `TESTROM_EXTRA`: GitHub +annotations, a step summary, and `results.json` / `results.xml` (JUnit) +artifacts. -Commercial ROMs can't be committed, so `roms/` ships empty (CI is a green no-op) -until you add **license-clean** ROMs — see [`roms/README.md`](roms/README.md). +Commercial ROMs can't be committed, so `roms/` ships empty (that half of CI is +a green no-op) until you add **license-clean** ROMs — see +[`roms/README.md`](roms/README.md). ## Useful options @@ -236,6 +352,8 @@ until you add **license-clean** ROMs — see [`roms/README.md`](roms/README.md). --jobs N ROMs to run in parallel (default: CPU count) --buttons LIST buttons to mash (default: start,a; e.g. start,a,select) --no-input don't inject any controller input (input is on by default) +--test-mode MODE read a test ROM's own verdict: off (default), auto + (blargg's $6000 report), nestest ($C000 entry, $02/$03) --blank-colors N distinct-color threshold for a flat screen (default 1) --min-content-frac F share of non-background pixels a screen needs to count as drawn (default 0.001 = 0.1%) @@ -253,6 +371,8 @@ until you add **license-clean** ROMs — see [`roms/README.md`](roms/README.md). ## Tests ```sh +make -C test-framework unit-tests +# or, equivalently: python3 -m unittest discover -s test-framework/tests ``` @@ -264,9 +384,14 @@ test-framework/ ├── rnestest/ # Python package (romheader, mappers, runner, │ # analyze, baseline, report, __main__) ├── run_tests.py # CLI wrapper +├── fetch-testroms.py # downloads the test ROMs (pinned + checksummed) ├── roms/ # license-clean CI ROMs (empty by default) -├── roms-ci.txt # ROM list used by CI -├── baseline.json # expected verdicts +│ └── external/ # downloaded test ROMs (git-ignored) +├── roms-ci.txt # game ROM list used by CI +├── baseline.json # expected verdicts for roms-ci.txt +├── testroms.json # test ROM manifest: upstream commit + SHA-256s +├── testroms-ci.txt # test ROM list used by CI +├── testroms-baseline.json # expected verdicts for testroms-ci.txt ├── tests/ # unit tests -└── Makefile # builds rnes_headless (no SDL) +└── Makefile # builds rnes_headless (no SDL), runs the test ROMs ``` diff --git a/test-framework/fetch-testroms.py b/test-framework/fetch-testroms.py new file mode 100755 index 0000000..8736ce7 --- /dev/null +++ b/test-framework/fetch-testroms.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# rNES test framework - test ROM fetcher +# Copyright 2021-2026 Renê de Souza Pinto +"""Download the NESdev test ROMs the CI suite runs against. + +The ROMs are *not* part of this repository. blargg's NES test suites and +kevtris's nestest are freely available and universally used to validate +emulators, but neither ships a licence, so rNES -- which is BSD-3-Clause -- +links to them rather than redistributing them. This script fetches them from +GitHub at a pinned commit, into a directory that is git-ignored. + +Every ROM is verified against the SHA-256 recorded in testroms.json, so a run +either reproduces the exact bytes the baseline was built from or fails; an +upstream force-push or a truncated download cannot quietly change a verdict. + + ./fetch-testroms.py # fetch into roms/external/ + ./fetch-testroms.py --check # verify what is there, download nothing + ./fetch-testroms.py --list # print the ROM list and exit + +Nothing outside the standard library is needed, so CI installs no extra +packages to run it. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_MANIFEST = os.path.join(HERE, "testroms.json") +DEFAULT_DEST = os.path.join(HERE, "roms", "external") +TIMEOUT = 60 +RETRIES = 3 + + +def load_manifest(path: str) -> dict: + with open(path, "r", encoding="utf-8") as fh: + return json.load(fh) + + +def rom_url(manifest: dict, rom: dict) -> str: + src = manifest["sources"][rom["source"]] + return src["raw"].format( + repo=src["repo"], + commit=src["commit"], + path=urllib.parse.quote(rom["path"]), + ) + + +def sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(65536), b""): + h.update(chunk) + return h.hexdigest() + + +def download(url: str) -> bytes: + """GET `url`, retrying a few times: CI networks drop connections.""" + last = None + for attempt in range(1, RETRIES + 1): + try: + with urllib.request.urlopen(url, timeout=TIMEOUT) as resp: + return resp.read() + except (urllib.error.URLError, OSError) as exc: # noqa: PERF203 + last = exc + if attempt < RETRIES: + print(f" retry {attempt}/{RETRIES - 1}: {exc}", + file=sys.stderr) + raise RuntimeError(f"download failed: {url} ({last})") + + +def fetch_one(manifest: dict, rom: dict, dest_dir: str, check_only: bool, + force: bool) -> str: + """Return one of 'ok' / 'fetched' / 'missing' / 'bad'.""" + target = os.path.join(dest_dir, rom["dest"]) + want = rom["sha256"] + + if os.path.exists(target) and not force: + if sha256_file(target) == want: + return "ok" + if check_only: + return "bad" + # A wrong file is replaced rather than trusted: it is either a + # partial download or a ROM from a different upstream revision. + print(f" ! {rom['dest']}: checksum mismatch, re-downloading") + elif check_only: + return "missing" + + data = download(rom_url(manifest, rom)) + got = hashlib.sha256(data).hexdigest() + if got != want: + raise RuntimeError( + f"{rom['dest']}: SHA-256 mismatch\n" + f" expected {want}\n got {got}\n" + f" from {rom_url(manifest, rom)}\n" + "The upstream file changed. Verify the new bytes by hand before " + "updating testroms.json, then regenerate the baseline." + ) + os.makedirs(os.path.dirname(target), exist_ok=True) + # Write-then-rename: an interrupted run leaves no half file behind for + # the next one to mistake for a complete download. + tmp = target + ".part" + with open(tmp, "wb") as fh: + fh.write(data) + os.replace(tmp, target) + return "fetched" + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser( + description="Fetch the NESdev test ROMs used by the rNES CI suite.") + ap.add_argument("--manifest", default=DEFAULT_MANIFEST, + help="ROM manifest (default: testroms.json)") + ap.add_argument("--dest", default=DEFAULT_DEST, + help="where to put them (default: roms/external/)") + ap.add_argument("--check", action="store_true", + help="verify the ROMs already present and exit non-zero " + "if any is missing or altered; downloads nothing") + ap.add_argument("--force", action="store_true", + help="re-download even ROMs that are already correct") + ap.add_argument("--list", action="store_true", + help="print the manifest's ROMs and exit") + args = ap.parse_args(argv) + + try: + manifest = load_manifest(args.manifest) + except OSError as exc: + print(f"cannot read manifest: {exc}", file=sys.stderr) + return 2 + + roms = manifest.get("roms") or [] + if args.list: + for rom in roms: + print(f"{rom['dest']}\t{rom['author']}\t{rom['path']}") + return 0 + + for name, src in manifest.get("sources", {}).items(): + print(f"{name}: {src['url']} @ {src['commit'][:12]}") + print(f"{len(roms)} ROMs -> {args.dest}") + + tally = {"ok": 0, "fetched": 0, "missing": 0, "bad": 0} + try: + for rom in roms: + state = fetch_one(manifest, rom, args.dest, args.check, args.force) + tally[state] += 1 + if state == "fetched": + print(f" + {rom['dest']}") + elif state == "missing": + print(f" - {rom['dest']}: not downloaded") + elif state == "bad": + print(f" ! {rom['dest']}: checksum mismatch") + except RuntimeError as exc: + print(f"\nerror: {exc}", file=sys.stderr) + return 1 + + print(f"verified {tally['ok']}, downloaded {tally['fetched']}" + + (f", missing {tally['missing']}" if tally["missing"] else "") + + (f", corrupt {tally['bad']}" if tally["bad"] else "")) + return 1 if (tally["missing"] or tally["bad"]) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test-framework/harness/rnes_headless.c b/test-framework/harness/rnes_headless.c index 0963620..2840abf 100644 --- a/test-framework/harness/rnes_headless.c +++ b/test-framework/harness/rnes_headless.c @@ -76,6 +76,8 @@ static uint8_t pc_seen[8192]; /* 1 bit per 16-bit PC address */ static unsigned long instr_count; /* total instructions executed */ static uint16_t last_pc; /* most recently executed PC */ static int cpu_jammed; /* set by the CPU kill (JAM) callback */ +static uint16_t watch_pc; /* PC to watch for (0 = none) */ +static int watch_pc_hit; /* set once watch_pc was reached */ static void debug_cb(cpu650x_state_t st) { @@ -83,6 +85,8 @@ static void debug_cb(cpu650x_state_t st) pc_seen[pc >> 3] |= (uint8_t)(1u << (pc & 7)); last_pc = pc; instr_count++; + if (watch_pc != 0 && pc == watch_pc) + watch_pc_hit = 1; } static void kill_cb(cpu650x_state_t st) @@ -99,6 +103,167 @@ static int popcount_pc_seen(void) return n; } +/* ------------------------------------------------------------------ */ +/* Test-ROM result protocols */ +/* ------------------------------------------------------------------ */ +/* + * Purpose-built test ROMs do not have to be judged by how their screen + * looks: they say outright whether they passed. Two conventions cover the + * suites this framework runs, and both are far more precise than any + * heuristic -- a test that finishes parks the CPU in a `jmp *`, which the + * screen/CPU analysis would otherwise read as a hang. + * + * blargg (instr_test-v5, instr_misc, ppu_vbl_nmi, cpu_interrupts_v2, ...) + * Once running, the ROM writes a three-byte signature to $6001-$6003 and + * keeps a status byte at $6000: $80 while the test runs, $81 when it wants + * a soft reset, and the final result code otherwise (0 = passed). A + * NUL-terminated ASCII summary follows at $6004. + * + * nestest (kevtris) + * Entered at $C000 it runs every test unattended and leaves its result in + * two zero-page bytes, $02 and $03, both 0 when everything passed. There + * is no signature to watch for, so this mode is requested explicitly. + * + * blargg's 2005-era runtime (sprite_hit_tests, sprite_overflow_tests, ...) + * Predates the $6000 report. It keeps a result byte at $F8 ($F0 in the + * July/September 2005 builds, see --result-addr) -- the number + * of the test being run, replaced by 1 if every test passed -- prints it + * on screen, beeps it, and then parks the CPU in a `jmp` to itself. $F8 + * holds the current test number throughout, so it says nothing about + * whether the run is over; the park is what says that. No signature + * either, so this mode is requested explicitly too. + * + * Polling is non-intrusive: sbus_read() serves $0000-$07FF from CPU RAM and + * $6000-$7FFF from the cartridge's PRG-RAM, neither of which has read side + * effects, so looking every frame cannot disturb what is being measured. + */ +enum test_mode { + TEST_OFF = 0, /* judge by screen/CPU heuristics alone (default) */ + TEST_AUTO, /* watch for blargg's $6000 report */ + TEST_NESTEST, /* enter nestest at $C000, read its $02/$03 result */ + TEST_BLARGG2005 /* read the result byte of blargg's older runtime */ +}; + +#define BLARGG_STATUS 0x6000 +#define BLARGG_TEXT (BLARGG_STATUS + 4) +#define BLARGG_TEXT_END 0x7fff /* last PRG-RAM address; never read past it */ +#define BLARGG_RUNNING 0x80 +#define BLARGG_NEED_RESET 0x81 +#define BLARGG_TEXT_MAX 512 +/* $81 asks for a reset "after at least 100 ms"; 8 frames is ~133 ms. */ +#define BLARGG_RESET_DELAY 8 +/* A ROM that keeps asking for resets is stuck in a loop, not making + * progress. The reset-driven suites need a handful, never dozens. */ +#define BLARGG_MAX_RESETS 8 +#define NESTEST_ENTRY 0xc000 +/* + * Where the unattended sweep ends: the RTS at $C66E, the last instruction + * of the canonical reference trace (nestest.log). Entered with a JMP, that + * RTS has no return address to pop, so it returns into whatever the stack + * happens to hold and everything past it is derailed code -- which sooner + * or later executes a JAM. Stopping here is what makes $02/$03 a result + * rather than a guess, and it keeps that artificial jam out of the report. + */ +#define NESTEST_DONE 0xc66e + +/* + * blargg's older runtime: default result byte, and the value meaning + * "passed". The runtime moved its result byte during 2005: the October + * suites (sprite_hit_tests, sprite_overflow_tests, vbl_nmi_timing, ...) + * keep it at $F8, the July/September ones (blargg_apu_2005.07.30, + * blargg_ppu_tests_2005.09.15b) at $F0. --result-addr picks the other one. + */ +#define B2005_RESULT 0x00f8 +#define B2005_PASSED 1 +/* + * How few distinct PCs in a frame count as "parked", and for how many + * frames in a row. A `jmp` to itself is one PC, but so is a vblank wait of + * two instructions, and that can span most of a frame -- it cannot span + * three of them, which is what the streak rules out. + */ +#define B2005_PARK_PCS 4 +#define B2005_PARK_FRAMES 3 + +/* True once the ROM has written blargg's $DE $B0 $61 signature. */ +static int blargg_signature_present(void) +{ + return sbus_read(BLARGG_STATUS + 1) == 0xde && + sbus_read(BLARGG_STATUS + 2) == 0xb0 && + sbus_read(BLARGG_STATUS + 3) == 0x61; +} + +/* + * Copy the NUL-terminated summary at $6004 out of PRG-RAM, flattened into + * something a JSON report and a CI annotation can carry on one line. + * + * The report spans several lines and is decorated with ANSI colour escapes + * (ESC '[' ... final byte), which would otherwise show up verbatim as + * "[0;37mTEST: ..." in every failure message. Escape sequences are dropped + * whole; newlines and any other control character become a space, and runs + * of spaces collapse. Anything non-ASCII means we are reading uninitialized + * RAM rather than a report, so it is treated the same way. + */ +static void blargg_read_text(char *buf, size_t max) +{ + /* Wider than the address it holds, so walking off the end of PRG-RAM + * stops the loop instead of wrapping to $0000 and reading the PPU and + * APU ports -- where a read is not free ($2002 clears the vblank flag, + * $2007 steps the VRAM address). */ + uint32_t src = BLARGG_TEXT; + size_t n = 0; + int space = 1; /* so leading whitespace is dropped too */ + uint8_t c = 0; + + while (n + 1 < max && src <= BLARGG_TEXT_END) { + c = sbus_read((uint16_t)src++); + if (c == 0) + break; + + if (c == 0x1b) { + /* ESC [ : dropped whole. */ + if (src > BLARGG_TEXT_END || sbus_read((uint16_t)src) != '[') + continue; + src++; + while (src <= BLARGG_TEXT_END) { + c = sbus_read((uint16_t)src++); + if (c == 0 || (c >= 0x40 && c <= 0x7e)) + break; + } + if (c == 0) + break; + continue; + } + + if (c < 0x20 || c >= 0x7f || c == ' ') { + space = 1; + continue; + } + if (space && n > 0) + buf[n++] = ' '; + space = 0; + if (n + 1 < max) + buf[n++] = (char)c; + } + buf[n] = '\0'; +} + +/* + * Point the reset vector at `entry` by patching the PRG image held in + * memory -- never the file on disk, which stays byte-for-byte upstream. + * $FFFC-$FFFD live in the 16 KiB bank mapped at $C000, which on the NROM + * boards this is used with (nestest among them) is the tail of PRG-ROM. + */ +static int patch_reset_vector(cartridge_t *cart, uint16_t entry) +{ + uint8_t *prg = cart->rom->prg_rom; + + if (prg == NULL || cart->rom->prg_size < 4) + return -1; + prg[cart->rom->prg_size - 4] = (uint8_t)(entry & 0xff); + prg[cart->rom->prg_size - 3] = (uint8_t)(entry >> 8); + return 0; +} + /* ------------------------------------------------------------------ */ /* Per-frame framebuffer analysis */ /* ------------------------------------------------------------------ */ @@ -204,6 +369,12 @@ static uint64_t run_one_frame(uint64_t tick_cap) uint8_t cur; sbus_clock(); ticks++; + /* Stop the instant a watched PC is reached rather than carrying on + * to the frame boundary: for nestest that PC is the end of the + * unattended sweep, and what runs after it is derailed code whose + * jam would otherwise land in this frame's report. */ + if (watch_pc_hit) + return ticks; cur = ppu.PPUSTATUS.reg.vblank; if (cur && !prev) return ticks; /* vblank rising edge -> frame complete */ @@ -284,7 +455,9 @@ static void usage(const char *argv0) fprintf(stderr, "Usage: %s [--frames N] [--warmup N] [--interval N] " "[--freeze-frames N] [--blank-frames N] [--buttons list] " - "[--ram-fill BYTE] [--no-input] [--json] \n" + "[--ram-fill BYTE] [--test MODE] [--result-addr ADDR] " + "[--no-input] [--json] " + "\n" " --freeze-frames N if the screen stays static, keep running up " "to N frames\n" " to confirm it is really frozen (0 = off)\n" @@ -296,8 +469,18 @@ static void usage(const char *argv0) "start,a,b,select,up,down,left,right (default: %s)\n" " --ram-fill BYTE power-on value for the 2 KiB of CPU RAM; keeps " "runs\n" - " reproducible (default 0x%02x)\n", - argv0, DEF_BLANK_FRAMES, DEF_BUTTONS, DEF_RAM_FILL); + " reproducible (default 0x%02x)\n" + " --test MODE read a test ROM's own verdict instead of " + "guessing from\n" + " the screen: off (default), auto (blargg's " + "$6000 report),\n" + " nestest (enter at $C000, read $02/$03),\n" + " blargg2005 (older runtime, result at " + "$F8)\n" + " --result-addr ADDR where blargg2005 reads its result (default " + "0x%02x;\n" + " the July/Sept 2005 suites use 0xf0)\n", + argv0, DEF_BLANK_FRAMES, DEF_BUTTONS, DEF_RAM_FILL, B2005_RESULT); } /* Map a button name to its raw controller_reg_t byte (0 if unknown). */ @@ -409,12 +592,22 @@ int main(int argc, char *argv[]) uint64_t base_hash = 0; const char *buttons_arg = DEF_BUTTONS; int ram_fill = DEF_RAM_FILL; + uint16_t b2005_result = B2005_RESULT; uint8_t btn_masks[MAX_BUTTONS]; int n_buttons = 0; rom_t *rom = NULL; cartridge_t *cart = NULL; rom_video_t video; int ret; + enum test_mode test_mode = TEST_OFF; + const char *test_proto = NULL; /* protocol that produced a result */ + int test_code = 0, test_done = 0, test_resets = 0; + int reset_pending = 0; /* frames left before a soft reset */ + int reset_served = 0; /* $81 already acted on, awaiting $80 */ + int parked_frames = 0; /* consecutive frames the CPU sat still */ + char test_text[BLARGG_TEXT_MAX]; + + test_text[0] = '\0'; /* Parse command line */ for (i = 1; i < argc; i++) { @@ -432,6 +625,24 @@ int main(int argc, char *argv[]) buttons_arg = argv[++i]; } else if (!strcmp(argv[i], "--ram-fill") && i + 1 < argc) { ram_fill = (int)strtol(argv[++i], NULL, 0) & 0xff; + } else if (!strcmp(argv[i], "--result-addr") && i + 1 < argc) { + b2005_result = (uint16_t)strtol(argv[++i], NULL, 0); + } else if (!strcmp(argv[i], "--test") && i + 1 < argc) { + const char *mode = argv[++i]; + if (!strcmp(mode, "off")) + test_mode = TEST_OFF; + else if (!strcmp(mode, "auto") || !strcmp(mode, "blargg")) + test_mode = TEST_AUTO; + else if (!strcmp(mode, "nestest")) + test_mode = TEST_NESTEST; + else if (!strcmp(mode, "blargg2005")) + test_mode = TEST_BLARGG2005; + else { + fprintf(stderr, "%s: bad --test mode: \"%s\"\n", argv[0], + mode); + usage(argv[0]); + return EXIT_USAGE; + } } else if (!strcmp(argv[i], "--no-input")) { inject_input = 0; } else if (!strcmp(argv[i], "--json")) { @@ -507,6 +718,19 @@ int main(int argc, char *argv[]) /* Initialize modules (same order as the real interface) */ sbus_init(); sbus_set_cartridge(cart); + + /* nestest's unattended mode is entered at $C000 rather than at the + * ROM's own reset vector, so the vector has to be redirected before + * cpu_reset() reads it. */ + if (test_mode == TEST_NESTEST) { + if (patch_reset_vector(cart, NESTEST_ENTRY) < 0) { + emit_status(rom_path, "error", "cannot_patch_reset_vector", + mapper); + return EXIT_LOADERR; + } + watch_pc = NESTEST_DONE; + } + fill_cpu_ram((uint8_t)ram_fill); /* deterministic power-on RAM */ cpu_init(); ppu_init(); @@ -570,6 +794,69 @@ int main(int argc, char *argv[]) if (ppu.PPUMASK.reg.show_background || ppu.PPUMASK.reg.show_sprites) rendering_ever = 1; + /* Has the test reported in yet? */ + if (test_mode == TEST_AUTO) { + if (reset_pending > 0) { + /* Serve a pending $81: a soft reset touches CPU, PPU and + * APU only -- CPU RAM and PRG-RAM keep their contents, + * which is exactly where the test's state lives. */ + if (--reset_pending == 0) { + cpu_reset(); + ppu_reset(); + apu_soft_reset(); + } + } else if (blargg_signature_present()) { + uint8_t st = sbus_read(BLARGG_STATUS); + + test_proto = "blargg"; + if (st == BLARGG_RUNNING) { + /* Back to running: whatever comes next is a fresh + * request, not the one already served. */ + reset_served = 0; + } else if (st == BLARGG_NEED_RESET) { + /* A reset leaves PRG-RAM alone, so $6000 still reads + * $81 for as long as it takes the ROM to restart and + * write $80 again. Acting on it twice resets the ROM + * a second time at an arbitrary point -- which is + * exactly what a reset test is measuring. Wait for + * $80 before honouring another request. */ + if (!reset_served && test_resets < BLARGG_MAX_RESETS) { + reset_pending = BLARGG_RESET_DELAY; + test_resets++; + reset_served = 1; + } + } else { + test_code = st; + test_done = 1; + blargg_read_text(test_text, sizeof(test_text)); + } + } + } + + /* blargg's older runtime parks the CPU once it has reported, so a + * run of frames with almost no PC movement is the signal that $F8 + * has stopped being "the test now running" and become the verdict. + * pc_seen still holds this frame at this point. */ + if (test_mode == TEST_BLARGG2005 && !test_done) { + if (popcount_pc_seen() <= B2005_PARK_PCS) + parked_frames++; + else + parked_frames = 0; + if (parked_frames >= B2005_PARK_FRAMES) { + uint8_t res = sbus_read(b2005_result); + + test_proto = "blargg2005"; + test_code = (res == B2005_PASSED) ? 0 : res; + test_done = 1; + if (res == B2005_PASSED) + snprintf(test_text, sizeof(test_text), "passed"); + else + snprintf(test_text, sizeof(test_text), + "failed #%u ($%02X=%02x)", res, b2005_result, + res); + } + } + /* Sample after warm-up, every `interval` frames */ if (frames_run + 1 > warmup && ((frames_run + 1) % interval) == 0 && nsamples < MAX_SAMPLES) { @@ -625,6 +912,14 @@ int main(int argc, char *argv[]) } prev_instr = instr_count; + /* A test ROM that has published its verdict has nothing left to + * say; the `frames_run++` turns this frame's 0-based index into + * the 1-based count the JSON report carries. */ + if (test_done || watch_pc_hit) { + frames_run++; + break; + } + /* Once past the base run, stop as soon as the question that kept us * running is answered: the screen animates, and -- when we extended * to wait out a blank screen -- something has actually been drawn. */ @@ -633,6 +928,40 @@ int main(int argc, char *argv[]) break; } + /* + * nestest reports through zero page instead of a signature, so its + * result is collected once the run is over. The unattended sweep takes + * about 27,000 cycles -- less than one NTSC frame -- and then falls + * through into the ROM's interactive menu, which leaves $02/$03 alone; + * reading them at the end of the run is therefore the same as reading + * them the moment the sweep finished. + * + * Two bytes of zero only mean "passed" if the sweep actually ran to the + * end, and reaching $C66E is what says so. Without that, every test the + * sweep never got to would read as a test that passed. + */ + /* Never parked: the run used up its frames without reporting. Say so + * with whatever the result byte held, which names the test it was + * stuck on. */ + if (test_mode == TEST_BLARGG2005 && !test_done) { + uint8_t res = sbus_read(b2005_result); + + test_proto = "blargg2005"; + test_code = res; + snprintf(test_text, sizeof(test_text), + "never reported (stuck in test #%u)", res); + } + + if (test_mode == TEST_NESTEST) { + int lo = sbus_read(0x0002); + int hi = sbus_read(0x0003); + + test_proto = "nestest"; + test_code = (hi << 8) | lo; + test_done = status_ok && watch_pc_hit; + snprintf(test_text, sizeof(test_text), "$02=%02x $03=%02x", lo, hi); + } + /* -------------------------------------------------------------- */ /* Emit the full JSON report */ /* -------------------------------------------------------------- */ @@ -655,6 +984,16 @@ int main(int argc, char *argv[]) json_puts_escaped(buttons_arg); } printf(",\"cpu_jammed\":%s", cpu_jammed ? "true" : "false"); + if (test_proto != NULL) { + printf(",\"test_protocol\":"); + json_puts_escaped(test_proto); + printf(",\"test_done\":%s", test_done ? "true" : "false"); + if (test_done) + printf(",\"test_code\":%d", test_code); + printf(",\"test_resets\":%d", test_resets); + printf(",\"test_text\":"); + json_puts_escaped(test_text); + } printf(",\"total_instructions\":%lu", instr_count); printf(",\"rendering_ever_enabled\":%s", rendering_ever ? "true" : "false"); printf(",\"samples\":["); diff --git a/test-framework/rnestest/__main__.py b/test-framework/rnestest/__main__.py index 9a04c4b..1a15b2e 100644 --- a/test-framework/rnestest/__main__.py +++ b/test-framework/rnestest/__main__.py @@ -9,6 +9,7 @@ import argparse import concurrent.futures +import dataclasses import datetime import os import sys @@ -20,7 +21,8 @@ from .report import (TestRecord, github_annotations, print_summary, write_json, write_junit) from .romheader import parse_rom_header -from .runner import RunConfig, default_key, make_skip_result, run_rom +from .runner import (TEST_MODES, RunConfig, default_key, make_skip_result, + run_rom) def _default_harness() -> str: @@ -63,6 +65,12 @@ def parse_args(argv=None) -> argparse.Namespace: help="comma list of buttons to mash " "(start,a,b,select,up,down,left,right); " "default: start,a") + p.add_argument("--test-mode", choices=TEST_MODES, default="off", + help="read a test ROM's own verdict instead of guessing " + "from the screen: off (default), auto (blargg's " + "$6000 report), nestest (enter nestest at $C000 and " + "read $02/$03). A per-ROM 'test=' option in the ROM " + "list overrides this") p.add_argument("--ram-fill", type=lambda v: int(v, 0), default=None, help="byte the 2 KiB of CPU RAM powers on with (default " "0x00). The emulator itself leaves it to malloc(); " @@ -114,16 +122,89 @@ def parse_args(argv=None) -> argparse.Namespace: return p.parse_args(argv) +#: Per-ROM options: list-file key -> (RunConfig field, parser). +_ROM_OPTIONS = { + "test": ("test_mode", lambda v: _one_of(v, TEST_MODES, "test")), + "frames": ("frames", int), + "warmup": ("warmup", int), + "interval": ("interval", int), + "timeout": ("timeout", float), + "ram-fill": ("ram_fill", lambda v: int(v, 0)), + "result-addr": ("result_addr", lambda v: int(v, 0)), +} + +#: Separator between a ROM path and its options. Deliberately not plain +#: whitespace: ROM filenames are full of spaces ("Some Game (USA).nes"). +OPT_SEP = " | " + + +def _one_of(value: str, allowed, what: str) -> str: + if value not in allowed: + raise ValueError(f"{what} must be one of {', '.join(allowed)}") + return value + + +def parse_rom_options(text: str) -> dict: + """Parse ``key=value key=value`` into RunConfig overrides. + + ``input=none`` turns button mashing off; any other value is a button + list ("start,a,select"), which implies mashing is on. + """ + overrides = {} + for token in text.split(): + if "=" not in token: + raise ValueError(f"option {token!r} is not key=value") + key, value = token.split("=", 1) + if key == "input": + if value == "none": + overrides["inject_input"] = False + else: + overrides["inject_input"] = True + overrides["buttons"] = value + continue + if key not in _ROM_OPTIONS: + known = ", ".join(sorted(list(_ROM_OPTIONS) + ["input"])) + raise ValueError(f"unknown option {key!r} (known: {known})") + field, parse = _ROM_OPTIONS[key] + try: + overrides[field] = parse(value) + except ValueError as exc: + raise ValueError(f"bad value for {key!r}: {exc}") from exc + return overrides + + def read_rom_list(path: str, roms_dir: str) -> list: - """Return a list of (resolved_path, key) from the list file.""" + """Return a list of (resolved_path, key, overrides) from the list file. + + A line is a ROM path, optionally followed by ``" | "`` and per-ROM + options that override the command line for that ROM alone:: + + roms/external/blargg/instr_test-v5/official_only.nes | test=auto + roms/external/nestest/nestest.nes | test=nestest frames=120 input=none + """ entries = [] + seen = {} with open(path, "r", encoding="utf-8") as fh: - for raw in fh: + for lineno, raw in enumerate(fh, 1): line = raw.strip() if not line or line.startswith("#"): continue - resolved = line if os.path.isabs(line) else os.path.join(roms_dir, line) - entries.append((os.path.normpath(resolved), default_key(line))) + rom, _, opt_text = line.partition(OPT_SEP) + rom = rom.strip() + try: + overrides = parse_rom_options(opt_text) if opt_text else {} + except ValueError as exc: + raise ValueError(f"{path}:{lineno}: {exc}") from exc + resolved = rom if os.path.isabs(rom) else os.path.join(roms_dir, rom) + key = default_key(rom) + # The key names the ROM in the baseline. Two ROMs sharing one + # would overwrite each other's verdict without a word. + if key in seen: + raise ValueError(f"{path}:{lineno}: {rom!r} has the same " + f"baseline key {key!r} as line {seen[key]}; " + "rename one of them") + seen[key] = lineno + entries.append((os.path.normpath(resolved), key, overrides)) return entries @@ -134,7 +215,11 @@ def main(argv=None) -> int: print(f"rom list not found: {args.rom_list}", file=sys.stderr) return 2 roms_dir = args.roms or os.path.dirname(os.path.abspath(args.rom_list)) - entries = read_rom_list(args.rom_list, roms_dir) + try: + entries = read_rom_list(args.rom_list, roms_dir) + except ValueError as exc: + print(exc, file=sys.stderr) + return 2 if not entries: print(f"No ROMs listed in {args.rom_list}; nothing to test.") @@ -159,6 +244,7 @@ def main(argv=None) -> int: freeze_frames=(args.freeze_frames if args.detect_freeze else None), blank_frames=args.blank_frames, ram_fill=args.ram_fill, + test_mode=args.test_mode, ) if not os.path.isfile(args.harness): @@ -168,10 +254,10 @@ def main(argv=None) -> int: return 2 # Decide per-ROM action: skip immediately, or run the harness. - to_run = [] # (idx, path, key, info) + to_run = [] # (idx, path, key, cfg) prebuilt = {} # idx -> RunResult (for pre-skips) infos = {} # idx -> RomInfo - for idx, (path, key) in enumerate(entries): + for idx, (path, key, overrides) in enumerate(entries): if not os.path.exists(path): prebuilt[idx] = make_skip_result(path, key, "file not found") infos[idx] = None @@ -185,7 +271,8 @@ def main(argv=None) -> int: prebuilt[idx] = make_skip_result( path, key, f"unsupported mapper {info.mapper}", info.mapper) else: - to_run.append((idx, path, key, info)) + rom_cfg = dataclasses.replace(cfg, **overrides) if overrides else cfg + to_run.append((idx, path, key, rom_cfg)) # Run the supported ROMs (parallel). results = dict(prebuilt) @@ -195,8 +282,8 @@ def main(argv=None) -> int: with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, args.jobs)) as ex: futs = { - ex.submit(run_rom, path, key, cfg): idx - for (idx, path, key, info) in to_run + ex.submit(run_rom, path, key, rom_cfg): idx + for (idx, path, key, rom_cfg) in to_run } # account for pre-skipped ones in the progress counter done = len(prebuilt) @@ -211,7 +298,7 @@ def main(argv=None) -> int: # Build records. records = [] - for idx, (path, key) in enumerate(entries): + for idx, (path, key, _opts) in enumerate(entries): res = results[idx] v = analyze(res, th) info = infos.get(idx) diff --git a/test-framework/rnestest/analyze.py b/test-framework/rnestest/analyze.py index e45fa33..64914d1 100644 --- a/test-framework/rnestest/analyze.py +++ b/test-framework/rnestest/analyze.py @@ -1,9 +1,15 @@ """Turn a harness :class:`RunResult` into a PASS / FAIL / SKIP verdict. -Heuristics, in priority order (first match wins). All post-warmup *samples* -are considered; a signal only fails the ROM if it holds for the whole window, -which keeps false positives low (a screen that animates even once, or a CPU -that ever escapes a tight loop, passes). +Purpose-built test ROMs (blargg's suites, nestest) are judged first and on +their own terms: run under ``--test``, the harness reports the verdict the +ROM itself published, and none of the heuristics below apply to them. They +would in fact get it backwards -- a blargg test that has finished parks the +CPU in a `jmp *`, which "CPU stuck in tight loop" reads as a hang. + +Everything else is judged by heuristics, in priority order (first match +wins). All post-warmup *samples* are considered; a signal only fails the ROM +if it holds for the whole window, which keeps false positives low (a screen +that animates even once, or a CPU that ever escapes a tight loop, passes). Definitive failures (very low false-positive rate): - crashed (killed by a signal) @@ -37,6 +43,11 @@ FAIL = "FAIL" SKIP = "SKIP" +# blargg's summary runs to a few hundred characters; a CI annotation and a +# console line only have room for the part that says what went wrong, which +# the ROM puts first. +REASON_MAX = 220 + @dataclass class Thresholds: @@ -56,6 +67,42 @@ class Verdict: category: str # short machine-friendly tag +def _clip(text: str) -> str: + text = " ".join(text.split()) + if len(text) <= REASON_MAX: + return text + return text[:REASON_MAX - 1].rstrip() + "\u2026" + + +def testrom_verdict(report: dict) -> Optional[Verdict]: + """Verdict published by the ROM itself, or None if it published none. + + Only set when the harness ran with ``--test`` *and* the ROM actually + reported: ``--test auto`` on a ROM with no blargg signature leaves the + field absent, and the caller falls back to the screen heuristics. + """ + proto = report.get("test_protocol") + if not proto: + return None + + text = (report.get("test_text") or "").strip() + detail = f": {text}" if text else "" + + if not report.get("test_done"): + # Stopped somewhere in the middle: the frame budget ran out, the CPU + # jammed, or the emulator stopped producing frames. Whatever the ROM + # had written by then is a partial result, not a verdict. + return Verdict(FAIL, _clip(f"{proto} test did not finish{detail}"), + "test-incomplete") + + code = report.get("test_code") + if code == 0: + return Verdict(PASS, _clip(text or f"{proto} test passed"), + "test-pass") + return Verdict(FAIL, _clip(f"{proto} test failed (code {code}){detail}"), + "test-fail") + + def analyze(res: RunResult, th: Optional[Thresholds] = None) -> Verdict: th = th or Thresholds() @@ -91,9 +138,17 @@ def analyze(res: RunResult, th: Optional[Thresholds] = None) -> Verdict: return Verdict(FAIL, f"unexpected status '{status}'", "bad-status") # --- Emulation-level failures --------------------------------------- + # Ahead of the ROM's own verdict: a jammed CPU means the test stopped + # early, so whatever it had written down is not the answer it meant to + # give -- and "JAM/KIL opcode" says far more than "did not finish". if report.get("cpu_jammed"): return Verdict(FAIL, "CPU jammed (JAM/KIL opcode)", "cpu-jam") + # --- A test ROM's own verdict ---------------------------------------- + tv = testrom_verdict(report) + if tv is not None: + return tv + samples = report.get("samples") or [] if not samples: # Ran but produced no samples (frames < warmup). Not enough to judge. diff --git a/test-framework/rnestest/runner.py b/test-framework/rnestest/runner.py index e6111fd..0a3acaf 100644 --- a/test-framework/rnestest/runner.py +++ b/test-framework/rnestest/runner.py @@ -23,6 +23,19 @@ class RunConfig: # Power-on CPU RAM byte. None = the harness default (0x00). Fixed rather # than left to malloc() so a verdict is reproducible; see --ram-fill. ram_fill: Optional[int] = None + # Read a test ROM's own verdict instead of guessing from the screen: + # "off" heuristics only (default; what a game sweep wants) + # "auto" watch for blargg's $6000 report + # "nestest" enter nestest at $C000 and read its $02/$03 result + # "blargg2005" blargg's older runtime, result byte at $F8 + test_mode: str = "off" + # Where "blargg2005" reads its result byte. None = the harness default + # ($F8); the July/September 2005 suites keep it at $F0. + result_addr: Optional[int] = None + + +#: ``--test`` modes the harness accepts. +TEST_MODES = ("off", "auto", "nestest", "blargg2005") @dataclass @@ -62,6 +75,10 @@ def run_rom(rom_path: str, key: str, cfg: RunConfig) -> RunResult: cmd += ["--blank-frames", str(cfg.blank_frames)] if cfg.ram_fill is not None: cmd += ["--ram-fill", str(cfg.ram_fill)] + if cfg.test_mode and cfg.test_mode != "off": + cmd += ["--test", cfg.test_mode] + if cfg.result_addr is not None: + cmd += ["--result-addr", hex(cfg.result_addr)] cmd.append(rom_path) try: proc = subprocess.run( diff --git a/test-framework/roms/README.md b/test-framework/roms/README.md index 27fb449..e4ddc8d 100644 --- a/test-framework/roms/README.md +++ b/test-framework/roms/README.md @@ -31,3 +31,58 @@ locally against your own list file: ```sh python3 test-framework/run_tests.py --rom-list ~/my-roms.txt ``` + +## `external/` — the downloaded test ROMs + +CI also runs blargg's NES test suites and kevtris's **nestest**, which check +the emulator against documented hardware behaviour rather than just looking at +the screen. Those ROMs live in `external/`, are **not** committed, and are +git-ignored. Fetch them with: + +```sh +make -C test-framework fetch-testroms # or just: make -C test-framework testroms +``` + +which downloads every file listed in [`../testroms.json`](../testroms.json) +from [christopherpow/nes-test-roms](https://github.com/christopherpow/nes-test-roms) +at a **pinned commit** and verifies each against a recorded SHA-256. + +### Why they are downloaded instead of committed + +rNES is BSD-3-Clause, and everything in this repository has to be compatible +with that. These ROMs are not: + +- **blargg's NES test ROMs** (Shay Green) ship a `readme.txt` describing the + test and nothing else. There is no copyright notice, no licence, and no + statement of permission to redistribute. +- **ppu_read_buffer** (Joel Yliluoma, "Bisqwit") is the same: a long + `readme.txt` and source, no licence. +- **nestest** (Kevin Horton) is the same: `nestest.txt` documents the failure + codes and says nothing about licensing. +- The upstream archive that hosts them all has no repository-wide `LICENSE` file + either; it is a community mirror, not a relicensing. + +Both are freely downloadable and are the standard way to validate an NES +emulator, so *running* them in CI is uncontroversial. **Redistributing** them +from a BSD-3-Clause repository is a different act, and one nobody has given +permission for. Fetching at test time keeps rNES's own licensing clean: the +repository ships a URL, a commit hash and a checksum — never the bytes. + +The pinned commit and the per-file SHA-256 are what make this reproducible: a +force-push upstream, a replaced file or a truncated download fails the fetch +instead of quietly changing a verdict. + +### Adding another test ROM + +Only add ROMs that are downloadable from a **GitHub** repository, and check +what the licence actually permits before deciding whether to fetch it (like +these) or commit it (like a permissively-licensed homebrew ROM). + +1. Add an entry to `../testroms.json` with its upstream `path`, the `dest` it + should land under, its `author`, `size` and `sha256`. +2. List it in [`../testroms-ci.txt`](../testroms-ci.txt), with per-ROM options + after `" | "` if it needs any. +3. Regenerate the baseline and commit it: + ```sh + make -C test-framework testroms-baseline + ``` diff --git a/test-framework/testroms-baseline.json b/test-framework/testroms-baseline.json new file mode 100644 index 0000000..d7bcf2f --- /dev/null +++ b/test-framework/testroms-baseline.json @@ -0,0 +1,456 @@ +{ + "meta": { + "generated": "2026-10-25T19:28:33", + "note": "generated by rnestest from testroms-ci.txt" + }, + "roms": { + "01-abs_x_wrap.nes": { + "verdict": "PASS", + "reason": "01-abs_x_wrap Passed" + }, + "01-basics.nes": { + "verdict": "PASS", + "reason": "01-basics Passed" + }, + "01-vbl_basics.nes": { + "verdict": "PASS", + "reason": "01-vbl_basics Passed" + }, + "01.basics.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "01.len_ctr.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "02-branch_wrap.nes": { + "verdict": "PASS", + "reason": "02-branch_wrap Passed" + }, + "02-implied.nes": { + "verdict": "PASS", + "reason": "02-implied Passed" + }, + "02-vbl_set_time.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): T+ 1 2 00 - V 01 - V 02 - V 03 - V 04 - V 05 V - 06 V - 07 V - 08 V - 4103C340 02-vbl_set_time Failed" + }, + "02.alignment.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "02.len_table.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "03-dummy_reads.nes": { + "verdict": "PASS", + "reason": "03-dummy_reads Passed" + }, + "03-immediate.nes": { + "verdict": "PASS", + "reason": "03-immediate Passed" + }, + "03-vbl_clear_time.nes": { + "verdict": "PASS", + "reason": "00 V 01 V 02 V 03 V 04 V 05 V 06 - 07 - 08 - 03-vbl_clear_time Passed" + }, + "03.corners.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "03.irq_flag.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "04-dummy_reads_apu.nes": { + "verdict": "PASS", + "reason": "04-dummy_reads_apu Passed" + }, + "04-nmi_control.nes": { + "verdict": "PASS", + "reason": "04-nmi_control Passed" + }, + "04-zero_page.nes": { + "verdict": "PASS", + "reason": "04-zero_page Passed" + }, + "04.clock_jitter.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "04.flip.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "05-nmi_timing.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): 00 4 01 3 02 3 03 3 04 3 05 3 06 3 07 2 08 2 09 2 21FD81DC 05-nmi_timing Failed" + }, + "05-zp_xy.nes": { + "verdict": "PASS", + "reason": "05-zp_xy Passed" + }, + "05.left_clip.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "05.len_timing_mode0.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 2): failed #2 ($F0=02)" + }, + "06-absolute.nes": { + "verdict": "PASS", + "reason": "06-absolute Passed" + }, + "06-suppression.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): 00 - N 01 - N 02 - N 03 - N 04 - N 05 V N 06 V N 07 V N 08 V N 09 V N 3FE15516 06-suppression Failed" + }, + "06.len_timing_mode1.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 2): failed #2 ($F0=02)" + }, + "06.right_edge.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "07-abs_xy.nes": { + "verdict": "PASS", + "reason": "07-abs_xy Passed" + }, + "07-nmi_on_timing.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): 00 N 01 N 02 N 03 N 04 N 05 N 06 - 07 - 08 - 2B1F5269 07-nmi_on_timing Failed" + }, + "07.irq_flag_timing.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "07.screen_bottom.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "08-ind_x.nes": { + "verdict": "PASS", + "reason": "08-ind_x Passed" + }, + "08-nmi_off_timing.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): 03 - 04 - 05 - 06 - 07 - 08 - 09 - 0A - 0B - 0C - FA8C430B 08-nmi_off_timing Failed" + }, + "08.double_height.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "08.irq_timing.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 2): failed #2 ($F0=02)" + }, + "09-even_odd_frames.nes": { + "verdict": "PASS", + "reason": "00 01 01 02 09-even_odd_frames Passed" + }, + "09-ind_y.nes": { + "verdict": "PASS", + "reason": "09-ind_y Passed" + }, + "09.reset_timing.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "09.timing_basics.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "1-cli_latency.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 10): CLI RTI should not allow any IRQs 1-cli_latency Failed #10" + }, + "1-instr_timing.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 4): Instruction timing test Takes about 25 seconds. Doesn't time the 8 branches and 12 illegal instructions. Official instructions... NOPs and alternate SBC... Unofficial instructions... 8B was…" + }, + "1-len_ctr.nes": { + "verdict": "PASS", + "reason": "1-len_ctr Passed" + }, + "1.Basics.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "1.Branch_Basics.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "1.Clocking.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 3): failed #3 ($F8=03)" + }, + "1.frame_basics.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "10-branches.nes": { + "verdict": "PASS", + "reason": "10-branches Passed" + }, + "10-even_odd_timing.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 3): 08 07 Clock is skipped too late, relative to enabling BG 10-even_odd_timing Failed #3" + }, + "10.len_halt_timing.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "10.timing_order.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "11-stack.nes": { + "verdict": "PASS", + "reason": "11-stack Passed" + }, + "11.edge_timing.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "11.len_reload_timing.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 4): failed #4 ($F0=04)" + }, + "12-jmp_jsr.nes": { + "verdict": "PASS", + "reason": "12-jmp_jsr Passed" + }, + "13-rts.nes": { + "verdict": "PASS", + "reason": "13-rts Passed" + }, + "14-rti.nes": { + "verdict": "PASS", + "reason": "14-rti Passed" + }, + "15-brk.nes": { + "verdict": "PASS", + "reason": "15-brk Passed" + }, + "16-special.nes": { + "verdict": "PASS", + "reason": "16-special Passed" + }, + "2-branch_timing.nes": { + "verdict": "PASS", + "reason": "2-branch_timing Passed" + }, + "2-len_table.nes": { + "verdict": "PASS", + "reason": "2-len_table Passed" + }, + "2-nmi_and_brk.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): NMI BRK 00 27 36 00 27 36 00 26 36 00 26 36 00 26 36 00 26 36 00 26 36 00 26 36 00 26 36 00 26 36 00 A58D3981 2-nmi_and_brk Failed" + }, + "2.Backward_Branch.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "2.Details.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "2.MMC3_Details.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 2): failed #2 ($F8=02)" + }, + "2.vbl_timing.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 8): failed #8 ($F8=08)" + }, + "3-irq_flag.nes": { + "verdict": "PASS", + "reason": "3-irq_flag Passed" + }, + "3-nmi_and_irq.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): NMI BRK 23 00 21 00 21 00 25 21 20 00 25 21 24 20 25 21 24 20 25 21 24 20 25 21 CE1F97FF 3-nmi_and_irq Failed" + }, + "3.A12_clocking.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 4): failed #4 ($F8=04)" + }, + "3.even_odd_frames.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 3): failed #3 ($F8=03)" + }, + "3.Forward_Branch.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "3.Timing.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 5): failed #5 ($F8=05)" + }, + "4-irq_and_dma.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): 0 +0 0 +1 1 +2 1 +3 2 +4 2 +5 4 +6 4 +7 7 +8 7 +9 7 +10 7 +11 8 +12 9 +13 ... 9 +524 9 +525 9 +526 9 +527 64D5E4BC 4-irq_and_dma Failed" + }, + "4-jitter.nes": { + "verdict": "PASS", + "reason": "4-jitter Passed" + }, + "4.Obscure.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "4.Scanline_timing.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "4.vbl_clear_timing.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "4015_cleared.nes": { + "verdict": "PASS", + "reason": "4015_cleared Passed" + }, + "4017_timing.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 3): Delay after effective $4017 write: 0 Frame IRQ flag should be set sooner after power/reset 4017_timing Failed #3" + }, + "4017_written.nes": { + "verdict": "PASS", + "reason": "4017_written Passed" + }, + "5-branch_delays_irq.nes": { + "verdict": "FAIL", + "reason": "blargg test did not finish" + }, + "5-len_timing.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 2): Channel: 0 First length of mode 0 is too soon 5-len_timing Failed #2" + }, + "5.Emulator.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "5.MMC3_rev_A.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 2): failed #2 ($F8=02)" + }, + "5.nmi_suppression.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 3): failed #3 ($F8=03)" + }, + "6-irq_flag_timing.nes": { + "verdict": "PASS", + "reason": "6-irq_flag_timing Passed" + }, + "6.MMC3_rev_B.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 2): failed #2 ($F8=02)" + }, + "6.nmi_disable.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 3): failed #3 ($F8=03)" + }, + "7-dmc_basics.nes": { + "verdict": "PASS", + "reason": "7-dmc_basics Passed" + }, + "7.nmi_timing.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 3): failed #3 ($F8=03)" + }, + "8-dmc_rates.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 3): Rate 0's period is too long 8-dmc_rates Failed #3" + }, + "cpu_dummy_writes_oam.nes": { + "verdict": "PASS", + "reason": "TEST: cpu_dummy_writes_oam This program verifies that the CPU does 2x writes properly. Any read-modify-write opcode should first write the origi- nal value; then the calculated value exactly 1 cycle later. Requirement:…" + }, + "cpu_dummy_writes_ppumem.nes": { + "verdict": "PASS", + "reason": "TEST: cpu_dummy_writes_ppumem This program verifies that the CPU does 2x writes properly. Any read-modify-write opcode should first write the origi- nal value; then the calculated value exactly 1 cycle later. Verifying…" + }, + "irq_flag_cleared.nes": { + "verdict": "PASS", + "reason": "irq_flag_cleared Passed" + }, + "len_ctrs_enabled.nes": { + "verdict": "PASS", + "reason": "len_ctrs_enabled Passed" + }, + "nestest.nes": { + "verdict": "PASS", + "reason": "$02=00 $03=00" + }, + "oam_read.nes": { + "verdict": "PASS", + "reason": "---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------…" + }, + "oam_stress.nes": { + "verdict": "PASS", + "reason": "---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------- ---------------…" + }, + "palette_ram.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "power_up_palette.nes": { + "verdict": "FAIL", + "reason": "blargg2005 test failed (code 2): failed #2 ($F0=02)" + }, + "ppu_open_bus.nes": { + "verdict": "PASS", + "reason": "ppu_open_bus Passed" + }, + "ram_after_reset.nes": { + "verdict": "PASS", + "reason": "ram_after_reset Passed" + }, + "registers.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 3): A X Y P S 00 00 00 34 FD EAE4AAFA Reset should set I flag, subtract 3 from S, nothing more registers Failed #3" + }, + "sprdma_and_dmc_dma.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): T+ Clocks (decimal) 00 399 01 399 02 144 03 144 04 399 05 399 06 399 07 399 08 144 09 144 0A 399 0B 399 0C 144 0D 144 0E 399 0F 399 4EB2EBBC SPRDMA and DMC DMA Failed" + }, + "sprdma_and_dmc_dma_512.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 1): T+ Clocks (decimal) 00 399 01 399 02 144 03 144 04 399 05 399 06 399 07 399 08 144 09 144 0A 399 0B 399 0C 144 0D 144 0E 399 0F 399 4EB2EBBC SPRDMA and DMC DMA Failed" + }, + "sprite_ram.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "test_cpu_exec_space_apu.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 2): TEST: test_cpu_exec_space_apu This program verifies that the CPU can execute code from any possible location that it can address, including I/O space. In this test, it is also verified that…" + }, + "test_cpu_exec_space_ppuio.nes": { + "verdict": "PASS", + "reason": "TEST:test_cpu_exec_space_ppuio This program verifies that the CPU can execute code from any possible location that it can address, including I/O space. In addition, it will be tested that an RTS instruction does a dummy…" + }, + "test_ppu_read_buffer.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 50): TEST:test_ppu_read_buffer ----------------------------- Testing basic PPU memory I/O. Performing tests that combine sprite 0 hit flag, $4014 DMA and the RAM mirroring... Graphical artifacts…" + }, + "vbl_clear_time.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "vram_access.nes": { + "verdict": "PASS", + "reason": "passed" + }, + "works_immediately.nes": { + "verdict": "FAIL", + "reason": "blargg test failed (code 2): At power, writes should work immediately works_immediately Failed #2" + } + } +} diff --git a/test-framework/testroms-ci.txt b/test-framework/testroms-ci.txt new file mode 100644 index 0000000..a1381bf --- /dev/null +++ b/test-framework/testroms-ci.txt @@ -0,0 +1,183 @@ +# Test ROMs for the GitHub CI job -- blargg's NES suites, Bisqwit's +# ppu_read_buffer and nestest. +# +# These files are NOT in the repository (see roms/README.md); fetch them +# first, then run the suite: +# +# make -C test-framework testroms +# +# which is `fetch-testroms.py` followed by run_tests.py with the flags a +# test ROM needs: --test-mode auto (read blargg's $6000 report instead of +# guessing from the screen), --no-input (these ROMs drive themselves) and +# a frame budget big enough for the longest suite. A run stops the moment +# a ROM publishes its verdict, so the budget only bounds the ones that +# never report. +# +# A line may carry per-ROM overrides after " | ". The 2005-era suites need +# test=blargg2005, which reads their result byte at $F8 -- or at $F0 with +# result-addr=0xf0, for the July/September 2005 builds; nestest needs +# test=nestest, because it has no signature and is entered at $C000. + +# apu_reset -- APU: state carried across a soft reset +roms/external/blargg/apu_reset/4015_cleared.nes +roms/external/blargg/apu_reset/4017_timing.nes +roms/external/blargg/apu_reset/4017_written.nes +roms/external/blargg/apu_reset/irq_flag_cleared.nes +roms/external/blargg/apu_reset/len_ctrs_enabled.nes +roms/external/blargg/apu_reset/works_immediately.nes + +# apu_test -- APU: length counter, IRQ flag, frame counter +roms/external/blargg/apu_test/1-len_ctr.nes +roms/external/blargg/apu_test/2-len_table.nes +roms/external/blargg/apu_test/3-irq_flag.nes +roms/external/blargg/apu_test/4-jitter.nes +roms/external/blargg/apu_test/5-len_timing.nes +roms/external/blargg/apu_test/6-irq_flag_timing.nes +roms/external/blargg/apu_test/7-dmc_basics.nes +roms/external/blargg/apu_test/8-dmc_rates.nes + +# cpu_dummy_writes -- CPU: read-modify-write dummy writes +roms/external/blargg/cpu_dummy_writes/cpu_dummy_writes_oam.nes +roms/external/blargg/cpu_dummy_writes/cpu_dummy_writes_ppumem.nes + +# cpu_exec_space -- CPU: executing code from PPU/APU address space +roms/external/blargg/cpu_exec_space/test_cpu_exec_space_apu.nes +roms/external/blargg/cpu_exec_space/test_cpu_exec_space_ppuio.nes + +# cpu_interrupts_v2 -- CPU: NMI/IRQ/BRK behaviour and timing +roms/external/blargg/cpu_interrupts_v2/1-cli_latency.nes +roms/external/blargg/cpu_interrupts_v2/2-nmi_and_brk.nes +roms/external/blargg/cpu_interrupts_v2/3-nmi_and_irq.nes +roms/external/blargg/cpu_interrupts_v2/4-irq_and_dma.nes +roms/external/blargg/cpu_interrupts_v2/5-branch_delays_irq.nes + +# cpu_reset -- CPU: registers and RAM across a soft reset +roms/external/blargg/cpu_reset/ram_after_reset.nes +roms/external/blargg/cpu_reset/registers.nes + +# instr_misc -- CPU: instruction edge cases +roms/external/blargg/instr_misc/01-abs_x_wrap.nes +roms/external/blargg/instr_misc/02-branch_wrap.nes +roms/external/blargg/instr_misc/03-dummy_reads.nes +roms/external/blargg/instr_misc/04-dummy_reads_apu.nes + +# instr_test-v5 -- CPU: every official and unofficial instruction +roms/external/blargg/instr_test-v5/01-basics.nes +roms/external/blargg/instr_test-v5/02-implied.nes +roms/external/blargg/instr_test-v5/03-immediate.nes +roms/external/blargg/instr_test-v5/04-zero_page.nes +roms/external/blargg/instr_test-v5/05-zp_xy.nes +roms/external/blargg/instr_test-v5/06-absolute.nes +roms/external/blargg/instr_test-v5/07-abs_xy.nes +roms/external/blargg/instr_test-v5/08-ind_x.nes +roms/external/blargg/instr_test-v5/09-ind_y.nes +roms/external/blargg/instr_test-v5/10-branches.nes +roms/external/blargg/instr_test-v5/11-stack.nes +roms/external/blargg/instr_test-v5/12-jmp_jsr.nes +roms/external/blargg/instr_test-v5/13-rts.nes +roms/external/blargg/instr_test-v5/14-rti.nes +roms/external/blargg/instr_test-v5/15-brk.nes +roms/external/blargg/instr_test-v5/16-special.nes + +# instr_timing -- CPU: instruction and branch timing +roms/external/blargg/instr_timing/1-instr_timing.nes +roms/external/blargg/instr_timing/2-branch_timing.nes + +# oam_read -- PPU: OAM readback through $2004 +roms/external/blargg/oam_read/oam_read.nes + +# oam_stress -- PPU: heavy OAM read/write traffic +roms/external/blargg/oam_stress/oam_stress.nes + +# ppu_open_bus -- PPU: open-bus decay on the register ports +roms/external/blargg/ppu_open_bus/ppu_open_bus.nes + +# ppu_vbl_nmi -- PPU: VBL flag and NMI timing +roms/external/blargg/ppu_vbl_nmi/01-vbl_basics.nes +roms/external/blargg/ppu_vbl_nmi/02-vbl_set_time.nes +roms/external/blargg/ppu_vbl_nmi/03-vbl_clear_time.nes +roms/external/blargg/ppu_vbl_nmi/04-nmi_control.nes +roms/external/blargg/ppu_vbl_nmi/05-nmi_timing.nes +roms/external/blargg/ppu_vbl_nmi/06-suppression.nes +roms/external/blargg/ppu_vbl_nmi/07-nmi_on_timing.nes +roms/external/blargg/ppu_vbl_nmi/08-nmi_off_timing.nes +roms/external/blargg/ppu_vbl_nmi/09-even_odd_frames.nes +roms/external/blargg/ppu_vbl_nmi/10-even_odd_timing.nes + +# sprite_hit_tests -- PPU: sprite 0 hit, including its timing +roms/external/blargg/sprite_hit_tests/01.basics.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/02.alignment.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/03.corners.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/04.flip.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/05.left_clip.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/06.right_edge.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/07.screen_bottom.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/08.double_height.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/09.timing_basics.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/10.timing_order.nes | test=blargg2005 +roms/external/blargg/sprite_hit_tests/11.edge_timing.nes | test=blargg2005 + +# sprite_overflow_tests -- PPU: the sprite overflow flag in $2002 bit 5 +roms/external/blargg/sprite_overflow_tests/1.Basics.nes | test=blargg2005 +roms/external/blargg/sprite_overflow_tests/2.Details.nes | test=blargg2005 +roms/external/blargg/sprite_overflow_tests/3.Timing.nes | test=blargg2005 +roms/external/blargg/sprite_overflow_tests/4.Obscure.nes | test=blargg2005 +roms/external/blargg/sprite_overflow_tests/5.Emulator.nes | test=blargg2005 + +# apu_2005 -- APU: blargg's 2005 suite (length counter, frame IRQ, reset) +# timing. Its runtime keeps the result at $F0, not $F8. +roms/external/blargg/apu_2005/01.len_ctr.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/02.len_table.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/03.irq_flag.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/04.clock_jitter.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/05.len_timing_mode0.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/06.len_timing_mode1.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/07.irq_flag_timing.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/08.irq_timing.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/09.reset_timing.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/10.len_halt_timing.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/apu_2005/11.len_reload_timing.nes | test=blargg2005 result-addr=0xf0 + +# ppu_tests_2005 -- PPU: palette and sprite RAM, VRAM access, power-up +# palette, VBL clear time. Result at $F0, like apu_2005. +roms/external/blargg/ppu_tests_2005/palette_ram.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/ppu_tests_2005/power_up_palette.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/ppu_tests_2005/sprite_ram.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/ppu_tests_2005/vbl_clear_time.nes | test=blargg2005 result-addr=0xf0 +roms/external/blargg/ppu_tests_2005/vram_access.nes | test=blargg2005 result-addr=0xf0 + +# branch_timing_tests -- CPU: cycles taken by branches, page crossing included +roms/external/blargg/branch_timing_tests/1.Branch_Basics.nes | test=blargg2005 +roms/external/blargg/branch_timing_tests/2.Backward_Branch.nes | test=blargg2005 +roms/external/blargg/branch_timing_tests/3.Forward_Branch.nes | test=blargg2005 + +# vbl_nmi_timing -- PPU: blargg's 2005 VBL flag and NMI timing suite +roms/external/blargg/vbl_nmi_timing/1.frame_basics.nes | test=blargg2005 +roms/external/blargg/vbl_nmi_timing/2.vbl_timing.nes | test=blargg2005 +roms/external/blargg/vbl_nmi_timing/3.even_odd_frames.nes | test=blargg2005 +roms/external/blargg/vbl_nmi_timing/4.vbl_clear_timing.nes | test=blargg2005 +roms/external/blargg/vbl_nmi_timing/5.nmi_suppression.nes | test=blargg2005 +roms/external/blargg/vbl_nmi_timing/6.nmi_disable.nes | test=blargg2005 +roms/external/blargg/vbl_nmi_timing/7.nmi_timing.nes | test=blargg2005 + +# mmc3_irq_tests -- mapper 4: the MMC3 scanline counter and its IRQ. +# 2.Details.nes is renamed: sprite_overflow_tests has one too, and a +# baseline key is a filename. +roms/external/blargg/mmc3_irq_tests/1.Clocking.nes | test=blargg2005 +roms/external/blargg/mmc3_irq_tests/2.MMC3_Details.nes | test=blargg2005 +roms/external/blargg/mmc3_irq_tests/3.A12_clocking.nes | test=blargg2005 +roms/external/blargg/mmc3_irq_tests/4.Scanline_timing.nes | test=blargg2005 +roms/external/blargg/mmc3_irq_tests/5.MMC3_rev_A.nes | test=blargg2005 +roms/external/blargg/mmc3_irq_tests/6.MMC3_rev_B.nes | test=blargg2005 + +# sprdma_and_dmc_dma -- CPU: cycles stolen when sprite and DMC DMA overlap +roms/external/blargg/sprdma_and_dmc_dma/sprdma_and_dmc_dma.nes +roms/external/blargg/sprdma_and_dmc_dma/sprdma_and_dmc_dma_512.nes + +# ppu_read_buffer -- PPU: the $2007 read buffer (Bisqwit's test, mapper 3) +roms/external/bisqwit/ppu_read_buffer/test_ppu_read_buffer.nes + +# nestest -- kevtris's CPU test, run unattended from $C000 with the +# result read out of $02/$03. The whole sweep is over inside one frame; +# the rest of the budget just lets the ROM settle. +roms/external/nestest/nestest.nes | test=nestest frames=120 input=none diff --git a/test-framework/testroms.json b/test-framework/testroms.json new file mode 100644 index 0000000..d0cfcd2 --- /dev/null +++ b/test-framework/testroms.json @@ -0,0 +1,910 @@ +{ + "_comment": "Test ROMs fetched by fetch-testroms.py. They are NOT vendored into this repository: neither blargg's NES suites, Bisqwit's ppu_read_buffer nor nestest carries a licence, so rNES (BSD-3-Clause) links to them instead of redistributing them. See roms/README.md.", + "sources": { + "nes-test-roms": { + "repo": "christopherpow/nes-test-roms", + "url": "https://github.com/christopherpow/nes-test-roms", + "commit": "95d8f621ae55cee0d09b91519a8989ae0e64753b", + "raw": "https://raw.githubusercontent.com/{repo}/{commit}/{path}", + "note": "Community archive of NESdev test ROMs. No repository-wide licence; each suite keeps its author's original readme." + } + }, + "roms": [ + { + "source": "nes-test-roms", + "path": "apu_reset/4015_cleared.nes", + "dest": "blargg/apu_reset/4015_cleared.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "ef83bc2831f0ddb9e563ac5cbcfa21b129f092911ef42adabae7d47b3e990d95" + }, + { + "source": "nes-test-roms", + "path": "apu_reset/4017_timing.nes", + "dest": "blargg/apu_reset/4017_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "0e6072c6dcee98fb73dc7f3af2e48face78be300e8510515ed48dc75d15c1f13" + }, + { + "source": "nes-test-roms", + "path": "apu_reset/4017_written.nes", + "dest": "blargg/apu_reset/4017_written.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "022bd3b45a733179d9a0a9bf0311d09ca81419d7e7434e6f559e42650b39616b" + }, + { + "source": "nes-test-roms", + "path": "apu_reset/irq_flag_cleared.nes", + "dest": "blargg/apu_reset/irq_flag_cleared.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "e2435b213bf21065b7c9c645359500c1c860a7395d12d051232ab14dad1b0bb5" + }, + { + "source": "nes-test-roms", + "path": "apu_reset/len_ctrs_enabled.nes", + "dest": "blargg/apu_reset/len_ctrs_enabled.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "e05546cbfaa1414d9193b0212084b324ea6b13130af5f171a34c9e574f5ac373" + }, + { + "source": "nes-test-roms", + "path": "apu_reset/works_immediately.nes", + "dest": "blargg/apu_reset/works_immediately.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "c750113762ee375319b1bfbf65c457875dbb194649d7e7b3594fba38eb8eefa1" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/1-len_ctr.nes", + "dest": "blargg/apu_test/1-len_ctr.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "aacf86c1d773badd11392e54506a43a06a3dd0b67a4c255909d1daf770a4a1e2" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/2-len_table.nes", + "dest": "blargg/apu_test/2-len_table.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "c002ff1483b4dfb36a6eb004d49739cd58a2dff16e0bac167d5a7c12235caeeb" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/3-irq_flag.nes", + "dest": "blargg/apu_test/3-irq_flag.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "dd888551665937391a2d691b1f96d1858316dbfce4951306146e9e396367f079" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/4-jitter.nes", + "dest": "blargg/apu_test/4-jitter.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "bff573d72d0f134fe307f0bb8b968b8d2ffdb85e8aadad9c152839068d6db32a" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/5-len_timing.nes", + "dest": "blargg/apu_test/5-len_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "4d88f8cc0b21303dc151af4d0f4169d79284634a73082d7ea1ae5cfafedd1e46" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/6-irq_flag_timing.nes", + "dest": "blargg/apu_test/6-irq_flag_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "fc1daff82dd1a49c7c1242392ffbf1c6f44fb70156868582117f2a844cc4dffd" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/7-dmc_basics.nes", + "dest": "blargg/apu_test/7-dmc_basics.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "547324867ee0ba2aa11401001d8d1288530aa4e0ecaaac1667ce79980a388ec1" + }, + { + "source": "nes-test-roms", + "path": "apu_test/rom_singles/8-dmc_rates.nes", + "dest": "blargg/apu_test/8-dmc_rates.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "5d9a79a505b37fa277cacc95a362f7e2a56e59ace7a698213d78432cc06a8867" + }, + { + "source": "nes-test-roms", + "path": "cpu_dummy_writes/cpu_dummy_writes_oam.nes", + "dest": "blargg/cpu_dummy_writes/cpu_dummy_writes_oam.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "7c1d71a38b2e873d0874add8b823ff39b99151bb29f50096d8021787020c566c" + }, + { + "source": "nes-test-roms", + "path": "cpu_dummy_writes/cpu_dummy_writes_ppumem.nes", + "dest": "blargg/cpu_dummy_writes/cpu_dummy_writes_ppumem.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "f59ac329f4872277ccbeff9dd595b901d861af8d53e8a43dcca93bb86752a6b3" + }, + { + "source": "nes-test-roms", + "path": "cpu_exec_space/test_cpu_exec_space_apu.nes", + "dest": "blargg/cpu_exec_space/test_cpu_exec_space_apu.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "9036b3d64b7f4e0ac7d44ed85eafb4b2f93ffd2ca2ddecc19f1e0d2bec3574fe" + }, + { + "source": "nes-test-roms", + "path": "cpu_exec_space/test_cpu_exec_space_ppuio.nes", + "dest": "blargg/cpu_exec_space/test_cpu_exec_space_ppuio.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "9f7f24d420033e032f5da0e1eb721cce1d843e032b31973acddb0255ef479251" + }, + { + "source": "nes-test-roms", + "path": "cpu_interrupts_v2/rom_singles/1-cli_latency.nes", + "dest": "blargg/cpu_interrupts_v2/1-cli_latency.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "e402d36118f77dcbbe8ddca90c15fc76a46bcb30b25cb028c383e4a621de5fc0" + }, + { + "source": "nes-test-roms", + "path": "cpu_interrupts_v2/rom_singles/2-nmi_and_brk.nes", + "dest": "blargg/cpu_interrupts_v2/2-nmi_and_brk.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "6e6bf6205930afcfebdc213c583df53986a688a8b36f8856b805ef4c1853e6eb" + }, + { + "source": "nes-test-roms", + "path": "cpu_interrupts_v2/rom_singles/3-nmi_and_irq.nes", + "dest": "blargg/cpu_interrupts_v2/3-nmi_and_irq.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "3008a9524d174a8aca562ff0361eba81da53e38cf1ebb5125322fe151f14d945" + }, + { + "source": "nes-test-roms", + "path": "cpu_interrupts_v2/rom_singles/4-irq_and_dma.nes", + "dest": "blargg/cpu_interrupts_v2/4-irq_and_dma.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "6d7b4c1947ada64679af56cf0c227286b2408afe1747dfaa4dc7363d57ff87f6" + }, + { + "source": "nes-test-roms", + "path": "cpu_interrupts_v2/rom_singles/5-branch_delays_irq.nes", + "dest": "blargg/cpu_interrupts_v2/5-branch_delays_irq.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "f9e10b4a24d8f3cd3e51fb7457c72858aab96a6467fdbbd806d0661c2d32fdc7" + }, + { + "source": "nes-test-roms", + "path": "cpu_reset/ram_after_reset.nes", + "dest": "blargg/cpu_reset/ram_after_reset.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "f1802a5618aaaa0c4d592caa45b0b13c54082af93fc311bda0c27bceacbc7c7f" + }, + { + "source": "nes-test-roms", + "path": "cpu_reset/registers.nes", + "dest": "blargg/cpu_reset/registers.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "a30f33fb6c9f56012fba38dc85ddc3dccc06bfc0b25fef7711b63f8207279715" + }, + { + "source": "nes-test-roms", + "path": "instr_misc/rom_singles/01-abs_x_wrap.nes", + "dest": "blargg/instr_misc/01-abs_x_wrap.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "892e892b3b5d3526913b5491252f628d987baf44509dd28a7d109efc8a16bacc" + }, + { + "source": "nes-test-roms", + "path": "instr_misc/rom_singles/02-branch_wrap.nes", + "dest": "blargg/instr_misc/02-branch_wrap.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "2e535f572a16ab0e27bf6e076de6aaa2a9885c9ae1b02acda7853810529b24c5" + }, + { + "source": "nes-test-roms", + "path": "instr_misc/rom_singles/03-dummy_reads.nes", + "dest": "blargg/instr_misc/03-dummy_reads.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "9210b8a274c060e0abf9cf7a9f4a140598a8fa01ce852b0a9aef500c9728e822" + }, + { + "source": "nes-test-roms", + "path": "instr_misc/rom_singles/04-dummy_reads_apu.nes", + "dest": "blargg/instr_misc/04-dummy_reads_apu.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "3e4e974a57b82247861576bc855b589fa6d00291ec159f79795c9e3f50c39fef" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/01-basics.nes", + "dest": "blargg/instr_test-v5/01-basics.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "4dd1cdd406bc3f747972e7da314ce8ca89321eb7a836c1ced569ee54ae44a384" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/02-implied.nes", + "dest": "blargg/instr_test-v5/02-implied.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "1c4d4fa130cf6feebc072543a5cd3627ae71063b56b08642bf43e9a6c6f44996" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/03-immediate.nes", + "dest": "blargg/instr_test-v5/03-immediate.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "6f7ad8ff31c762c37deaee0f323df03eb94025cf1f3b0343ebe6fe567da0e943" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/04-zero_page.nes", + "dest": "blargg/instr_test-v5/04-zero_page.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "7a8feada4bb4460250c8f05401e5d728878bbe71956756d0b11d488e57eb12fd" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/05-zp_xy.nes", + "dest": "blargg/instr_test-v5/05-zp_xy.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "767f422dc4e651e331456b207f7c6d60d19329fde0c0827e83591dbd91ae5e23" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/06-absolute.nes", + "dest": "blargg/instr_test-v5/06-absolute.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "98df36dc4fcc4f37d9eb0539c71283020776b1e5dc6a6ce58671739a8d6534af" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/07-abs_xy.nes", + "dest": "blargg/instr_test-v5/07-abs_xy.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "9ff58d77d8d384cc918fcd3ed877898c5e7330cd475ed2dafb11cbe80ff32eff" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/08-ind_x.nes", + "dest": "blargg/instr_test-v5/08-ind_x.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "2ec6f5d4a8caee5d8295cebe563f203c26ea9bc05f1dbc967feb88f5dc4f261f" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/09-ind_y.nes", + "dest": "blargg/instr_test-v5/09-ind_y.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "0fbc8b228d5daa83a4a083bf87ae3a61b5247ebdd91a6b91c8cf8c42784804ac" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/10-branches.nes", + "dest": "blargg/instr_test-v5/10-branches.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "63ab768e88931db6f7dfcfafe43d5e29ebc3dcb80da8fc7fcda8c930f34aef54" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/11-stack.nes", + "dest": "blargg/instr_test-v5/11-stack.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "c534191fe3ea4c8940944fda98dd58eb42710268d453f97e8e2c4ae7f15f9cdb" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/12-jmp_jsr.nes", + "dest": "blargg/instr_test-v5/12-jmp_jsr.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "f5b4652690fc04e6b573a2b3b54a29407ad0615d3c264e7cb618b6694b50de55" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/13-rts.nes", + "dest": "blargg/instr_test-v5/13-rts.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "b711d25bc55585c252046a1304a0bc64c13cacce7c96a1bac5c8e91f9fc2597f" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/14-rti.nes", + "dest": "blargg/instr_test-v5/14-rti.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "f084b00605be1840946b53935032581e68abe1bb24479942751cfe46ddfcb280" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/15-brk.nes", + "dest": "blargg/instr_test-v5/15-brk.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "da7ae9a191c4483b540771e15b1f6f18df68f1d1ecd717b59ea8b1ee3596ec3e" + }, + { + "source": "nes-test-roms", + "path": "instr_test-v5/rom_singles/16-special.nes", + "dest": "blargg/instr_test-v5/16-special.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "7d03410b61784e49920901e84b00a4f31a19078391f20005c6fac9036d2190f7" + }, + { + "source": "nes-test-roms", + "path": "instr_timing/rom_singles/1-instr_timing.nes", + "dest": "blargg/instr_timing/1-instr_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "e260068839fe3d0402376e97e4ee15f5790ee77c701fd0700bba057527910222" + }, + { + "source": "nes-test-roms", + "path": "instr_timing/rom_singles/2-branch_timing.nes", + "dest": "blargg/instr_timing/2-branch_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "0afaa393f375844ab98834c1ecba7fa6d8c44880c8b6e738936d0f04a84c8538" + }, + { + "source": "nes-test-roms", + "path": "oam_read/oam_read.nes", + "dest": "blargg/oam_read/oam_read.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "f298973dabeb61ca35007445f7a615f77e87703c958c870986af83b1aabde926" + }, + { + "source": "nes-test-roms", + "path": "oam_stress/oam_stress.nes", + "dest": "blargg/oam_stress/oam_stress.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "95882d72a7acabe928fd277e3b3e0372f21ef3d41e36d7d8fb17fc017a356f70" + }, + { + "source": "nes-test-roms", + "path": "ppu_open_bus/ppu_open_bus.nes", + "dest": "blargg/ppu_open_bus/ppu_open_bus.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "d4208a3ff6340532dd0fced7f9d408d5b6585853a0ddc9c1f64ee1722ef08e67" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/01-vbl_basics.nes", + "dest": "blargg/ppu_vbl_nmi/01-vbl_basics.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "06aea5af4edab4e3141c939cd5ac9936f8758203b25dcaf84ae1a09db49e024a" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/02-vbl_set_time.nes", + "dest": "blargg/ppu_vbl_nmi/02-vbl_set_time.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "dd98856130078844e3aa4bd95a9be8ab501ea84c089f1d8ad49a1b20af4b3a80" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/03-vbl_clear_time.nes", + "dest": "blargg/ppu_vbl_nmi/03-vbl_clear_time.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "787fdaa4dd6c5b6df5f4308fb6d55b57e2c2f69bd5ecdf8ad5c69735db4fcc72" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/04-nmi_control.nes", + "dest": "blargg/ppu_vbl_nmi/04-nmi_control.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "84722c75b896c47c8642f83220230fe14f0a31e55e26ecb83c400e6a26d91b32" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/05-nmi_timing.nes", + "dest": "blargg/ppu_vbl_nmi/05-nmi_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "72e515d689d7404ae5779b8c9c4c7b3563a755a94bd44864516f1b03df044482" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/06-suppression.nes", + "dest": "blargg/ppu_vbl_nmi/06-suppression.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "811dd5997bbf48c2e5687ab06845f17ea76b2be472786596c334137582cc72aa" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/07-nmi_on_timing.nes", + "dest": "blargg/ppu_vbl_nmi/07-nmi_on_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "1ed154363660b5775b112ae63ce9bb4e400ebde2afef4d0ac12fc433efda3702" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/08-nmi_off_timing.nes", + "dest": "blargg/ppu_vbl_nmi/08-nmi_off_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "1d2a4093091c8e58a7f99d6a3531bbc6346b52cfc59bcb17ca04c1f2376cf2fc" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/09-even_odd_frames.nes", + "dest": "blargg/ppu_vbl_nmi/09-even_odd_frames.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "1ac04283021ddd9294cc74ee709c55e20a350dc4815c15a8a93b3654837e858d" + }, + { + "source": "nes-test-roms", + "path": "ppu_vbl_nmi/rom_singles/10-even_odd_timing.nes", + "dest": "blargg/ppu_vbl_nmi/10-even_odd_timing.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "7217d2d172ce11ad45c4da40c2f22201cf0eb758bc2cd8dd39d2cf0a7d4ca83e" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/01.basics.nes", + "dest": "blargg/sprite_hit_tests/01.basics.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "51819e8e502bd88fe3b7244198a074dbeef2e848f66c587be04b04f1f0d4bb52" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/02.alignment.nes", + "dest": "blargg/sprite_hit_tests/02.alignment.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "125bbb3ce1e67370f1f4559c2ad3221e52a3e98880b9789400292b5f3a8b39e6" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/03.corners.nes", + "dest": "blargg/sprite_hit_tests/03.corners.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "9dd57776bc6267fe6183c5521d67cbe3fccc6662ae545eb2c419949bf39644d3" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/04.flip.nes", + "dest": "blargg/sprite_hit_tests/04.flip.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "5f7142bddb51b7577f93fa22f9f668efebbeea00346d7255089e1863acb9d46a" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/05.left_clip.nes", + "dest": "blargg/sprite_hit_tests/05.left_clip.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "69b329658c17b953f149c2f0de77eb272089df22c815bd2fd3d6f43206791c13" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/06.right_edge.nes", + "dest": "blargg/sprite_hit_tests/06.right_edge.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "8e6653fcb869e06873e29e5e4423122ea72ba0bf38f3ba9e39f471420db759a4" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/07.screen_bottom.nes", + "dest": "blargg/sprite_hit_tests/07.screen_bottom.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "05849956f80267838c5b6556310266b794078a4300841cbb36339fd141905a0b" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/08.double_height.nes", + "dest": "blargg/sprite_hit_tests/08.double_height.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "127fd966b6b32d6d88a53c5f59d7e938827783c9ad056091f119be1c4ab21c71" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/09.timing_basics.nes", + "dest": "blargg/sprite_hit_tests/09.timing_basics.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "311698c717e50150edd0b5fd0016c41de686463205c20efb5630d6adb90859fd" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/10.timing_order.nes", + "dest": "blargg/sprite_hit_tests/10.timing_order.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "0f36bc07bfe51c416e3cc1a5231053572aa6b15aa60e6d2fd0568be49b6dc2e9" + }, + { + "source": "nes-test-roms", + "path": "sprite_hit_tests_2005.10.05/11.edge_timing.nes", + "dest": "blargg/sprite_hit_tests/11.edge_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "5a7c121f6e76617be88a0a7035c0e402293be5c685c95b97190a8d70835736ab" + }, + { + "source": "nes-test-roms", + "path": "sprite_overflow_tests/1.Basics.nes", + "dest": "blargg/sprite_overflow_tests/1.Basics.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "1a6782f63ccb3a3dd1aa6a24272036c9c3aa232c2d1ff0b21e872741a3ee4fe2" + }, + { + "source": "nes-test-roms", + "path": "sprite_overflow_tests/2.Details.nes", + "dest": "blargg/sprite_overflow_tests/2.Details.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "6405a7ff1042fe7a50d9bfe521e43460a3251799425bd3e1863b29b67e7cd587" + }, + { + "source": "nes-test-roms", + "path": "sprite_overflow_tests/3.Timing.nes", + "dest": "blargg/sprite_overflow_tests/3.Timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "2252ec8fc35932b408f409ef9b6863edf084aa871fc10ad180b0ac4c2468ef8c" + }, + { + "source": "nes-test-roms", + "path": "sprite_overflow_tests/4.Obscure.nes", + "dest": "blargg/sprite_overflow_tests/4.Obscure.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "aebf2199344321465ae0d8dcd81f6c528c7f661f31f1814711365b4e573a8263" + }, + { + "source": "nes-test-roms", + "path": "sprite_overflow_tests/5.Emulator.nes", + "dest": "blargg/sprite_overflow_tests/5.Emulator.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "cf994454219696de82794f0b84f2bd63458444d12a1171c85bba8697ab94acb4" + }, + { + "source": "nes-test-roms", + "path": "other/nestest.nes", + "dest": "nestest/nestest.nes", + "author": "kevtris (Kevin Horton)", + "size": 24592, + "sha256": "f67d55fd6b3cf0bad1cc85f1df0d739c65b53e79cecb7fea8f77ec0eadab0004" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/01.len_ctr.nes", + "dest": "blargg/apu_2005/01.len_ctr.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "e1e3a29ab5369ab84a6f5f2f426c64bde86b9a1d26a906739d43fbf624bb8829" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/02.len_table.nes", + "dest": "blargg/apu_2005/02.len_table.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "63cc6a57fae3da5e30df9520d02b723c5b93789e8a6eef6793f876345b245b51" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/03.irq_flag.nes", + "dest": "blargg/apu_2005/03.irq_flag.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "6f71c7e3de4b6c00da92c20a86c1c2095196a55dc201ee3286004ecf33a08c2f" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/04.clock_jitter.nes", + "dest": "blargg/apu_2005/04.clock_jitter.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "46fa69b26fe8c24dc1d0b5908f90ab0141972eeb607bd563d28f53d6f4543fe6" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/05.len_timing_mode0.nes", + "dest": "blargg/apu_2005/05.len_timing_mode0.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "606802d6849ccfcf74e907a8512c03a50d443752d1f616e62a242a1fa7eca0ff" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/06.len_timing_mode1.nes", + "dest": "blargg/apu_2005/06.len_timing_mode1.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "0f34e26d56ad235d8d6d63565ed4728fbd0b9b8590a4fd4048eaf64283b429d4" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/07.irq_flag_timing.nes", + "dest": "blargg/apu_2005/07.irq_flag_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "851c9698941d51da34b4bfbc9644aa08ee41b39c6c814cc9b8412c204ea68085" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/08.irq_timing.nes", + "dest": "blargg/apu_2005/08.irq_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "0a20a2b9ca9a8e78d65b500b161294c889399f0ff048edd104b07b15255946ca" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/09.reset_timing.nes", + "dest": "blargg/apu_2005/09.reset_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "bb04f8328a51abb2d17e6e5362b8375f3cbfbf0641733d068b22f23e7dc588e6" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/10.len_halt_timing.nes", + "dest": "blargg/apu_2005/10.len_halt_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "cbdaa9a5cf9c19ba2360d3349a47922eec25a3e610374d963c422f2b67c57ac9" + }, + { + "source": "nes-test-roms", + "path": "blargg_apu_2005.07.30/11.len_reload_timing.nes", + "dest": "blargg/apu_2005/11.len_reload_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "40e633285a4a8710780bfd80d346dee62406f4be161eb75615f469cd9e84e132" + }, + { + "source": "nes-test-roms", + "path": "blargg_ppu_tests_2005.09.15b/palette_ram.nes", + "dest": "blargg/ppu_tests_2005/palette_ram.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "11c1de16c67795b247169819a34b9d4dd46165478d6de19d7cd6e7311e906a80" + }, + { + "source": "nes-test-roms", + "path": "blargg_ppu_tests_2005.09.15b/power_up_palette.nes", + "dest": "blargg/ppu_tests_2005/power_up_palette.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "4a02958db36d1e1affb6e6b80d556e4a6840847cc42027d2c875fbe917db9f49" + }, + { + "source": "nes-test-roms", + "path": "blargg_ppu_tests_2005.09.15b/sprite_ram.nes", + "dest": "blargg/ppu_tests_2005/sprite_ram.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "81275795ea64dfe22f05e35ef3316e6cf84eba40fc9538fc637a08bf263a3110" + }, + { + "source": "nes-test-roms", + "path": "blargg_ppu_tests_2005.09.15b/vbl_clear_time.nes", + "dest": "blargg/ppu_tests_2005/vbl_clear_time.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "1a1d3ae8062ce0d4f1f3eafa35bf3c3e5df94fe198e7630a09678a60b03b73a0" + }, + { + "source": "nes-test-roms", + "path": "blargg_ppu_tests_2005.09.15b/vram_access.nes", + "dest": "blargg/ppu_tests_2005/vram_access.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "a0cde2a3c98d9f62afcba62d4e8f4f4228574ff96ce96fbfdc777e1393c16ee6" + }, + { + "source": "nes-test-roms", + "path": "branch_timing_tests/1.Branch_Basics.nes", + "dest": "blargg/branch_timing_tests/1.Branch_Basics.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "7b69e3044eaeb86317147a900d1f4a467b666f59d375ec1ba6658233f23786cd" + }, + { + "source": "nes-test-roms", + "path": "branch_timing_tests/2.Backward_Branch.nes", + "dest": "blargg/branch_timing_tests/2.Backward_Branch.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "f7966e9b86b04b4adb987439a442e926e9cfe6bb71436dd5dd56f41f9eb029a4" + }, + { + "source": "nes-test-roms", + "path": "branch_timing_tests/3.Forward_Branch.nes", + "dest": "blargg/branch_timing_tests/3.Forward_Branch.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "d0fbc6b1899bc948172c45f37a981eb0dade212d7e807cc56efedae93d6f9c9b" + }, + { + "source": "nes-test-roms", + "path": "vbl_nmi_timing/1.frame_basics.nes", + "dest": "blargg/vbl_nmi_timing/1.frame_basics.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "719bf4b4ea9022aad4216b159f59846ea57f98c8bddc5069b70286f5b810e998" + }, + { + "source": "nes-test-roms", + "path": "vbl_nmi_timing/2.vbl_timing.nes", + "dest": "blargg/vbl_nmi_timing/2.vbl_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "39231f002784b62e5d0ab67735855198b6b62c6c6ef06b0549333cd818879fb2" + }, + { + "source": "nes-test-roms", + "path": "vbl_nmi_timing/3.even_odd_frames.nes", + "dest": "blargg/vbl_nmi_timing/3.even_odd_frames.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "268d5e305ab73a29ff524906f92db7da1944763ea3de464e14a44c5b4199049e" + }, + { + "source": "nes-test-roms", + "path": "vbl_nmi_timing/4.vbl_clear_timing.nes", + "dest": "blargg/vbl_nmi_timing/4.vbl_clear_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "8552a65de49b677cdf7fd419f5ac51f96766786e1e1442f6f9d62c87f4c15c2c" + }, + { + "source": "nes-test-roms", + "path": "vbl_nmi_timing/5.nmi_suppression.nes", + "dest": "blargg/vbl_nmi_timing/5.nmi_suppression.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "059c8df6a9b41a95e827608f1c0edb496b763e0c2b861f22614f5f228f1e10aa" + }, + { + "source": "nes-test-roms", + "path": "vbl_nmi_timing/6.nmi_disable.nes", + "dest": "blargg/vbl_nmi_timing/6.nmi_disable.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "4573dcff3c57351b0e6ce76d86334586ba3d1cd6d4f9ec014887e94912f543cf" + }, + { + "source": "nes-test-roms", + "path": "vbl_nmi_timing/7.nmi_timing.nes", + "dest": "blargg/vbl_nmi_timing/7.nmi_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "26a2475828ef9ebd6178129f530a8aaa6f9f1ddd493616f1524d8323d6e56a4c" + }, + { + "source": "nes-test-roms", + "path": "mmc3_irq_tests/1.Clocking.nes", + "dest": "blargg/mmc3_irq_tests/1.Clocking.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "699d0644bd2b6ff4c9ba598c9609f4a3da536594b6363585b2caf82cf337ac88" + }, + { + "source": "nes-test-roms", + "path": "mmc3_irq_tests/2.Details.nes", + "dest": "blargg/mmc3_irq_tests/2.MMC3_Details.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "0af95238b69806c072c28aed0fa8ad812157dfee928a6c9cea8d5420268baade" + }, + { + "source": "nes-test-roms", + "path": "mmc3_irq_tests/3.A12_clocking.nes", + "dest": "blargg/mmc3_irq_tests/3.A12_clocking.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "3b936e1079f12bdc5e55aa82def017ddc76fd79e3879ff0478d41ca718302e7c" + }, + { + "source": "nes-test-roms", + "path": "mmc3_irq_tests/4.Scanline_timing.nes", + "dest": "blargg/mmc3_irq_tests/4.Scanline_timing.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "3369e8f73a96ec97918c6c9440804a369a19256c57df91e62881555f21528894" + }, + { + "source": "nes-test-roms", + "path": "mmc3_irq_tests/5.MMC3_rev_A.nes", + "dest": "blargg/mmc3_irq_tests/5.MMC3_rev_A.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "6b662c2d08ee4094d89b6d1ddde330e47f81929fb642dc217b6e4c33f8926944" + }, + { + "source": "nes-test-roms", + "path": "mmc3_irq_tests/6.MMC3_rev_B.nes", + "dest": "blargg/mmc3_irq_tests/6.MMC3_rev_B.nes", + "author": "blargg (Shay Green)", + "size": 16400, + "sha256": "d8a2af42cdafe8046b36109e6f6ff71ca0d7f5d62c7d5953e0aa1d1828a86088" + }, + { + "source": "nes-test-roms", + "path": "sprdma_and_dmc_dma/sprdma_and_dmc_dma.nes", + "dest": "blargg/sprdma_and_dmc_dma/sprdma_and_dmc_dma.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "db3199bc1b0bdc07a316b3ab999d8fd8bb361456d2154e364c132cb06a26a10f" + }, + { + "source": "nes-test-roms", + "path": "sprdma_and_dmc_dma/sprdma_and_dmc_dma_512.nes", + "dest": "blargg/sprdma_and_dmc_dma/sprdma_and_dmc_dma_512.nes", + "author": "blargg (Shay Green)", + "size": 40976, + "sha256": "3789f5134b0561b4344e3f4ce08b4d2a416f67435e083917a80d87fdb9d3583c" + }, + { + "source": "nes-test-roms", + "path": "ppu_read_buffer/test_ppu_read_buffer.nes", + "dest": "bisqwit/ppu_read_buffer/test_ppu_read_buffer.nes", + "author": "Bisqwit (Joel Yliluoma)", + "size": 49168, + "sha256": "230a52fc557c098eba163801d1b6bbf9f57fe8c5ff79a3f968c804dedb1290ba" + } + ] +} diff --git a/test-framework/tests/test_testroms.py b/test-framework/tests/test_testroms.py new file mode 100644 index 0000000..11e9de3 --- /dev/null +++ b/test-framework/tests/test_testroms.py @@ -0,0 +1,246 @@ +"""Unit tests for the test-ROM path: protocol verdicts and per-ROM options.""" +import json +import os +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from rnestest.__main__ import (parse_rom_options, # noqa: E402 + read_rom_list) +from rnestest.analyze import FAIL, PASS, REASON_MAX, analyze # noqa: E402 +from rnestest.runner import RunConfig, RunResult, run_rom # noqa: E402 + + +def testrom_result(**report): + """A RunResult whose harness report carries a test-ROM verdict.""" + base = {"status": "ok", "mapper": 1, "cpu_jammed": False, + "rendering_ever_enabled": True, "samples": []} + base.update(report) + return RunResult(rom="t.nes", key="t.nes", ran=True, returncode=0, + report=base, mapper=base["mapper"]) + + +class TestProtocolVerdict(unittest.TestCase): + def test_blargg_code_zero_passes(self): + v = analyze(testrom_result(test_protocol="blargg", test_done=True, + test_code=0, + test_text="All 16 tests passed")) + self.assertEqual(v.verdict, PASS) + self.assertEqual(v.category, "test-pass") + self.assertEqual(v.reason, "All 16 tests passed") + + def test_blargg_nonzero_code_fails_with_the_roms_own_words(self): + v = analyze(testrom_result(test_protocol="blargg", test_done=True, + test_code=3, + test_text="01-vbl_basics Failed #2")) + self.assertEqual(v.verdict, FAIL) + self.assertEqual(v.category, "test-fail") + self.assertIn("code 3", v.reason) + self.assertIn("01-vbl_basics Failed #2", v.reason) + + def test_unfinished_test_fails(self): + # The frame budget ran out before the ROM published anything: a + # partial result is not a verdict. + v = analyze(testrom_result(test_protocol="blargg", test_done=False)) + self.assertEqual(v.verdict, FAIL) + self.assertEqual(v.category, "test-incomplete") + + def test_long_report_is_clipped(self): + v = analyze(testrom_result(test_protocol="blargg", test_done=True, + test_code=1, test_text="x " * 400)) + self.assertLessEqual(len(v.reason), REASON_MAX) + + def test_jam_outranks_the_roms_own_verdict(self): + # nestest leaves $02/$03 at zero for every test it never reached, so + # a jammed CPU must not be read as "everything passed". + v = analyze(testrom_result(test_protocol="nestest", test_done=False, + cpu_jammed=True, + test_text="$02=00 $03=00")) + self.assertEqual(v.verdict, FAIL) + self.assertEqual(v.category, "cpu-jam") + + def test_finished_test_is_not_judged_by_the_heuristics(self): + # A finished test ROM sits in a `jmp *` on an unchanging screen -- + # a tight loop and a frozen screen both, yet it passed. + v = analyze(testrom_result( + test_protocol="blargg", test_done=True, test_code=0, + samples=[{"frame": f, "frame_hash": "same", "distinct_colors": 2, + "dominant_color": "000000", "dominant_frac": 0.97, + "distinct_pcs": 1, "instructions": 10, "pc": "e755", + "render_enabled": True, "nmi_enabled": True} + for f in (60, 90, 120)])) + self.assertEqual(v.verdict, PASS) + + def test_no_protocol_falls_back_to_the_heuristics(self): + # --test auto on a ROM with no blargg signature reports nothing, and + # the screen/CPU analysis has to judge it as before. + v = analyze(testrom_result(samples=[ + {"frame": f, "frame_hash": f"h{f}", "distinct_colors": 1, + "dominant_color": "000000", "dominant_frac": 1.0, + "distinct_pcs": 200, "instructions": 10, "pc": "c000", + "render_enabled": True, "nmi_enabled": True} + for f in (60, 90, 120)])) + self.assertEqual(v.verdict, FAIL) + self.assertEqual(v.category, "blank-screen") + + +class TestTestModeFlag(unittest.TestCase): + def capture(self, **kw): + completed = subprocess.CompletedProcess(args=[], returncode=0, + stdout="{}", stderr="") + with mock.patch("subprocess.run", return_value=completed) as m: + run_rom("t.nes", "t.nes", RunConfig(harness="./h", **kw)) + return m.call_args[0][0] + + def test_off_is_not_passed(self): + self.assertNotIn("--test", self.capture()) + + def test_mode_is_passed(self): + cmd = self.capture(test_mode="nestest") + self.assertEqual(cmd[cmd.index("--test") + 1], "nestest") + + def test_result_addr_is_passed(self): + cmd = self.capture(test_mode="blargg2005", result_addr=0xf0) + self.assertEqual(cmd[cmd.index("--result-addr") + 1], "0xf0") + + def test_result_addr_defaults_to_the_harness(self): + self.assertNotIn("--result-addr", + self.capture(test_mode="blargg2005")) + + +class TestRomOptions(unittest.TestCase): + def test_options_map_to_runconfig_fields(self): + self.assertEqual( + parse_rom_options("test=nestest frames=120 timeout=45"), + {"test_mode": "nestest", "frames": 120, "timeout": 45.0}) + + def test_input_none_disables_mashing(self): + self.assertEqual(parse_rom_options("input=none"), + {"inject_input": False}) + + def test_input_list_enables_mashing(self): + self.assertEqual(parse_rom_options("input=start,select"), + {"inject_input": True, "buttons": "start,select"}) + + def test_ram_fill_accepts_hex(self): + self.assertEqual(parse_rom_options("ram-fill=0xff"), + {"ram_fill": 0xff}) + + def test_result_addr_accepts_hex(self): + self.assertEqual(parse_rom_options("result-addr=0xf0"), + {"result_addr": 0xf0}) + + def test_unknown_option_is_an_error(self): + # Silently ignoring it would run the ROM under flags nobody asked + # for and record the resulting verdict as the baseline. + with self.assertRaises(ValueError): + parse_rom_options("nosuchoption=1") + + def test_bad_test_mode_is_an_error(self): + with self.assertRaises(ValueError): + parse_rom_options("test=sideways") + + def test_option_without_value_is_an_error(self): + with self.assertRaises(ValueError): + parse_rom_options("frames") + + +class TestReadRomList(unittest.TestCase): + def read(self, text): + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "list.txt") + with open(path, "w", encoding="utf-8") as fh: + fh.write(text) + return read_rom_list(path, "/roms") + + def test_plain_path_has_no_overrides(self): + entries = self.read("sub/game.nes\n") + self.assertEqual(entries, [("/roms/sub/game.nes", "game.nes", {})]) + + def test_comments_and_blanks_ignored(self): + self.assertEqual(self.read("# note\n\n \nx.nes\n"), + [("/roms/x.nes", "x.nes", {})]) + + def test_options_after_the_separator(self): + path, key, opts = self.read("a/nestest.nes | test=nestest\n")[0] + self.assertEqual(path, "/roms/a/nestest.nes") + self.assertEqual(key, "nestest.nes") + self.assertEqual(opts, {"test_mode": "nestest"}) + + def test_spaces_in_rom_names_survive(self): + # ROM filenames are full of spaces, which is why the separator is + # " | " rather than whitespace. + entries = self.read("Some Game (USA).nes\n") + self.assertEqual(entries[0][1], "Some Game (USA).nes") + + def test_spaces_in_rom_names_survive_alongside_options(self): + path, key, opts = self.read("Some Game (USA).nes | frames=90\n")[0] + self.assertEqual(key, "Some Game (USA).nes") + self.assertEqual(opts, {"frames": 90}) + + def test_two_roms_with_one_key_are_an_error(self): + # Keys are basenames, so the same filename in two suites would share + # one baseline entry. + with self.assertRaises(ValueError) as ctx: + self.read("a/2.Details.nes\nb/2.Details.nes\n") + self.assertIn(":2:", str(ctx.exception)) + + def test_bad_option_names_the_line(self): + with self.assertRaises(ValueError) as ctx: + self.read("a.nes\nb.nes | bogus=1\n") + self.assertIn(":2:", str(ctx.exception)) + + +class TestManifest(unittest.TestCase): + """The pinned test-ROM manifest has to stay consistent with the list.""" + + @classmethod + def setUpClass(cls): + here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + cls.here = here + with open(os.path.join(here, "testroms.json"), encoding="utf-8") as fh: + cls.manifest = json.load(fh) + + def test_every_rom_names_a_known_source(self): + sources = self.manifest["sources"] + for rom in self.manifest["roms"]: + self.assertIn(rom["source"], sources) + + def test_hashes_are_sha256(self): + for rom in self.manifest["roms"]: + self.assertRegex(rom["sha256"], r"^[0-9a-f]{64}$") + + def test_sources_pin_a_full_commit(self): + for src in self.manifest["sources"].values(): + self.assertRegex(src["commit"], r"^[0-9a-f]{40}$") + + def test_destinations_are_unique_and_relative(self): + dests = [rom["dest"] for rom in self.manifest["roms"]] + self.assertEqual(len(dests), len(set(dests))) + for dest in dests: + self.assertFalse(os.path.isabs(dest)) + self.assertNotIn("..", dest.split("/")) + + def test_ci_list_and_manifest_agree(self): + listed = { + os.path.relpath(path, os.path.join(self.here, "roms", "external")) + for path, _key, _opts in read_rom_list( + os.path.join(self.here, "testroms-ci.txt"), + self.here) + } + self.assertEqual(listed, {r["dest"] for r in self.manifest["roms"]}) + + def test_baseline_covers_every_listed_rom(self): + with open(os.path.join(self.here, "testroms-baseline.json"), + encoding="utf-8") as fh: + baseline = json.load(fh)["roms"] + keys = {os.path.basename(r["dest"]) for r in self.manifest["roms"]} + self.assertEqual(set(baseline), keys) + + +if __name__ == "__main__": + unittest.main()