Check every public function against cross-platform reference values

.github/workflows/ci-cross.yml

@@ -44,10 +44,11 @@ jobs:
           apt-get update -q && apt-get install -yq jq curl
           Scripts/install-swift-sdk.sh wasm
 
-      - name: Build (including tests)
+      # Runs the tests (including the cross-platform reference values) under WasmKit.
+      - name: Run Tests
         run: |
           SDK=$(swift sdk list | grep -E '_wasm$' | head -n 1)
-          swift build --swift-sdk "$SDK" --build-tests -Xswiftc -warnings-as-errors
+          swift test --swift-sdk "$SDK" -Xswiftc -warnings-as-errors
 
   android:
     name: Android

.github/workflows/ci-macos.yml

@@ -41,5 +41,8 @@ jobs:
       - name: Check Differential Test Coverage
         run: python3 Scripts/check-simd-differential-coverage.py
 
+      - name: Check Reference Value Coverage
+        run: python3 Scripts/check-reference-coverage.py
+
       - name: Check @inlinable Coverage
         run: Scripts/check-inlinable.sh

Makefile

@@ -11,7 +11,7 @@ DOCS_VERSION_PATH ?= main
 HOSTING_BASE_PATH ?= $(REPO_NAME)/$(DOCS_VERSION_PATH)
 
 # Targets
-.PHONY: setup lint lint-fix check-inlinable check-simd-coverage test-portable test test-coverage clean pre-commit docs docs-preview docs-generate docs-coverage profile-stats benchmark-commits benchmark benchmark-thresholds-check benchmark-thresholds-update
+.PHONY: setup lint lint-fix check-inlinable check-simd-coverage check-reference-coverage reference-values test-portable test test-coverage clean pre-commit docs docs-preview docs-generate docs-coverage profile-stats benchmark-commits benchmark benchmark-thresholds-check benchmark-thresholds-update
 
 setup:
 	@echo "Detected Package Swift Version: $(PACKAGE_SWIFT_VERSION)"
@@ -55,6 +55,14 @@ check-inlinable:
 check-simd-coverage:
 	python3 Scripts/check-simd-differential-coverage.py
 
+check-reference-coverage:
+	python3 Scripts/check-reference-coverage.py
+
+# Regenerates the cross-platform reference values. Run on macOS with the default traits.
+reference-values:
+	FRB_MATH_REFERENCE_OUTPUT=$(CURDIR)/Tests/FirebladeMathTests/ReferenceValues/ReferenceData.swift \
+		swift test $(SWIFT_FLAGS) -c release --filter ReferenceValueTests
+
 # Portable implementations compared against Apple's simd (Apple platforms only).
 test-portable:
 	swift test $(SWIFT_FLAGS) --disable-default-traits --parallel -Xswiftc -warnings-as-errors

Scripts/check-reference-coverage.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+"""Checks that every public FirebladeMath function has cross-platform reference values.
+
+Covered declarations:
+- every public top-level function (except deprecated ones),
+- every public static function and initializer declared in extensions of the concrete
+  matrix and quaternion types (Mat2x2f ... Mat4x4d, Quat4f, Quat4d).
+
+Each must be registered in Tests/FirebladeMathTests/ReferenceValues/ReferenceFunctions.swift,
+keyed `name(ParameterTypes)` or `Type.name(ParameterTypes)`; generic functions are registered
+once per instantiation with a ` [Float]` / ` [Double]` suffix. Fails on missing and on
+stale registrations.
+"""
+
+import importlib.util
+import pathlib
+import re
+import sys
+
+ROOT = pathlib.Path(__file__).resolve().parent.parent
+REGISTRY = ROOT / "Tests" / "FirebladeMathTests" / "ReferenceValues" / "ReferenceFunctions.swift"
+
+# Reuse the declaration parsing of the differential coverage check (without leaving
+# __pycache__ behind).
+sys.dont_write_bytecode = True
+_spec = importlib.util.spec_from_file_location("simd_coverage", ROOT / "Scripts" / "check-simd-differential-coverage.py")
+simd_coverage = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(simd_coverage)
+
+TOP_LEVEL = re.compile(
+    r"^(?P<attributes>(?:[ \t]*@[^\n]*\n)*)public\s+func\s+(?P<name>\w+)\s*(?:<[^>(]*>)?\s*\((?P<parameters>[^)]*)\)",
+    re.MULTILINE,
+)
+CONCRETE_EXTENSION = re.compile(r"^extension\s+(Mat[234]x[234][fd]|Quat4[fd])\b")
+MEMBER = re.compile(r"^\s+(?:@\w+\s+)*public\s+(?:static\s+func\s+(?P<name>\w+)|init)\s*\((?P<parameters>[^)]*)\)")
+REGISTERED = re.compile(r'r\.add\("(?P<key>[^"]+)"')
+GENERIC_SUFFIX = re.compile(r" \[(?:Float|Double)\]$")
+
+
+def member_key(type_name, name, parameters):
+    # Default values (`= .zero`) are not part of the key.
+    parameters = re.sub(r"\s*=\s*[^,)]+", "", parameters)
+    return f"{type_name}.{simd_coverage.key(name, parameters)}"
+
+
+def public_functions():
+    keys = set()
+    for path in sorted(simd_coverage.SOURCES.rglob("*.swift")):
+        text = path.read_text()
+        for match in TOP_LEVEL.finditer(text):
+            if "deprecated" not in match.group("attributes"):
+                keys.add(simd_coverage.key(match.group("name"), match.group("parameters")))
+        extension = None
+        for line in text.split("\n"):
+            if found := CONCRETE_EXTENSION.match(line):
+                extension = found.group(1)
+                continue
+            if re.match(r"^(extension|struct|public|func)\b", line):
+                extension = None
+            if extension and (member := MEMBER.match(line)):
+                keys.add(member_key(extension, member.group("name") or "init", member.group("parameters")))
+    return keys
+
+
+def registered_functions():
+    keys = set()
+    for match in REGISTERED.finditer(REGISTRY.read_text()):
+        keys.add(GENERIC_SUFFIX.sub("", match.group("key")))
+    return keys
+
+
+def main():
+    functions = public_functions()
+    registered = registered_functions()
+
+    missing = sorted(functions - registered)
+    # Registrations of computed properties (`Type.property`, no parameter list) are extras.
+    stale = sorted(key for key in registered - functions if "(" in key)
+
+    if missing:
+        print(f"error: {len(missing)} public function(s) have no reference values; register them in {REGISTRY.relative_to(ROOT)}:")
+        for key in missing:
+            print(f"  {key}")
+    if stale:
+        print(f"error: {len(stale)} registration(s) name no public function:")
+        for key in stale:
+            print(f"  {key}")
+    if missing or stale:
+        return 1
+
+    print(f"All {len(functions)} public functions have cross-platform reference values.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

Tests/FirebladeMathTests/FunctionTests.swift

@@ -20,28 +20,28 @@ struct FunctionTests {
         #expect(abs(FirebladeMath.acos(Float(0.5)) - 1.0471975) <= 1e-6)
     }
     @Test func acoshFunc() {
-        #expect(FirebladeMath.acosh(Double(123)) == 5.505315010967267)
-        #expect(FirebladeMath.acosh(Float(123)) == 5.505315)
+        #expect(isWithinULPs(FirebladeMath.acosh(Double(123)), 5.505315010967267))
+        #expect(isWithinULPs(FirebladeMath.acosh(Float(123)), 5.505315))
     }
     @Test func asinFunc() {
         #expect(abs(FirebladeMath.asin(Double(0.5)) - 0.5235987755982988) <= 1e-14)
-        #expect(FirebladeMath.asin(Float(0.5)) == 0.5235988)
+        #expect(isWithinULPs(FirebladeMath.asin(Float(0.5)), 0.5235988))
     }
     @Test func asinhFunc() {
         #expect(abs(FirebladeMath.asinh(Double(123)) - 5.505348060078276) <= 1e-14)
-        #expect(FirebladeMath.asinh(Float(123)) == 5.505348)
+        #expect(isWithinULPs(FirebladeMath.asinh(Float(123)), 5.505348))
     }
     @Test func atanFunc() {
-        #expect(FirebladeMath.atan(Double(123)) == 1.5626664246149526)
-        #expect(FirebladeMath.atan(Float(123)) == 1.5626664)
+        #expect(isWithinULPs(FirebladeMath.atan(Double(123)), 1.5626664246149526))
+        #expect(isWithinULPs(FirebladeMath.atan(Float(123)), 1.5626664))
     }
     @Test func atan2Func() {
-        #expect(FirebladeMath.atan2(Double(123), Double(123)) == 0.7853981633974483)
-        #expect(FirebladeMath.atan2(Float(123), Float(123)) == 0.7853982)
+        #expect(isWithinULPs(FirebladeMath.atan2(Double(123), Double(123)), 0.7853981633974483))
+        #expect(isWithinULPs(FirebladeMath.atan2(Float(123), Float(123)), 0.7853982))
     }
     @Test func atanhFunc() {
         #expect(abs(FirebladeMath.atanh(Double(0.5)) - 0.5493061443340549) <= 1e-14)
-        #expect(FirebladeMath.atanh(Float(0.5)) == 0.54930615)
+        #expect(isWithinULPs(FirebladeMath.atanh(Float(0.5)), 0.54930615))
     }
     @Test func ceilFunc() {
         #expect(FirebladeMath.ceil(Double(123.78)) == 124)
@@ -52,8 +52,8 @@ struct FunctionTests {
         #expect(FirebladeMath.clamp(Float(123), Float(1), Float(50)) == 50)
     }
     @Test func cosFunc() {
-        #expect(FirebladeMath.cos(Double(123)) == -0.8879689066918555)
-        #expect(FirebladeMath.cos(Float(123)) == -0.8879689)
+        #expect(isWithinULPs(FirebladeMath.cos(Double(123)), -0.8879689066918555))
+        #expect(isWithinULPs(FirebladeMath.cos(Float(123)), -0.8879689))
     }
     @Test func coshFunc() {
         #expect(FirebladeMath.cosh(Double.pi) == 11.591953275521519)
@@ -77,12 +77,12 @@ struct FunctionTests {
         #expect(FirebladeMath.dot(SIMD4<Float>(1, 2, 3, 4), SIMD4<Float>(5, 6, 7, 8)) == 70.0)
     }
     @Test func expFunc() {
-        #expect(FirebladeMath.exp(Double(13)) == 442413.3920089205)
-        #expect(FirebladeMath.exp(Float(13)) == 442413.4)
+        #expect(isWithinULPs(FirebladeMath.exp(Double(13)), 442413.3920089205))
+        #expect(isWithinULPs(FirebladeMath.exp(Float(13)), 442413.4))
     }
     @Test func exp2Func() {
-        #expect(FirebladeMath.exp2(Double(9)) == 512.0)
-        #expect(FirebladeMath.exp2(Float(9)) == 512.0)
+        #expect(isWithinULPs(FirebladeMath.exp2(Double(9)), 512.0))
+        #expect(isWithinULPs(FirebladeMath.exp2(Float(9)), 512.0))
     }
     @Test func faceforwardFunc() {
         #expect(FirebladeMath.faceforward(n: Double(1), i: Double(2), nref: Double(3)) == -1)
@@ -97,8 +97,8 @@ struct FunctionTests {
         #expect(FirebladeMath.fract(Float(123.56)) == 0.55999756)
     }
     @Test func hypotFunc() {
-        #expect(FirebladeMath.hypot(Double(1), Double(2)) == 2.23606797749979)
-        #expect(FirebladeMath.hypot(Float(1), Float(2)) == 2.236068)
+        #expect(isWithinULPs(FirebladeMath.hypot(Double(1), Double(2)), 2.23606797749979))
+        #expect(isWithinULPs(FirebladeMath.hypot(Float(1), Float(2)), 2.236068))
     }
     @Test func isInfiniteFunc() {
         #expect(FirebladeMath.isInfinite(Double(123)) == false)
@@ -130,16 +130,16 @@ struct FunctionTests {
         #expect(FirebladeMath.length(SIMD4<Float>(1, 2, 3, 4)) == 5.477226)
     }
     @Test func logFunc() {
-        #expect(FirebladeMath.log(Double(123)) == 4.812184355372417)
-        #expect(FirebladeMath.log(Float(123)) == 4.8121843)
+        #expect(isWithinULPs(FirebladeMath.log(Double(123)), 4.812184355372417))
+        #expect(isWithinULPs(FirebladeMath.log(Float(123)), 4.8121843))
     }
     @Test func log2Func() {
-        #expect(FirebladeMath.log2(Double(123)) == 6.94251450533924)
-        #expect(FirebladeMath.log2(Float(123)) == 6.9425144)
+        #expect(isWithinULPs(FirebladeMath.log2(Double(123)), 6.94251450533924))
+        #expect(isWithinULPs(FirebladeMath.log2(Float(123)), 6.9425144))
     }
     @Test func log10Func() {
-        #expect(FirebladeMath.log10(Double(123)) == 2.089905111439398)
-        #expect(FirebladeMath.log10(Float(123)) == 2.089905)
+        #expect(isWithinULPs(FirebladeMath.log10(Double(123)), 2.089905111439398))
+        #expect(isWithinULPs(FirebladeMath.log10(Float(123)), 2.089905))
     }
     @Test
     @available(*, deprecated, message: "Testing deprecated API")
@@ -175,12 +175,12 @@ struct FunctionTests {
         #endif
     }
     @Test func powFunc() {
-        #expect(FirebladeMath.pow(Double(123), Double(2)) == 15129.0)
-        #expect(FirebladeMath.pow(Float(123), Float(2)) == 15129.0)
+        #expect(isWithinULPs(FirebladeMath.pow(Double(123), Double(2)), 15129.0))
+        #expect(isWithinULPs(FirebladeMath.pow(Float(123), Float(2)), 15129.0))
     }
     @Test func pow2Func() {
-        #expect(FirebladeMath.pow2(Double(123)) == 1.0633823966279327e+37)
-        #expect(FirebladeMath.pow2(Float(123)) == 1.0633824e+37)
+        #expect(isWithinULPs(FirebladeMath.pow2(Double(123)), 1.0633823966279327e+37))
+        #expect(isWithinULPs(FirebladeMath.pow2(Float(123)), 1.0633824e+37))
     }
     @Test func radiansFunc() {
         #expect(FirebladeMath.radians(Double(123)) == 2.1467549799530254)
@@ -203,12 +203,12 @@ struct FunctionTests {
         #expect(FirebladeMath.sign(Float(-123)) == -1.0)
     }
     @Test func sinFunc() {
-        #expect(FirebladeMath.sin(Double(123)) == -0.45990349068959124)
+        #expect(isWithinULPs(FirebladeMath.sin(Double(123)), -0.45990349068959124))
         #expect(abs(FirebladeMath.sin(Float(123)) - (-0.45990348)) <= 1e-7)
     }
     @Test func sinhFunc() {
-        #expect(FirebladeMath.sinh(Double(0.5)) == 0.5210953054937474)
-        #expect(FirebladeMath.sinh(Float(0.5)) == 0.5210953)
+        #expect(isWithinULPs(FirebladeMath.sinh(Double(0.5)), 0.5210953054937474))
+        #expect(isWithinULPs(FirebladeMath.sinh(Float(0.5)), 0.5210953))
     }
     @Test func smoothstepFunc() {
         #expect(FirebladeMath.smoothstep(Double(12), Double(24), Double(13)) == 0.019675925925925927)
@@ -224,11 +224,11 @@ struct FunctionTests {
     }
     @Test func tanFunc() {
         #expect(abs(FirebladeMath.tan(Double(123)) - 0.5179274715856551) <= 1e-14)
-        #expect(FirebladeMath.tan(Float(123)) == 0.51792747)
+        #expect(isWithinULPs(FirebladeMath.tan(Float(123)), 0.51792747))
     }
     @Test func tanhFunc() {
-        #expect(FirebladeMath.tanh(Double(0.5)) == 0.46211715726000974)
-        #expect(FirebladeMath.tanh(Float(0.5)) == 0.46211717)
+        #expect(isWithinULPs(FirebladeMath.tanh(Double(0.5)), 0.46211715726000974))
+        #expect(isWithinULPs(FirebladeMath.tanh(Float(0.5)), 0.46211717))
     }
     @Test func remapFunc() {
         #expect(Double(5).remaped(clampingIn: 0...10, to: 0...100) == 50.0)
@@ -305,4 +305,10 @@ private func expectEqualElements<T, S>(_ expression1: @autoclosure () throws ->
     } catch {
         Issue.record(error, Comment(stringLiteral: "Error thrown during element comparison"), sourceLocation: sourceLocation)
     }
-}
+}
+
+/// C math library results may differ by a few ulp between platforms (Apple libm, glibc,
+/// ucrt, musl/WASI); FirebladeMath only guarantees identical results up to rounding.
+private func isWithinULPs<T: BinaryFloatingPoint>(_ value: T, _ expected: T, _ ulps: T = 4) -> Bool {
+    Swift.abs(value - expected) <= ulps * expected.ulp
+}

Tests/FirebladeMathTests/Matrix4x4CofactorTests.swift

@@ -15,7 +15,7 @@ struct Matrix4x4CofactorTests {
         var state: UInt64 = 0x9E37_79B9_7F4A_7C15
         func next() -> Double {
             state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407
-            return Double(state >> 11) / Double(1 << 53) * 4 - 2
+            return Double(state >> 11) * 0x1p-53 * 4 - 2
         }
         var result: [[Double]] = []
         for _ in 0..<32 {

Tests/FirebladeMathTests/PortableVsSIMD/PortableVsSIMDSupport.swift

@@ -31,7 +31,7 @@ struct SplitMix {
         z = (z ^ (z >> 30)) &* 0xBF58_476D_1CE4_E5B9
         z = (z ^ (z >> 27)) &* 0x94D0_49BB_1331_11EB
         z ^= z >> 31
-        let unit = Double(z >> 11) / Double(1 << 53)
+        let unit = Double(z >> 11) * 0x1p-53 // no `1 << 53`: Int is 32-bit on wasm32
         return range.lowerBound + unit * (range.upperBound - range.lowerBound)
     }
 

Tests/FirebladeMathTests/QuaternionMultiplicationTests.swift

@@ -15,7 +15,7 @@ struct QuaternionMultiplicationTests {
         var state: UInt64 = 0xD1B5_4A32_D192_ED03
         func next() -> Double {
             state = state &* 6_364_136_223_846_793_005 &+ 1_442_695_040_888_963_407
-            return Double(state >> 11) / Double(1 << 53) * 4 - 2
+            return Double(state >> 11) * 0x1p-53 * 4 - 2
         }
         return (0..<64).map { _ in ((0..<4).map { _ in next() }, (0..<4).map { _ in next() }) }
     }()