[Engine] run Mirai's Qwen3.8-Flash-Next-S (QNS trellis) packs

CMakeLists.txt

@@ -408,6 +408,8 @@ if(STRATA_ENABLE_CUDA OR STRATA_ENABLE_HIP)
                           src/kernels/cuda/qsa_select.cu
                           src/kernels/cuda/verify_kernels.cu
                           src/kernels/cuda/iq_kernels.cu
+                          src/kernels/cuda/qns_kernels.cu
+                          src/kernels/cuda/qns_gather.cu
                           src/kernels/ngram.cpp
                           src/ngram/ple_reader.cpp
                           src/platform/direct_file.cpp)
@@ -1541,7 +1543,8 @@ if(STRATA_NATIVE_EXPERTS)
                                            src/kernels/cpu/iq_avx2.cpp src/kernels/cpu/kq_avx2.cpp
                                            src/kernels/cpu/kq_avx1.cpp src/kernels/cpu/q2_avx1.cpp src/kernels/cpu/iq_avx1.cpp
                                            src/kernels/cpu/nvfp4_avx512.cpp src/kernels/cpu/nvfp4_avx2.cpp
-                                           src/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp)
+                                           src/kernels/cpu/tq_cpu.cpp src/kernels/cpu/tq_avx2.cpp src/kernels/cpu/tq_avx512.cpp
+                                           src/kernels/cpu/qns_avx2.cpp)
   target_include_directories(strata_kernels_cpu PRIVATE third_party/ggml)
   if(MSVC)
     set_source_files_properties(src/kernels/cpu/iq_avx512.cpp src/kernels/cpu/nvfp4_avx512.cpp
@@ -1551,6 +1554,7 @@ if(STRATA_NATIVE_EXPERTS)
     set_source_files_properties(src/kernels/cpu/nvfp4_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
     set_source_files_properties(src/kernels/cpu/tq_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
     set_source_files_properties(src/kernels/cpu/tq_avx512.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX512")
+    set_source_files_properties(src/kernels/cpu/qns_avx2.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX2")
     # the router lookahead's AVX1 dot (older CPUs; called behind cpu_avx1_ok): no FMA, no 256-bit integer ops
     set_source_files_properties(src/kernels/cpu/kq_avx1.cpp PROPERTIES COMPILE_OPTIONS "/arch:AVX")
     # the older CPUs' Q2_0 rows and activation quantizer (q2_avx1.cpp, issue #1699): 128-bit integer lanes,
@@ -1568,6 +1572,8 @@ if(STRATA_NATIVE_EXPERTS)
     set_source_files_properties(src/kernels/cpu/tq_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
     set_source_files_properties(src/kernels/cpu/tq_avx512.cpp PROPERTIES COMPILE_OPTIONS
                                 "-mavx512f;-mavx512bw;-mavx512vl;-mavx512vnni;-mavx512vbmi;-mavx2;-mfma;-mf16c")
+    # Qwen Next S trellis experts (qns.hpp): AVX2 only, the floor of the target desktops and Zen 2 servers
+    set_source_files_properties(src/kernels/cpu/qns_avx2.cpp PROPERTIES COMPILE_OPTIONS "-mavx2;-mfma;-mf16c")
     set_source_files_properties(src/kernels/cpu/kq_avx1.cpp PROPERTIES COMPILE_OPTIONS "-mavx")
     # the older CPUs' Q2_0 rows and activation quantizer (q2_avx1.cpp, issue #1699): SSE4.2 + AVX with
     # AVX2/FMA/F16C explicitly off, so a -march=native build host cannot leak AVX2 code into the file that

docs/QNS.md

@@ -0,0 +1,57 @@
+# Mirai Qwen3.8-Flash-Next-S (QNS trellis)
+
+Validated on Linux (Ubuntu 22.04, CUDA 12.8) with RTX 3090s at 300 W, an EPYC 7642 (AVX2) and 125 GB RAM on
+2026-10-20.
+
+Mirai's Flash-Next-S is a trellis build of Qwen3.8-Flash-Next: the routed experts take 1 to 5 bits each (2.4 on
+average), the backbone and head 5 bits. Mirai packed it for Strata as two files: `Qwen3.8-Flash-Next-S-qns.gguf`
+(the experts and a Q8_0 backbone) and `Qwen3.8-Flash-Next-S-qns-dense.gguf` (the backbone and head as trellis, which
+takes less VRAM). There is also a 32 GB build, S-small. The weights are on Hugging Face from Mirai
+(`trymirai/Qwen3.8-Flash-Next-S-strata-experimental`).
+
+Every expert is its own blob of its own size, so the RAM arena is compact and each VRAM slot is sized to its expert.
+The format, the CUDA and AVX2 kernels and the packer support come from Mirai's `strata-qns.patch` for v0.1.43.
+
+## Pack
+
+```sh
+.venv/bin/python tools/iq_pack.py --gguf /models/Qwen3.8-Flash-Next-S-qns.gguf --out packs/qns
+```
+
+It takes seconds and writes `native_experts.txt` v6 with `qns_offsets.bin` and `qns_rotation.bin`. A pack made by the
+v0.1.43 patch (v5) is refused with a note to repack: v5 means Hadamard-folded experts in this engine.
+
+## Run
+
+The n-gram table is the PLE shard of any ISTA-DASLab size (shard 2, 28.8 GB); the MTP layer and the expert profile
+are the original model's.
+
+```sh
+build/strata --serve --pack packs/qns \
+  --native /models/Qwen3.8-Flash-Next-S-qns.gguf --native-dense-gguf /models/Qwen3.8-Flash-Next-S-qns.gguf \
+  --qns-dense-gguf /models/Qwen3.8-Flash-Next-S-qns-dense.gguf \
+  --ple-gguf /models/IQ3_S/Qwen3.8-Flash-Next-GSQ-RCO-IQ3_S-00002-of-00002.gguf \
+  --expert-profile data/expert-profile.bin --expert-cache auto --prefill auto --spec 4 --mtp <mtp/rt> --kv int8
+```
+
+`--layer-split auto` with `--gpu 0,1` runs it on two cards.
+
+## Measured
+
+Greedy, thinking off, 32K context, MTP on, the server's own tok/s per request (six short requests and a 6,210-token
+document); one run each, so the ranges are indicative, not a benchmark:
+
+| Model | Cards | Experts in RAM | Decode | 6,210-token prompt |
+|---|---|---:|---:|---:|
+| S | 1 | 33.8 GiB | 35-81 tok/s | 4.1 s |
+| S | 2 (layer split) | 33.8 GiB | 60-125 tok/s | 4.2 s |
+| S-small | 1 | 26.5 GiB | 47-106 tok/s | 3.9 s |
+| S-small | 2 (layer split) | 26.5 GiB | 83-166 tok/s | 4.0 s |
+
+On S-small the greedy answers of this engine and of v0.1.43 with Mirai's patch are the same on five of six probes
+(the sixth parts on a near-tie after 10-55 words), on one card and on two.
+
+## Limits
+
+- NVIDIA (CUDA) and the AVX2 CPU pool. Not built or run on AMD (HIP), Intel (SYCL) or Mac.
+- Setup and the Terminal do not offer these models yet; start them with a config like the command above.

include/strata/artifact/gguf_reader.hpp

@@ -230,6 +230,14 @@ inline bool block_geometry(uint32_t t, int& elems, int& bytes) {
         elems = 32;
         bytes = 17;
         return true;
+    case 26:   // I32: a qns backbone's input permutation (<name>.qns_perm)
+        elems = 1;
+        bytes = 4;
+        return true;
+    case 27:   // I64: a qns model's per-layer expert index (blk.N.ffn_exps_qns_idx)
+        elems = 1;
+        bytes = 8;
+        return true;
     case 42:
         elems = 64;
         bytes = 18;

include/strata/core/expert_cache.hpp

@@ -232,6 +232,8 @@ class ExpertCache {
         if (s < 0 || s >= slots_ || base_ == nullptr) return nullptr;
         return slot_room(s) >= blob + need ? base_ + slot_offset(s) + blob : nullptr;
     }
+    /// Bytes slot `s` holds.
+    uint64_t slot_bytes(int64_t s) const { return slot_offset(s + 1) - slot_offset(s); }
 
 private:
 #if defined(STRATA_USE_HIP)

include/strata/core/mtp.hpp

@@ -261,6 +261,8 @@ class MtpDrafter {
     float *h_prob_ = nullptr, *m_prob_ = nullptr;   // each draft's probability under the draft layer
     // the draft head: the main head's rows for a token subset (rt/draft_vocab.bin), or the whole head
     uint8_t* dhead_ = nullptr;
+    float* dscales_ = nullptr;               ///< a trellis head (kQnsDenseType): the draft subset's row scales
+    uint8_t* qns_act_ = nullptr;             ///< and the drafter's rotated, quantized head input
     int32_t* dvocab_ = nullptr;
     int64_t n_dvocab_ = 0;
     std::string rt_dir_;

include/strata/core/native_dense.hpp

@@ -31,6 +31,10 @@ class NativeDense {
     /// Layer split: load only blocks [lb, le) (every other `blk.N.` projection belongs to another GPU's stage; the
     /// PLE tensors are loaded everywhere).  Process-wide, read by the next `load`; (-1, -1) = all layers.
     static void set_layer_range(int lb, int le);
+    /// --qns-dense-gguf: the GGUF that holds Qwen Next S backbone matrices as trellis (<name>.qns_words / .qns_scales /
+    /// .qns_perm); `load` serves each matrix it has from there instead of the model's Q8_0 blocks.  Empty: off.
+    static void set_qns_dense(const std::string& path);
+    static const std::string& qns_dense();
     /// Layer split: the device bytes `load` would allocate for the matrices of layers [lb, le) - the same walk and
     /// filters, each matrix rounded up to the granule cudaMalloc backs it with - reading the GGUF headers and `table`
     /// only, so it allocates nothing and needs no device (#1238).  `table` must hold every tensor the shards carry.

include/strata/core/native_head.hpp

@@ -31,10 +31,15 @@ class NativeHead {
     int device() const { return device_; }
     /// Bytes of one vocabulary row.
     size_t row_bytes() const { return n_out_ > 0 ? (size_t) (bytes_ / (uint64_t) n_out_) : 0; }
+    /// type() == kQnsDenseType (the head as trellis from --qns-dense-gguf): its row scales and T bits per step.
+    const float* qns_scales() const { return qns_scales_; }
+    int qns_bits() const { return qns_bits_; }
 
 private:
     void* weights_ = nullptr;
     void* scratch_ = nullptr;
+    float* qns_scales_ = nullptr;
+    int qns_bits_ = 0;
     uint64_t bytes_ = 0;
     int n_in_ = 0, n_out_ = 0;
     int type_ = -1;

include/strata/core/verify.hpp

@@ -612,6 +612,9 @@ class Verifier {
                                   // unless some layer's gate/up is folded
     const float* had_sx_ = nullptr;   // APR: device sign vectors for x (n_embd) and h (n_ff); nullptr = identity
     const float* had_sh_ = nullptr;
+    uint8_t* qns_xq_ = nullptr;   // a qns pack's rotated two-plane activations (qns_cuda.hpp)
+    uint8_t* qns_xd_ = nullptr;   // a trellis backbone's 2560-wide layer input, quantized (qns_dense_quant)
+    uint8_t* qns_od_ = nullptr;   // and its 6144-wide output-projection input
     float* hit_xs_ = nullptr;
     void* hit_scratch_ = nullptr;
     float *head_mixed_ = nullptr, *head_inj_ = nullptr, *head_logits_ = nullptr;

include/strata/core/weights.hpp

@@ -45,6 +45,10 @@ enum class WeightKind : int {
     F16InF32 = 3,   ///< an f16 value promoted to f32: a real f32->f16 conversion, not a truncation
 };
 
+/// A Qwen Next S backbone matrix served as its trellis (--qns-dense-gguf; not a ggml type id): `native_data` holds the
+/// row tapes as u32 words in 32-row groups, `qns_scales` the row scales (qns_cuda.hpp).
+inline constexpr int kQnsDenseType = 251;
+
 struct WeightRef {
     const void* data = nullptr;   ///< device pointer inside the arena
     uint64_t bytes = 0;
@@ -90,6 +94,11 @@ struct WeightRef {
     const void* native_data = nullptr;
     void* native_q8_1 = nullptr;
     int native_type = -1;
+    /// native_type == kQnsDenseType: the row scales, T bits per step, and for an input whose order differs from the
+    /// checkpoint's columns (GDN out_proj) the gather: checkpoint column c is the engine's input index qns_perm[c].
+    const float* qns_scales = nullptr;
+    const int32_t* qns_perm = nullptr;
+    int qns_bits = 0;
     /// S23 experiment (STRATA_HC_Q8=1): a hyper-connection projection's Q8_0 bytes as the GGUF stores them, owned by
     /// NativeDense, for the verify window's read (every other path keeps the pack's BF16 `data`)
     const void* hc_q8 = nullptr;

include/strata/kernels/cpu/expert_layout.hpp

@@ -15,6 +15,10 @@
 
 namespace strata::kernels::cpu {
 
+/// Qwen Next S trellis experts (not a ggml type id): one self-describing qns blob per expert (header with its own bit
+/// widths, trellis words in 32-row groups, row scales).  native_experts.txt v6 gives such a layer `kQnsType kQnsType`.
+inline constexpr int kQnsType = 250;
+
 struct ExpertLayout {
     bool native = false;
     int64_t n_layers = 0, n_expert = NE;
@@ -31,14 +35,28 @@ struct ExpertLayout {
     int version = 0;                      ///< native_experts.txt's header version (0 = none given)
     uint64_t max_blob = BLOB;
     uint64_t total = 0;                   ///< experts.bin size
+    /// qns packs (qns_offsets.bin): each expert's blob has its own size, so the arena is compact and a blob's offset
+    /// is this table's, n_layers x (n_expert + 1) with each layer's end last.  `bytes` stays the layer's largest blob:
+    /// the VRAM slot and the copy size.  A copy of `bytes` may run into the next blob; `total` ends with max_blob of
+    /// slack so it never runs past the arena.
+    std::vector<uint64_t> qns_off;
+    /// qns packs (qns_rotation.bin): the experts' input rotations, signs[2560], Q[5][5] (2560 = 512 x 5), signs[640],
+    /// Q[5][5] (640 = 128 x 5).
+    std::vector<float> qns_rotation;
 
+    bool qns() const { return !qns_off.empty(); }
     uint64_t blob_bytes(int64_t layer) const { return native ? bytes[(size_t) layer] : (uint64_t) BLOB; }
     uint64_t layer_offset(int64_t layer) const {
         return native ? offset[(size_t) layer] : (uint64_t) layer * (uint64_t) n_expert * (uint64_t) BLOB;
     }
     uint64_t blob_offset(int64_t layer, int64_t expert) const {
+        if (qns()) return qns_off[(size_t) (layer * (n_expert + 1) + expert)];
         return layer_offset(layer) + (uint64_t) expert * blob_bytes(layer);
     }
+    /// One expert's own blob: a qns blob's size (its VRAM slot is sized to it), else the layer's.
+    uint64_t expert_bytes(int64_t layer, int64_t expert) const {
+        return qns() ? blob_offset(layer, expert + 1) - blob_offset(layer, expert) : blob_bytes(layer);
+    }
 };
 
 /// Plan v0.3 P6: whether this CPU (and its OS) runs the AVX-512 kernels (F, BW, VL, VNNI, VBMI).  Probed in a
@@ -98,6 +116,8 @@ bool expert_layout_load(const std::string& pack_dir, int64_t n_layers, int64_t n
 /// written only when some layer's roles are in different shards (every other pack stays v3, byte for byte).
 /// v5 = v4 plus `hadamard.txt` beside it: Hadamard-folded experts (hadamard.hpp), written only for such models,
 /// so an engine that cannot rotate the activations refuses the pack instead of running it wrong.
-inline constexpr int kExpertLayoutVersion = 5;
+/// v6 = qns layers (`kQnsType kQnsType offset max_blob payload_off 0 0`) with qns_offsets.bin and qns_rotation.bin
+/// (strata-qns.patch for 0.1.43 wrote these as v5; such a pack is refused with a note to repack).
+inline constexpr int kExpertLayoutVersion = 6;
 
 }  // namespace strata::kernels::cpu

include/strata/kernels/cpu/native_expert.hpp

@@ -13,10 +13,11 @@
 
 namespace strata::kernels::cpu {
 
-/// Bytes of the largest quantized activation any native layer uses (2560 values as Q8_K: 10 x 292).
-inline constexpr size_t kNativeActBytes = 4096;
-/// Bytes of the largest quantized down activation (640 values as Q8_0: 20 x 34, or Q8_K 3 x 292).
-inline constexpr size_t kNativeHBytes = 1024;
+/// Bytes of the largest quantized activation any native layer uses (2560 values as Q8_K: 10 x 292; a qns layer's
+/// rotated two-plane int8 activation is larger, qns.hpp).
+inline constexpr size_t kNativeActBytes = 16384;
+/// Bytes of the largest quantized down activation (640 values as Q8_0: 20 x 34, or Q8_K 3 x 292; qns: qns.hpp).
+inline constexpr size_t kNativeHBytes = 4096;
 
 /// One layer's native expert geometry.
 struct NativeFmt {

include/strata/kernels/cpu/qns.hpp

@@ -0,0 +1,38 @@
+// include/strata/kernels/cpu/qns.hpp - Qwen Next S routed experts (trellis "qns" blobs, qns_blob.py) on AVX2.
+//
+// A blob row is a trellis path in a rotated basis: the input is rotated (x_rot = (H (x) Q)(signs * x)) and quantized
+// per token to x_rot ~ step * (q0 + q1 / 254) (two int8 planes), the kernel computes per row the exact integer
+// S = sum (level + 54) * (254 q0 + q1) = 254 C + F (C, F the two planes' sums of the GPU kernels), and finishes
+//     y = scale * ((step / RMS) * (S - 54 * sum (254 q0 + q1)) / 254 + d0 * E + d1 * O),   d_j = -MEAN_j / RMS
+// with E, O = step * sum (q0 + q1 / 254) over the even / odd columns. Same contract as the vLLM plugin's kernels.cu.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+
+namespace strata::kernels::cpu::qns {
+
+/// One token's quantized activation (4-byte aligned): a 16-byte header {float step; int32 sum of 254 q0 + q1;
+/// float E; float O}, then int16 254 q0 + q1 of the rotated columns, each 4 columns 4g .. 4g + 3 stored as 4g, 4g + 2,
+/// 4g + 1, 4g + 3.
+inline constexpr size_t kActBytes = 16 + 2 * 2560;  ///< expert input x (2560)
+inline constexpr size_t kHBytes = 16 + 2 * 640;     ///< down input h = silu(gate) * up (640)
+
+/// The rotation factors (rotation.signs.N / rotation.q.N of the package, q row-major 5 x 5). Once, at load.
+void set_rotation(const float* signs2560, const float* q2560, const float* signs640, const float* q640);
+/// x (2560 floats) -> kActBytes.
+void quant_act(const float* x, void* dst);
+/// h (640 floats) -> kHBytes.
+void quant_h(const float* h, void* dst);
+
+/// ff[t][r] = silu(gate_r . x_t) * (up_r . x_t) for rows r in [r0, r1) of [0, 640), nt (1..8) tokens. Rows go in
+/// blocks of 8: a range that cuts a block computes the whole block (split ranges at multiples of 8).
+void gu_rows(const uint8_t* blob, const void* const* act, int nt, float* const* ff, int r0, int r1);
+/// out[t][r] = down_r . h_t for rows r in [r0, r1) of [0, 2560), nt (1..8) tokens, blocks of 8 as gu_rows.
+void down_rows(const uint8_t* blob, const void* const* hq, int nt, float* const* out, int r0, int r1);
+
+/// The kernels' core, exposed for the exactness test: s[t][i] = S of row 8 b + i of the gate_up matrix (rows 0-639
+/// gate, 640-1279 up; down = false) or of the down matrix, for act[0 .. nt).
+void block_sums(const uint8_t* blob, bool down, int b, const void* const* act, int nt, int64_t (*s)[8]);
+
+}  // namespace strata::kernels::cpu::qns