gpu: Tier A KERNEL_SERIALIZED PC sampling, duty-cycled, and its disclosure

gpu/conformance_test.go

@@ -100,6 +100,7 @@ type attemptSink struct {
 	pcAttempts     uint64
 	moduleAttempts uint64
 	eventAttempts  uint64
+	windowAttempts uint64
 }
 
 func newAttemptSink(inner EventSink) *attemptSink {
@@ -135,6 +136,11 @@ func (a *attemptSink) EmitEvent(e GPUTimelineEvent) error {
 	return a.inner.EmitEvent(e)
 }
 
+func (a *attemptSink) EmitSamplingWindow(w GPUSamplingWindow) error {
+	a.windowAttempts++
+	return a.inner.EmitSamplingWindow(w)
+}
+
 // conformanceHarness is a fresh Timeline + CountingSink pair, wired the way a
 // real backend is: producer -> CountingSink (admission control) ->
 // Timeline (join point). The clock is frozen (never advanced) so the token
@@ -254,6 +260,14 @@ func assertConformanceInvariants(t *testing.T, h *conformanceHarness) Snapshot {
 	assertPCSampleLossesAccounted(t, snap, sinkStats, h.attempt.pcAttempts)
 	assertPCAttribAccompaniesSamples(t, snap)
 
+	// The serialization disclosure's sum identity, on EVERY scenario in this
+	// file rather than only on the ones that emit windows. That is the point:
+	// the three outcomes are exhaustive and mutually exclusive, so they must
+	// add up in a run with no windows at all exactly as they do in one full of
+	// them, and an execution that reached the profile carrying no disclosure
+	// would otherwise be invisible.
+	assertSerializationOutcomesAccounted(t, snap)
+
 	return snap
 }
 
@@ -288,6 +302,31 @@ func assertPCAttribAccompaniesSamples(t *testing.T, snap Snapshot) {
 	}
 }
 
+// assertSerializationOutcomesAccounted extends invariant 5's discipline to
+// gpu_serialized: every execution in the snapshot takes exactly one of the
+// three outcomes, so they sum to len(snap.Executions).
+//
+// It also asserts the negative that matters more than the sum: with the
+// default harness configuration (SerializedSampling unset — nothing is ever
+// serialized) NO execution may read "true" or "unknown". A conformance run
+// that started reporting perturbation nobody caused would be as wrong as one
+// that stopped reporting perturbation that happened.
+func assertSerializationOutcomesAccounted(t *testing.T, snap Snapshot) {
+	t.Helper()
+	sum := snap.ExecutionsSerialized + snap.ExecutionsNotSerialized +
+		snap.ExecutionsSerializationUnknown
+	assert.Equal(t, uint64(len(snap.Executions)), sum,
+		"the three gpu_serialized outcomes must sum to the executions in the snapshot")
+	assert.Zero(t, snap.ExecutionsSerialized,
+		"nothing serializes kernels in this configuration")
+	assert.Zero(t, snap.ExecutionsSerializationUnknown,
+		"with no serialized sampling selected, \"false\" is unconditional and correct")
+	for _, v := range snap.Executions {
+		assert.Equal(t, SerializationNotSerialized, v.Serialized,
+			"execution %+v", v.Exec.Correlation)
+	}
+}
+
 // assertNoFabricatedLaunch is invariant 3: every non-nil view.Launch must
 // carry a Correlation that some emitted launch actually carried, never a
 // zero-value or otherwise invented value.

gpu/joinhealth.go

@@ -147,6 +147,18 @@ func joinSummary(snap Snapshot, anomalies int) string {
 			plural(uint64(snap.PendingModuleGroups), "kernel group", "kernel groups"))
 	}
 
+	// The serialization disclosure, and only when there is one to make. With
+	// PC sampling off or in continuous collection every execution is "false"
+	// and nothing was ever serialized, so a permanent "0 serialized" clause
+	// would be exactly the zero-valued noise this format avoids.
+	if snap.ExecutionsSerialized > 0 || snap.ExecutionsSerializationUnknown > 0 ||
+		snap.SamplingWindowsReceived > 0 {
+		fmt.Fprintf(&b, "; serialization %d true, %d false, %d unknown over %s",
+			snap.ExecutionsSerialized, snap.ExecutionsNotSerialized,
+			snap.ExecutionsSerializationUnknown,
+			plural(uint64(snap.SamplingWindowsHeld), "burst", "bursts"))
+	}
+
 	switch anomalies {
 	case 0:
 		b.WriteString("; no anomalies")
@@ -196,6 +208,48 @@ func joinAnomalies(snap Snapshot, proj ProjectionStats) []string {
 			"disagree with what is actually present; treat every figure below as unreliable",
 			outcomes, execs)
 	}
+	// The serialization disclosure's own sum identity, and it is raised in the
+	// same place and for the same reason as the join one above: three
+	// mutually-exclusive outcomes, every execution takes exactly one, so they
+	// must add up to what is actually in the snapshot. A shortfall means an
+	// execution reached the profile with no disclosure at all.
+	if serialization := snap.ExecutionsSerialized + snap.ExecutionsNotSerialized +
+		snap.ExecutionsSerializationUnknown; serialization != execs {
+		add("serialization outcomes sum to %d but the snapshot holds %d executions — some "+
+			"execution carries no gpu_serialized disclosure at all; treat every duration "+
+			"in this profile as unqualified",
+			serialization, execs)
+	}
+	// The whole point of Tier A's disclosure. Raised BEFORE the join
+	// anomalies below because it qualifies the durations themselves rather
+	// than what they were attributed to: a perturbed measurement joined
+	// perfectly is still a perturbed measurement.
+	if snap.ExecutionsSerialized > 0 {
+		add("%d of %d executions ran while GPU kernels were SERIALIZED by the profiler — "+
+			"their durations are inflated by the measurement and are marked "+
+			"gpu_serialized=\"true\". CPU and off-CPU samples taken during those bursts are "+
+			"distorted too and carry no marking at all",
+			snap.ExecutionsSerialized, execs)
+	}
+	if snap.ExecutionsSerializationUnknown > 0 && snap.SamplingWindowsReceived > 0 {
+		add("%d of %d executions cannot be said to have run unperturbed — no sampling window "+
+			"covers them (a dropped batch, a late attach, a sequence gap, or a burst that "+
+			"never closed). They are marked gpu_serialized=\"unknown\" and MUST NOT be read "+
+			"as \"false\"",
+			snap.ExecutionsSerializationUnknown, execs)
+	}
+	if snap.SamplingWindowsOpen > 0 {
+		add("%s still open — the producer stopped reporting mid-burst (a hard exit), so the "+
+			"end of the burst is unknown and every execution from its start onward is "+
+			"gpu_serialized=\"unknown\"",
+			plural(uint64(snap.SamplingWindowsOpen), "sampling window", "sampling windows"))
+	}
+	if dr.EvictedSamplingWindows > 0 {
+		add("%s evicted from the serialization disclosure store — executions that far back "+
+			"degrade from \"false\" to \"unknown\"; raise "+
+			"TimelineConfig.MaxSamplingWindowsPerPID or snapshot more often",
+			plural(dr.EvictedSamplingWindows, "sampling window", "sampling windows"))
+	}
 	if js.UnmatchedExecutionCount > 0 {
 		add("%d of %d executions unmatched — GPU time arrived with no launch to attach it to; "+
 			"it is in the profile under %s carrying no CPU stack",

gpu/joinhealth_test.go

@@ -20,6 +20,12 @@ func healthySnapshot() Snapshot {
 			ExactExecutionJoinCount: 512,
 		},
 		LaunchCache: LaunchCacheStats{Live: 256},
+		// PC sampling off, so every execution is "false" unconditionally and
+		// correctly: nothing was ever serialized. Set here rather than left
+		// zero because the zero value of SerializationState is "unknown" and
+		// the three counters must sum to len(Executions) — the same identity
+		// the join outcomes carry, checked in the same place.
+		ExecutionsNotSerialized: 512,
 	}
 }
 
@@ -57,6 +63,16 @@ func anomalousSnapshot() Snapshot {
 		AttributedPCSamples: 900,
 		PendingSamples:      12,
 		PendingCorrelations: 5,
+
+		// Tier A gone wrong in all three ways at once: bursts perturbed some
+		// executions, an unbroken history proved others were untouched, and a
+		// window that never closed leaves the rest unplaceable.
+		SamplingWindowsReceived:        41,
+		SamplingWindowsHeld:            21,
+		SamplingWindowsOpen:            1,
+		ExecutionsSerialized:           120,
+		ExecutionsNotSerialized:        300,
+		ExecutionsSerializationUnknown: 92,
 	}
 }
 
@@ -93,6 +109,10 @@ func TestJoinHealthAnomaliesEachGetTheirOwnLine(t *testing.T) {
 		"11 timeline events evicted",
 		"1 module evicted before this snapshot — kernels from evicted modules resolve",
 		"sink dropped 64 PC samples at admission",
+		"120 of 512 executions ran while GPU kernels were SERIALIZED",
+		"92 of 512 executions cannot be said to have run unperturbed",
+		"1 sampling window still open",
+		"serialization 120 true, 300 false, 92 unknown over 21 bursts",
 	} {
 		assert.Contains(t, joined, want)
 	}

gpu/projection.go

@@ -623,6 +623,28 @@ func projectionLabels(view ExecutionView) map[string]string {
 	if view.Ambiguous {
 		labels["gpu_ambiguous"] = "true"
 	}
+	// gpu_serialized is set UNCONDITIONALLY on every execution, exactly as
+	// gpu_join is and for exactly the same reason: an absent label would read
+	// as "not perturbed" to a consumer that does not know to check for its
+	// absence, and "not perturbed" is the one answer that must never be
+	// reachable by accident.
+	//
+	// It rides on every execution rather than only on PC-bearing ones because
+	// serialization is a property of the INTERVAL, not of whether a sample
+	// landed: every kernel that ran inside a burst ran serialized, sampled or
+	// not (see gpu/serialization.go).
+	//
+	// The value comes from SerializationState.String(), whose zero value is
+	// "unknown" — so a view that never reached the classifier degrades to
+	// "unknown" here rather than to "false".
+	//
+	// LIMITATION, stated rather than left to be discovered: this is the GPU
+	// projection. On-CPU and off-CPU samples taken during a burst are
+	// distorted too — serialization inflates precisely the synchronization
+	// wait that off-CPU profiling exists to measure — and they carry no
+	// marking at all, because those profilers know nothing about GPUs.
+	// joinhealth reports it whenever any window was recorded.
+	labels["gpu_serialized"] = view.Serialized.String()
 	// gpu_sample_period rides only on the population that actually carries a
 	// sampled stack, and only when the producer declared a period. It is the
 	// denominator a consumer needs to extrapolate "sampled GPU time" to "all

gpu/projection_test.go

@@ -943,3 +943,71 @@ func TestProjectionSourceLabelsCannotBeForgedByAbsence(t *testing.T) {
 		"gpu_src_status is unconditional on PC-DERIVED samples; an execution with none has nothing to say")
 	assert.NotContains(t, samples[2].Labels, "gpu_pc_attrib")
 }
+
+// gpu_serialized is on EVERY sample projected from EVERY execution, with one
+// of exactly three values. It is not omitempty and there is no "only if
+// interesting" branch: an absent label reads as "not perturbed" to a consumer
+// that does not know to check for its absence, which is the failure the whole
+// disclosure exists to prevent.
+func TestSerializedLabelIsUnconditionalAndHasThreeValues(t *testing.T) {
+	snap := Snapshot{Executions: []ExecutionView{
+		{Exec: GPUKernelExec{StartNs: 0, EndNs: 10, KernelName: "kA"},
+			Serialized: SerializationSerialized},
+		{Exec: GPUKernelExec{StartNs: 0, EndNs: 10, KernelName: "kB"},
+			Serialized: SerializationNotSerialized},
+		{Exec: GPUKernelExec{StartNs: 0, EndNs: 10, KernelName: "kC"},
+			Serialized: SerializationUnknown},
+		// A view nobody classified at all. The zero value of
+		// SerializationState is "unknown", so this degrades to the safe
+		// answer rather than to "false".
+		{Exec: GPUKernelExec{StartNs: 0, EndNs: 10, KernelName: "kD"}},
+	}}
+
+	samples := ProjectExecutions(snap)
+	require.Len(t, samples, 4)
+	assert.Equal(t, "true", samples[0].Labels["gpu_serialized"])
+	assert.Equal(t, "false", samples[1].Labels["gpu_serialized"])
+	assert.Equal(t, "unknown", samples[2].Labels["gpu_serialized"])
+	assert.Equal(t, "unknown", samples[3].Labels["gpu_serialized"],
+		"an unclassified execution must degrade to \"unknown\", never to \"false\"")
+	for i, s := range samples {
+		assert.Contains(t, s.Labels, "gpu_serialized", "sample %d", i)
+	}
+}
+
+// It rides on the PC-derived samples too — one per PC sample, all carrying the
+// execution's own disclosure. Serialization is a property of the interval, not
+// of whether a sample landed.
+func TestSerializedLabelReachesEveryPCDerivedSample(t *testing.T) {
+	snap := Snapshot{Executions: []ExecutionView{
+		{Exec: GPUKernelExec{StartNs: 0, EndNs: 100, KernelName: "kAdd"},
+			Serialized: SerializationSerialized,
+			PCSamples: []GPUPCSample{
+				{PCOffset: 0x10, Count: 1}, {PCOffset: 0x20, Count: 3}}},
+	}}
+
+	samples := ProjectExecutions(snap)
+	require.Len(t, samples, 2)
+	for i, s := range samples {
+		assert.Equal(t, "true", s.Labels["gpu_serialized"], "pc sample %d", i)
+	}
+}
+
+// A producer-supplied tag must not be able to forge the disclosure — this is
+// the reserved-name rule, and it matters more here than anywhere else: a tag
+// named gpu_serialized set to "false" would claim a perturbed measurement was
+// clean.
+func TestSerializedLabelBeatsAForgedTag(t *testing.T) {
+	snap := Snapshot{Executions: []ExecutionView{
+		{Exec: GPUKernelExec{StartNs: 0, EndNs: 10, KernelName: "kAdd"},
+			Serialized: SerializationSerialized,
+			Launch: &GPUKernelLaunch{Launch: LaunchContext{
+				Tags: map[string]string{"gpu_serialized": "false"},
+			}}},
+	}}
+
+	samples := ProjectExecutions(snap)
+	require.Len(t, samples, 1)
+	assert.Equal(t, "true", samples[0].Labels["gpu_serialized"],
+		"a tag named gpu_serialized must never override the derived disclosure")
+}