Captcha hardening: shared secret, 499 to 403, metrics polish

README.md

@@ -22,7 +22,7 @@ Designed to run as a Docker sidecar in front of your backend service (behind Ngi
 - Search engine bot verification (Google, Bing, Yandex, Apple, DuckDuckGo) via reverse DNS + IP ranges
 - Traffic filter rules (User-Agent, IP, country, platform, ASN, RPC method — AND semantics)
 - Request signing (HMAC-SHA256) — anti-replay, anti-tampering
-- Decision engine: cumulative score -> pass / CAPTCHA (499) / block (403)
+- Decision engine: cumulative score -> pass / CAPTCHA (403, `X-WAF-Action: captcha`) / block (403, `X-WAF-Action: block`); throttle (429, `X-WAF-Action: throttle`) for rate-limit
 - CAPTCHA challenge pages: [Cloudflare Turnstile](https://developers.cloudflare.com/turnstile/), [hCaptcha](https://www.hcaptcha.com/), or built-in Proof-of-Work (SHA-256)
 - Per-platform CAPTCHA policy with version gating and fallback actions
 - Captcha pass cache: HMAC cookie + IP fallback

cfg/local.toml.dist

@@ -207,13 +207,19 @@ Listen = "127.0.0.1:8081"
 # === CAPTCHA ===
 
 # [Captcha]
-# Provider = "turnstile"             # "turnstile" (Cloudflare) | "hcaptcha"
+# Provider = "turnstile"             # "turnstile" (Cloudflare) | "hcaptcha" | "pow"
+# Secret = ""                        # REQUIRED when Provider is set.
+#                                    # HMAC key for waf_pass cookie + PoW challenge signature.
+#                                    # Generate: openssl rand -hex 32
+#                                    # MUST be identical on every instance behind the LB,
+#                                    # otherwise cookies issued by one node won't validate on another.
 # SiteKey = ""
 # SecretKey = ""
 
 # hCaptcha test keys (no registration, always passes, no anti-bot protection):
 # [Captcha]
 # Provider = "hcaptcha"
+# Secret = "change-me-generate-via-openssl-rand-hex-32"
 # SiteKey = "10000000-ffff-ffff-ffff-000000000001"
 # SecretKey = "0x0000000000000000000000000000000000000000"
 

e2e/captcha_test.go

@@ -0,0 +1,194 @@
+package e2e
+
+import (
+	"net/http"
+	"strconv"
+	"strings"
+
+	"wafsrv/internal/waf"
+)
+
+// --- 07: Captcha (decide) ---
+//
+// Captcha is triggered via TrafficFilter rule "e2e-captcha-bot" (Action=captcha,
+// +5 score) at CaptchaThreshold=5. Each test uses a distinct UA so the
+// per-discriminator escalation counter (CaptchaToBlock=3) stays isolated.
+//
+// Covers commit 27f19a2 (Migrate captcha status code from 499 to 403):
+//   - captcha returns 403 (was 499)
+//   - captcha sets X-WAF-Action: captcha (disambiguates from block at same status)
+//   - captcha never sets Retry-After (semantics: solve, not wait)
+//   - block / soft_block / throttle DO set X-WAF-Action and Retry-After
+//   - Decision is set in middlewares so isBlocked metric reads from context
+
+func (s *E2ESuite) Test07_Captcha_Headers() {
+	req, _ := http.NewRequest(http.MethodGet, dataURL+"/", nil)
+	req.Header.Set("User-Agent", "E2ECaptchaBot/1.0")
+	resp, err := http.DefaultClient.Do(req)
+	s.Require().NoError(err)
+	defer resp.Body.Close()
+
+	s.Equal(http.StatusForbidden, resp.StatusCode, "captcha must return 403 (not 499)")
+	s.Equal(waf.ActionHeaderCaptcha, resp.Header.Get(waf.HeaderAction),
+		"captcha must set X-WAF-Action: captcha to disambiguate from block (both 403)")
+	s.Empty(resp.Header.Get("Retry-After"),
+		"captcha must NOT set Retry-After (solve, not wait)")
+}
+
+func (s *E2ESuite) Test07_Captcha_Body() {
+	req, _ := http.NewRequest(http.MethodGet, dataURL+"/", nil)
+	req.Header.Set("User-Agent", "E2ECaptchaBody/1.0")
+	resp, err := http.DefaultClient.Do(req)
+	s.Require().NoError(err)
+	defer resp.Body.Close()
+
+	body := s.readBody(resp)
+	s.Equal(http.StatusForbidden, resp.StatusCode)
+	// no Captcha.Provider configured in e2e.toml → falls back to plain "Captcha required" text
+	s.Contains(body, "Captcha required",
+		"unconfigured-provider captcha falls back to http.Error")
+}
+
+func (s *E2ESuite) Test07_Captcha_PerRequestIsolated() {
+	req, _ := http.NewRequest(http.MethodGet, dataURL+"/", nil)
+	req.Header.Set("User-Agent", "E2ECaptchaBot/1.0")
+
+	for i := range 2 {
+		resp, err := http.DefaultClient.Do(req)
+		s.Require().NoError(err)
+		s.Equal(http.StatusForbidden, resp.StatusCode, "request %d: captcha", i+1)
+		s.Equal(waf.ActionHeaderCaptcha, resp.Header.Get(waf.HeaderAction))
+		resp.Body.Close()
+	}
+}
+
+// Test07_Captcha_EscalationToSoftBlock walks through the captcha→soft_block
+// transition end-to-end and asserts the composite key isolates discriminators:
+//
+//	requests 1..3 with the captcha UA → 403 / X-WAF-Action: captcha
+//	on request 3 the engine reaches CaptchaToBlock=3 and arms BlockedUntil
+//	request 4 (same UA → same discriminator) → 403 / X-WAF-Action: block
+//	  with Retry-After equal to the remaining SoftBlockDuration
+//	a sibling UA from the same IP must NOT inherit the soft_block —
+//	  the (IP:Platform:fnv32(UA)) key is what makes wafsrv NAT-aware
+func (s *E2ESuite) Test07_Captcha_EscalationToSoftBlock() {
+	req, _ := http.NewRequest(http.MethodGet, dataURL+"/", nil)
+	req.Header.Set("User-Agent", "E2ECapEsc/1.0")
+
+	// CaptchaToBlock = 3 in e2e.toml — drive the counter to escalation.
+	for i := range 3 {
+		resp, err := http.DefaultClient.Do(req)
+		s.Require().NoError(err)
+		s.Equal(http.StatusForbidden, resp.StatusCode, "captcha #%d", i+1)
+		s.Equal(waf.ActionHeaderCaptcha, resp.Header.Get(waf.HeaderAction),
+			"escalation step %d must still render captcha", i+1)
+		s.Empty(resp.Header.Get("Retry-After"),
+			"captcha must not leak Retry-After mid-escalation")
+		resp.Body.Close()
+	}
+
+	// Next request with the same discriminator must be soft-blocked.
+	resp, err := http.DefaultClient.Do(req)
+	s.Require().NoError(err)
+
+	s.Equal(http.StatusForbidden, resp.StatusCode, "post-escalation must 403")
+	s.Equal(waf.ActionHeaderBlock, resp.Header.Get(waf.HeaderAction),
+		"soft_block must set X-WAF-Action: block (clients can't tell from status)")
+
+	retryAfter := resp.Header.Get("Retry-After")
+	s.NotEmpty(retryAfter, "soft_block must surface remaining TTL via Retry-After")
+
+	secs, err := strconv.Atoi(retryAfter)
+	s.Require().NoError(err, "Retry-After must be integer seconds")
+	s.Positive(secs, "remaining TTL must be positive")
+	// SoftBlockDuration=1m → cap at 60 (rounding may inflate by 1)
+	s.LessOrEqual(secs, 61, "Retry-After must not exceed SoftBlockDuration")
+	resp.Body.Close()
+
+	// Composite-key isolation: a different UA from the same IP must pass.
+	// Without the discriminator in the soft-block key, the whole NAT'd IP
+	// would lock out behind one bad client.
+	other, _ := http.NewRequest(http.MethodGet, dataURL+"/", nil)
+	other.Header.Set("User-Agent", "E2EUnrelatedClient/1.0") // not matched by captcha rule
+
+	resp, err = http.DefaultClient.Do(other)
+	s.Require().NoError(err)
+	defer resp.Body.Close()
+	s.Equal(http.StatusOK, resp.StatusCode,
+		"sibling UA on same IP must not inherit soft_block (composite key isolates discriminators)")
+}
+
+func (s *E2ESuite) Test07_Captcha_ZZ_Metrics() {
+	// Hit captcha once with a fresh UA so the test is self-contained even when
+	// previous test ordering changes (escalation test may have already moved
+	// captcha counter forward but metric value is what we assert).
+	req, _ := http.NewRequest(http.MethodGet, dataURL+"/", nil)
+	req.Header.Set("User-Agent", "E2ECaptchaMetrics/1.0")
+	resp, err := http.DefaultClient.Do(req)
+	s.Require().NoError(err)
+	resp.Body.Close()
+
+	body := s.getMetrics()
+	s.Contains(body, `wafsrv_decision_total{action="captcha"`,
+		"decision metric must expose captcha action label")
+}
+
+// --- 03b: Throttle response shape ---
+
+// Test03_RateLimit_ThrottleHeader extends Test03_RateLimit_PerMethod by asserting
+// the X-WAF-Action: throttle header introduced alongside the captcha→403 migration.
+// Uses pgd.diff (sibling rule, fresh counter) so the assertion does not depend on
+// Test03_RateLimit_PerMethod's execution order.
+func (s *E2ESuite) Test03_RateLimit_ThrottleHeader() {
+	for range 5 {
+		resp := s.postRPC(dataURL+"/rpc/", "pgd.diff", "{}")
+		resp.Body.Close()
+	}
+
+	resp := s.postRPC(dataURL+"/rpc/", "pgd.diff", "{}")
+	defer resp.Body.Close()
+
+	s.Equal(http.StatusTooManyRequests, resp.StatusCode)
+	s.Equal(waf.ActionHeaderThrottle, resp.Header.Get(waf.HeaderAction),
+		"429 must set X-WAF-Action: throttle to disambiguate from block/captcha 403")
+	s.NotEmpty(resp.Header.Get("Retry-After"),
+		"throttle must include Retry-After (RateLimit.RetryAfter, default 60s)")
+}
+
+// --- 04b: Block response shape ---
+
+// Test04_IP_Block_HeaderAndRetryAfter extends Test04_IP_BlockAndUnblock by
+// asserting the new X-WAF-Action: block header (added when both captcha and
+// block returned 403, status alone became ambiguous) and that a TTL'd
+// blacklist exposes the remaining duration via Retry-After.
+func (s *E2ESuite) Test04_IP_Block_HeaderAndRetryAfter() {
+	clientIP := s.detectClientIP()
+
+	result := s.mgmtRPC("block.add",
+		`{"blockType":"ip","value":"`+clientIP+`","reason":"e2e-headers","duration":"30s"}`)
+	s.Contains(result, `"result":true`)
+
+	resp := s.get(dataURL + "/")
+	defer resp.Body.Close()
+	defer s.mgmtRPC("block.remove", `{"blockType":"ip","value":"`+clientIP+`"}`)
+
+	s.Equal(http.StatusForbidden, resp.StatusCode)
+	s.Equal(waf.ActionHeaderBlock, resp.Header.Get(waf.HeaderAction),
+		"static/runtime block must set X-WAF-Action: block")
+
+	retryAfter := resp.Header.Get("Retry-After")
+	s.NotEmpty(retryAfter,
+		"TTL'd blacklist entry must surface remaining duration via Retry-After")
+
+	// duration "30s" was just set; allow a couple of seconds of drift
+	secs, err := strconv.Atoi(retryAfter)
+	s.Require().NoError(err)
+	s.Positive(secs)
+	s.LessOrEqual(secs, 31)
+
+	// body should be the block error text (RenderBlock isn't used here — ip middleware
+	// uses http.Error). Sanity check we didn't somehow render the captcha body.
+	body := s.readBody(resp)
+	s.NotContains(strings.ToLower(body), "captcha required",
+		"block path must not render captcha body")
+}

e2e/cfg/e2e-aerospike.toml

@@ -47,6 +47,11 @@ Name = "e2e-log-bot"
 Action = "log"
 UAPrefix = ["E2ELogBot/"]
 
+[[TrafficFilter.Rules]]
+Name = "e2e-captcha-bot"
+Action = "captcha"
+UAPrefix = ["E2ECaptchaBot/", "E2ECaptchaBody/", "E2ECaptchaMetrics/", "E2ECapEsc/"]
+
 [Signing]
 Enabled = true
 Mode = "detection"

e2e/cfg/e2e.toml

@@ -53,6 +53,11 @@ AttackOnly = true
 Action = "block"
 UAPrefix = ["E2EAttackBot/"]
 
+[[TrafficFilter.Rules]]
+Name = "e2e-captcha-bot"
+Action = "captcha"
+UAPrefix = ["E2ECaptchaBot/", "E2ECaptchaBody/", "E2ECaptchaMetrics/", "E2ECapEsc/"]
+
 [Signing]
 Enabled = true
 Mode = "detection"

internal/app/app.go

@@ -112,6 +112,7 @@ func New(appName string, sl embedlog.Logger, cfg Config) (*App, error) {
 
 	// AttackService must exist before WAF wire-up: filter and decide read it via waf.AttackState DI.
 	a.attackSvc = dashboard.NewAttackService()
+	a.metrics.registerAttackModeGauge(a.attackSvc.IsEnabled)
 
 	if err := a.initWAF(); err != nil {
 		return nil, err
@@ -300,6 +301,7 @@ func (a *App) initRPCInspector() {
 func (a *App) initDecision() {
 	// captcha cache
 	a.captchaCache = challenge.NewCache(challenge.CacheConfig{
+		Secret:     []byte(a.cfg.Captcha.Secret),
 		CookieName: a.cfg.Captcha.CookieName,
 		CookieTTL:  parseDuration(a.cfg.Captcha.CookieTTL, 30*time.Minute),
 		IPCacheTTL: parseDuration(a.cfg.Captcha.IPCacheTTL, 30*time.Minute),
@@ -369,6 +371,7 @@ func (a *App) initDecision() {
 		CaptchaToBlock:       a.cfg.Decision.CaptchaToBlock,
 		CaptchaToBlockWindow: parseDuration(a.cfg.Decision.CaptchaToBlockWindow, 10*time.Minute),
 		SoftBlockDuration:    parseDuration(a.cfg.Decision.SoftBlockDuration, 10*time.Minute),
+		BlockRetryAfter:      parseDuration(a.cfg.Decision.BlockRetryAfter, 0),
 		CaptchaProvider:      a.cfg.Captcha.Provider,
 		CaptchaSiteKey:       a.cfg.Captcha.SiteKey,
 		CaptchaSecretKey:     a.cfg.Captcha.SecretKey,

internal/app/config.go

@@ -3,6 +3,7 @@ package app
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"net/http"
 	"net/url"
 	"time"
@@ -129,11 +130,12 @@ func (c *TrafficFilterConfig) TrafficFilterEnabled() bool {
 type DecisionConfig struct {
 	CaptchaThreshold     float64 // default 5
 	BlockThreshold       float64 // default 8
-	CaptchaStatusCode    int     // default 499
+	CaptchaStatusCode    int     // default 403
 	BlockStatusCode      int     // default 403
 	CaptchaToBlock       int     // default 3
 	CaptchaToBlockWindow string  // default "10m"
 	SoftBlockDuration    string  // default "10m"
+	BlockRetryAfter      string  // duration, default "" = no Retry-After header on block responses
 	CaptchaFallback      string  // default "block"; "pass" | "block" | "log"
 	Platforms            []PlatformCaptchaConfig
 }
@@ -148,7 +150,12 @@ type PlatformCaptchaConfig struct {
 
 // CaptchaConfig configures the CAPTCHA provider.
 type CaptchaConfig struct {
-	Provider     string // "turnstile" | "hcaptcha" | "pow"
+	Provider string // "turnstile" | "hcaptcha" | "pow"
+	// Secret is the HMAC key used to sign waf_pass cookies and PoW challenges.
+	// Required when Provider != "". MUST be identical on every instance behind
+	// a load balancer, otherwise cookies issued by one instance won't validate
+	// on another. Generate via `openssl rand -hex 32` (>= 32 bytes).
+	Secret       string
 	SiteKey      string
 	SecretKey    string
 	CookieName   string // default "waf_pass"
@@ -210,6 +217,7 @@ type RateLimitConfig struct {
 	PerIP       string             // "100/min"
 	Action      string             // "block" | "throttle"
 	MaxCounters int                // LRU eviction
+	RetryAfter  string             // duration, default "60s"; sent as Retry-After on 429
 	Rules       []RateLimitRule    // RPC method matching
 	URLRules    []URLRateLimitRule // HTTP path/method/host matching
 }
@@ -501,6 +509,8 @@ func (c *Config) Validate() error {
 		{c.Captcha.IPCacheTTL, "Captcha.IPCacheTTL"},
 		{c.Decision.CaptchaToBlockWindow, "Decision.CaptchaToBlockWindow"},
 		{c.Decision.SoftBlockDuration, "Decision.SoftBlockDuration"},
+		{c.Decision.BlockRetryAfter, "Decision.BlockRetryAfter"},
+		{c.RateLimit.RetryAfter, "RateLimit.RetryAfter"},
 		{c.IP.Reputation.UpdateInterval, "IP.Reputation.UpdateInterval"},
 	}
 
@@ -528,9 +538,20 @@ func (c *Config) Validate() error {
 		return err
 	}
 
+	c.warnMultiInstance()
+
 	return nil
 }
 
+// warnMultiInstance flags the in-memory storage backend as a multi-instance hazard.
+func (c *Config) warnMultiInstance() {
+	if c.Captcha.Provider != "" && c.Storage.Backend == StorageMemory {
+		slog.Default().Warn("captcha enabled with memory storage backend",
+			"impact", "IP fallback, escalation counters and signing nonces won't be shared between instances",
+			"fix", `set Storage.Backend = "aerospike" for multi-instance deployments`)
+	}
+}
+
 // adaptiveBoostValidate ensures Adaptive.AutoAttack.ScoreBoost cannot push
 // every request straight into the block bucket while attack mode is on —
 // boost must leave at least the captcha gap below the captcha threshold.
@@ -584,11 +605,23 @@ func (c *RateLimitConfig) validate() error {
 
 var validCaptchaProviders = map[string]bool{"turnstile": true, "hcaptcha": true, "pow": true, "": true}
 
+const minCaptchaSecretBytes = 32
+
 func (c *CaptchaConfig) validate() error {
 	if !validCaptchaProviders[c.Provider] {
 		return fmt.Errorf("config: Captcha.Provider must be turnstile, hcaptcha or pow, got %q", c.Provider)
 	}
 
+	if c.Provider != "" && c.Secret == "" {
+		return errors.New("config: Captcha.Secret is required when Captcha.Provider is set " +
+			"(generate via `openssl rand -hex 32`, must be identical on all instances behind a load balancer)")
+	}
+
+	if c.Provider != "" && len(c.Secret) < minCaptchaSecretBytes {
+		return fmt.Errorf("config: Captcha.Secret must be at least %d bytes long (got %d)",
+			minCaptchaSecretBytes, len(c.Secret))
+	}
+
 	if err := validateDuration(c.PoW.Timeout, "Captcha.PoW.Timeout"); err != nil {
 		return err
 	}
@@ -779,6 +812,7 @@ func (c *Config) LimiterConfig() (limit.Config, error) {
 		PerIP:       perIP,
 		Action:      c.RateLimit.Action,
 		MaxCounters: c.RateLimit.MaxCounters,
+		RetryAfter:  parseDuration(c.RateLimit.RetryAfter, 60*time.Second),
 		Rules:       rules,
 		URLRules:    urlRules,
 	}, nil
@@ -1030,6 +1064,10 @@ func applyWAFDefaults(c *Config) {
 		c.RateLimit.MaxCounters = 100000
 	}
 
+	if c.RateLimit.RetryAfter == "" {
+		c.RateLimit.RetryAfter = "60s"
+	}
+
 	if c.IP.Whitelist.BotVerify.CacheSize == 0 {
 		c.IP.Whitelist.BotVerify.CacheSize = 10000
 	}
@@ -1075,7 +1113,7 @@ func applyDecisionDefaults(c *Config) {
 	}
 
 	if c.Decision.CaptchaStatusCode == 0 {
-		c.Decision.CaptchaStatusCode = 499
+		c.Decision.CaptchaStatusCode = http.StatusForbidden
 	}
 
 	if c.Decision.BlockStatusCode == 0 {