diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 96eec2f1f..7c1303dd4 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -13,6 +13,7 @@ fn make_config() -> HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index cdee3b222..e74ef4150 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -155,7 +155,9 @@ fn validate_enabled_integrations( validate_integration::(settings, "sourcepoint")?; validate_integration::(settings, "osano")?; validate_integration::(settings, "google_tag_manager")?; - validate_integration::(settings, "datadome")?; + if let Some(config) = settings.integration_config::("datadome")? { + crate::integrations::datadome::DataDomeIntegration::validate_config_for_startup(config)?; + } validate_integration::(settings, "gpt")?; validate_integration::(settings, "gpt_diagnostics")?; @@ -404,6 +406,44 @@ password = "production-admin-password-32-bytes" ); } + #[test] + fn deploy_validation_rejects_invalid_datadome_test_bypass() { + for (enable_protection, store, name, expected_message) in [ + ( + false, + "ts_secrets", + "datadome_test_bypass", + "requires enable_protection", + ), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), + ] { + let mut settings = valid_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "enable_protection": enable_protection, + "protection_test_bypass": { + "enabled": true, + "credential_secret_store": store, + "credential_secret_name": name, + }, + }), + ) + .expect("should insert DataDome config"); + + let err = validate_settings_for_deploy(&settings) + .expect_err("should reject invalid DataDome test bypass"); + assert!( + format!("{err:?}").contains(expected_message), + "error should mention the invalid bypass setting: {err:?}" + ); + } + } + #[test] fn validate_trait_reports_deploy_errors() { let mut settings = valid_settings(); diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 889234b56..4c827ace0 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -13,6 +13,7 @@ use lol_html::{ text, }; +use crate::integrations::datadome::{DATADOME_INTEGRATION_ID, DataDomeClientTagSuppressed}; use crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision; use crate::integrations::{ AttributeRewriteOutcome, IntegrationAttributeContext, IntegrationDocumentState, @@ -175,6 +176,8 @@ pub struct HtmlProcessorConfig { pub max_buffered_body_bytes: usize, /// Request-scoped conditional diagnostics delivery decision. pub gpt_diagnostics: Option, + /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. + pub suppress_datadome_client_side_tag: bool, } impl HtmlProcessorConfig { @@ -196,6 +199,7 @@ impl HtmlProcessorConfig { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -223,6 +227,13 @@ impl HtmlProcessorConfig { self.gpt_diagnostics = decision; self } + + /// Attach the request-scoped `DataDome` client-tag suppression decision. + #[must_use] + pub fn with_datadome_client_tag_suppression(mut self, suppress: bool) -> Self { + self.suppress_datadome_client_side_tag = suppress; + self + } } /// Create an HTML processor with URL replacement and integration hooks. @@ -235,6 +246,9 @@ impl HtmlProcessorConfig { pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcessor { let post_processors = config.integrations.html_post_processors(); let document_state = IntegrationDocumentState::default(); + if config.suppress_datadome_client_side_tag { + document_state.get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); + } // Simplified URL patterns structure - stores only core data and generates variants on-demand struct UrlPatterns { @@ -692,6 +706,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -950,6 +965,46 @@ mod tests { assert_eq!(config.request_scheme, "https"); } + #[test] + fn suppressed_datadome_tag_is_not_injected_into_processed_html() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &json!({ + "enabled": true, + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + "origin.example.com", + "test.example.com", + "https", + ) + .with_datadome_client_tag_suppression(true); + let mut processor = create_html_processor(config); + + let output = processor + .process_chunk(b"content", true) + .expect("should process HTML"); + let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + + assert!( + !html.contains("window.ddjskey"), + "should omit the DataDome client configuration" + ); + assert!( + !html.contains("/integrations/datadome/tags.js"), + "should omit the DataDome client tag URL" + ); + } + #[test] fn test_real_publisher_html() { // Test with publisher HTML from test_publisher.html @@ -1539,6 +1594,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1613,6 +1669,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1649,6 +1706,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); // Malformed HTML with two elements (common in CMS template pages) @@ -1684,6 +1742,7 @@ mod tests { ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1737,6 +1796,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor @@ -1764,6 +1824,7 @@ mod tests { ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut processor = create_html_processor(config); let output = processor diff --git a/crates/trusted-server-core/src/integrations/datadome.rs b/crates/trusted-server-core/src/integrations/datadome.rs index a46f626a6..2486c16be 100644 --- a/crates/trusted-server-core/src/integrations/datadome.rs +++ b/crates/trusted-server-core/src/integrations/datadome.rs @@ -88,7 +88,13 @@ pub use protection_scope::{ use protection_scope::ProtectionScope; -pub(super) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(crate) const DATADOME_INTEGRATION_ID: &str = "datadome"; +pub(crate) const HEADER_DATADOME_TEST_BYPASS: &str = "x-ts-datadome-bypass"; + +/// Request marker indicating that Trusted Server should omit its automatic +/// `DataDome` client-side tag for the current response. +#[derive(Debug, Clone, Copy)] +pub(crate) struct DataDomeClientTagSuppressed; /// Regex pattern for matching and rewriting `DataDome` URLs in script content. /// @@ -112,6 +118,28 @@ static DATADOME_URL_PATTERN: LazyLock = LazyLock::new(|| { .expect("DataDome URL rewrite regex should compile") }); +/// Temporary static-header bypass for server-side `DataDome` protection. +/// +/// This is intended only for an access-controlled staging environment. A +/// matching `x-ts-datadome-bypass` header bypasses the server-side Protection +/// API and is removed before the publisher origin receives the request. The +/// credential itself is loaded from the Secret Store at runtime. +#[derive(Debug, Default, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ProtectionTestBypassConfig { + /// Enables the bypass. Defaults to disabled when the section is present. + #[serde(default)] + pub enabled: bool, + + /// Secret Store containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_store")] + pub credential_secret_store: String, + + /// Secret name containing the temporary bypass credential. + #[serde(default = "default_protection_test_bypass_secret_name")] + pub credential_secret_name: String, +} + /// Configuration for `DataDome` integration. #[derive(Debug, Clone, Deserialize, Validate)] #[serde(deny_unknown_fields)] @@ -194,6 +222,10 @@ pub struct DataDomeConfig { )] pub protection_exclusion_rules: Vec, + /// Temporary static-header bypass for access-controlled staging tests. + #[serde(default)] + pub protection_test_bypass: Option, + /// Reserved flag for future GraphQL payload extraction. #[serde(default)] pub enable_graphql_support: bool, @@ -247,6 +279,14 @@ fn default_server_side_key_secret_name() -> String { "datadome_server_side_key".to_string() } +fn default_protection_test_bypass_secret_store() -> String { + "ts_secrets".to_string() +} + +fn default_protection_test_bypass_secret_name() -> String { + "datadome_test_bypass".to_string() +} + fn default_timeout_ms() -> u32 { 1500 } @@ -324,6 +364,7 @@ impl Default for DataDomeConfig { protection_excluded_ip_cidr_sources: Vec::new(), protection_ip_list_cache_ttl_seconds: default_protection_ip_list_cache_ttl_seconds(), protection_exclusion_rules: default_protection_exclusion_rules(), + protection_test_bypass: None, enable_graphql_support: false, client_side_key: String::new(), inject_client_side_tag: default_inject_client_side_tag(), @@ -357,6 +398,10 @@ impl DataDomeIntegration { config.server_side_key_secret_name = config.server_side_key_secret_name.trim().to_string(); config.protection_api_origin = config.protection_api_origin.trim().to_string(); config.client_side_tag_url = config.client_side_tag_url.trim().to_string(); + if let Some(bypass) = &mut config.protection_test_bypass { + bypass.credential_secret_store = bypass.credential_secret_store.trim().to_string(); + bypass.credential_secret_name = bypass.credential_secret_name.trim().to_string(); + } if config.enable_protection { if config.server_side_key_secret_store.is_empty() @@ -368,6 +413,7 @@ impl DataDomeIntegration { } Self::validate_protection_api_origin(&config.protection_api_origin)?; } + Self::validate_protection_test_bypass(&config)?; if config.inject_client_side_tag { Self::validate_client_side_tag_url(&config.client_side_tag_url)?; @@ -417,6 +463,37 @@ impl DataDomeIntegration { Ok(()) } + pub(crate) fn validate_config_for_startup( + config: DataDomeConfig, + ) -> Result<(), Report> { + Self::try_new(config).map(|_| ()) + } + + fn validate_protection_test_bypass( + config: &DataDomeConfig, + ) -> Result<(), Report> { + let Some(bypass) = config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return Ok(()); + }; + + if !config.enable_protection { + return Err(Report::new(Self::error( + "protection_test_bypass requires enable_protection to be true", + ))); + } + if bypass.credential_secret_store.is_empty() || bypass.credential_secret_name.is_empty() { + return Err(Report::new(Self::error( + "protection_test_bypass credential_secret_store and credential_secret_name must not be empty when enabled", + ))); + } + + Ok(()) + } + fn validate_client_side_tag_url(tag_url: &str) -> Result<(), Report> { if tag_url.starts_with('/') && !tag_url.starts_with("//") { if tag_url.chars().any(is_unsafe_client_side_tag_path_char) { @@ -765,7 +842,15 @@ impl IntegrationHeadInjector for DataDomeIntegration { DATADOME_INTEGRATION_ID } - fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + fn head_inserts(&self, ctx: &IntegrationHtmlContext<'_>) -> Vec { + if ctx + .document_state + .get::(DATADOME_INTEGRATION_ID) + .is_some() + { + return Vec::new(); + } + if !self.config.inject_client_side_tag || self.config.client_side_key.trim().is_empty() { return Vec::new(); } @@ -840,13 +925,25 @@ fn build( return Ok(None); }; + let integration = DataDomeIntegration::try_new(config)?; + let protection_test_bypass = integration + .config + .protection_test_bypass + .as_ref() + .is_some_and(|bypass| bypass.enabled); log::info!( - "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {})", - config.sdk_origin, - config.rewrite_sdk + "[datadome] Registering integration (sdk_origin: {}, rewrite_sdk: {}, enable_protection: {}, protection_test_bypass: {})", + integration.config.sdk_origin, + integration.config.rewrite_sdk, + integration.config.enable_protection, + if protection_test_bypass { + "enabled" + } else { + "disabled" + }, ); - Ok(Some(DataDomeIntegration::try_new(config)?)) + Ok(Some(integration)) } /// Register the `DataDome` integration with Trusted Server. @@ -1084,6 +1181,70 @@ mod tests { config.server_side_key_secret_name, "datadome_server_side_key" ); + assert!( + config.protection_test_bypass.is_none(), + "the temporary test bypass should be disabled by default" + ); + } + + #[test] + fn protection_test_bypass_deserializes_nested_configuration() { + let config: DataDomeConfig = toml::from_str( + r#" + enabled = true + enable_protection = true + + [protection_test_bypass] + enabled = true + credential_secret_store = "ts_secrets" + credential_secret_name = "datadome_test_bypass" + "#, + ) + .expect("should deserialize DataDome test bypass configuration"); + let bypass = config + .protection_test_bypass + .expect("should deserialize the nested test bypass configuration"); + + assert!(bypass.enabled, "should retain the enabled flag"); + assert_eq!( + bypass.credential_secret_store, "ts_secrets", + "should retain the configured credential Secret Store" + ); + assert_eq!( + bypass.credential_secret_name, "datadome_test_bypass", + "should retain the configured credential secret name" + ); + } + + #[test] + fn protection_test_bypass_requires_protection_and_secret_references() { + for (enable_protection, store, name, expected_message) in [ + ( + false, + "ts_secrets", + "datadome_test_bypass", + "requires enable_protection", + ), + (true, "", "datadome_test_bypass", "credential_secret_store"), + (true, "ts_secrets", "", "credential_secret_name"), + ] { + let mut config = test_config(); + config.enable_protection = enable_protection; + config.protection_test_bypass = Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: store.to_string(), + credential_secret_name: name.to_string(), + }); + + let err = match DataDomeIntegration::try_new(config) { + Ok(_) => panic!("should reject invalid protection test bypass configuration"), + Err(err) => err, + }; + assert!( + format!("{err:?}").contains(expected_message), + "should explain the invalid protection test bypass configuration" + ); + } } #[test] @@ -1248,6 +1409,20 @@ mod tests { #[test] fn head_injector_omits_client_side_tag_when_disabled_or_blank() { + let mut suppressed = test_config(); + suppressed.client_side_key = "test-client-key".to_string(); + let suppressed_integration = DataDomeIntegration::new(suppressed); + let suppressed_state = crate::integrations::IntegrationDocumentState::default(); + suppressed_state + .get_or_insert_with(DATADOME_INTEGRATION_ID, || DataDomeClientTagSuppressed); + let suppressed_ctx = html_context_for_tests(&suppressed_state); + assert!( + suppressed_integration + .head_inserts(&suppressed_ctx) + .is_empty(), + "should omit the tag when the request is IP-excluded" + ); + let mut blank_key = test_config(); blank_key.client_side_key = " ".to_string(); let integration = DataDomeIntegration::new(blank_key); diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index c2759a864..bc06214ff 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -4,6 +4,8 @@ use edgezero_core::body::Body as EdgeBody; use edgezero_core::http::{HeaderMap, HeaderName, request_builder}; use error_stack::{Report, ResultExt}; use http::{Method, Request, Response, StatusCode, header}; +use sha2::{Digest as _, Sha256}; +use subtle::ConstantTimeEq as _; use url::Url; use crate::error::TrustedServerError; @@ -34,9 +36,20 @@ enum ProtectionRequestError { impl DataDomeIntegration { pub(super) async fn filter_protection_request( &self, - input: RequestFilterInput<'_>, + mut input: RequestFilterInput<'_>, ) -> RequestFilterDecision { - if !self.config.enable_protection || !self.is_request_protected(&input) { + let test_bypass_matched = + self.take_protection_test_bypass_header(input.request, input.services); + if test_bypass_matched { + input + .request + .extensions_mut() + .insert(super::DataDomeClientTagSuppressed); + log_protection_test_bypass(&input); + return RequestFilterDecision::Continue(RequestFilterEffects::default()); + } + + if !self.config.enable_protection || !self.is_request_protected(&mut input) { return RequestFilterDecision::Continue(RequestFilterEffects::default()); } @@ -98,11 +111,17 @@ impl DataDomeIntegration { .change_context(Self::error("Failed to call DataDome Protection API")) .map_err(ProtectionRequestError::Runtime)?; - Ok(self.classify_protection_response(platform_response.response, input.request.method())) + let status = platform_response.response.status(); + let datadome_status = datadome_response_status(platform_response.response.headers()); + let decision = + self.classify_protection_response(platform_response.response, input.request.method()); + log_protection_result(&input, status, datadome_status, &decision); + + Ok(decision) } - fn is_request_protected(&self, input: &RequestFilterInput<'_>) -> bool { - let req = input.request; + fn is_request_protected(&self, input: &mut RequestFilterInput<'_>) -> bool { + let req = &*input.request; if req.method() == Method::OPTIONS { return false; } @@ -126,7 +145,14 @@ impl DataDomeIntegration { match self.protection_scope.evaluate(&facts, input.services) { ProtectionScopeDecision::Protect => {} ProtectionScopeDecision::Skip { rule_id, reason } => { - log::debug!("[datadome] Skipping Protection API for rule {rule_id} ({reason})"); + let client_tag_omitted = is_ip_exclusion_reason(reason); + if client_tag_omitted { + input + .request + .extensions_mut() + .insert(super::DataDomeClientTagSuppressed); + } + log_protection_skip(input, &rule_id, reason); return false; } } @@ -134,6 +160,48 @@ impl DataDomeIntegration { true } + fn take_protection_test_bypass_header( + &self, + req: &mut Request, + services: &RuntimeServices, + ) -> bool { + let Some(bypass) = self + .config + .protection_test_bypass + .as_ref() + .filter(|bypass| bypass.enabled) + else { + return false; + }; + let Some(value) = req.headers_mut().remove(super::HEADER_DATADOME_TEST_BYPASS) else { + return false; + }; + + let store_name = StoreName::from(bypass.credential_secret_store.as_str()); + let credential = match services + .secret_store() + .get_string(&store_name, &bypass.credential_secret_name) + { + Ok(credential) if !credential.is_empty() => credential, + Ok(_) => { + log::warn!( + "[datadome] DataDome test bypass credential is empty; ignoring bypass header" + ); + return false; + } + Err(err) => { + log::warn!( + "[datadome] Failed to load DataDome test bypass credential; ignoring bypass header: {err:?}" + ); + return false; + } + }; + + let actual = Sha256::digest(value.as_bytes()); + let expected = Sha256::digest(credential.as_bytes()); + bool::from(actual.ct_eq(&expected)) + } + fn protection_validate_url(&self) -> String { format!( "{}{}", @@ -194,7 +262,7 @@ impl DataDomeIntegration { input: &RequestFilterInput<'_>, server_side_key: &Redacted, ) -> ProtectionPayload { - let req = input.request; + let req = &*input.request; let client_info = input.services.client_info(); let mut fields = Vec::new(); let header_client_id = header_value(req, HEADER_DATADOME_CLIENT_ID); @@ -413,6 +481,65 @@ impl DataDomeIntegration { } } +fn is_ip_exclusion_reason(reason: &str) -> bool { + matches!( + reason, + "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source" + ) +} + +fn log_protection_test_bypass(input: &RequestFilterInput<'_>) { + log::info!( + "[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method={}", + input.request.method(), + ); +} + +fn log_protection_skip(input: &RequestFilterInput<'_>, rule_id: &str, reason: &str) { + if is_ip_exclusion_reason(reason) { + log::info!( + "[datadome] protection decision=skipped rule={} reason={} client_tag=omitted method={}", + rule_id, + reason, + input.request.method(), + ); + } else { + log::debug!( + "[datadome] protection decision=skipped rule={} reason={} method={}", + rule_id, + reason, + input.request.method(), + ); + } +} + +fn log_protection_result( + input: &RequestFilterInput<'_>, + status: StatusCode, + datadome_status: Option, + decision: &RequestFilterDecision, +) { + let method = input.request.method(); + + match decision { + RequestFilterDecision::Respond { .. } => log::info!( + "[datadome] protection decision=blocked status={} method={} route=short_circuit", + status.as_u16(), + method, + ), + RequestFilterDecision::Continue(_) + if status == StatusCode::OK && datadome_status == Some(status.as_u16()) => + { + log::info!( + "[datadome] protection decision=allowed status={} method={} route=continue", + status.as_u16(), + method, + ); + } + RequestFilterDecision::Continue(_) => {} + } +} + struct ProtectionPayload { fields: Vec<(String, String)>, uses_header_client_id: bool, @@ -645,11 +772,18 @@ fn truncate_utf8(value: &str, limit: i32) -> String { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; - use crate::integrations::datadome::DataDomeConfig; + use crate::integrations::datadome::{ + DataDomeConfig, ProtectionExclusionRuleConfig, ProtectionMatcherConfig, + ProtectionTestBypassConfig, + }; + use crate::platform::GeoInfo; use crate::platform::test_support::{ - HashMapSecretStore, NoopConfigStore, NoopSecretStore, build_services_with_config_and_secret, + HashMapConfigStore, HashMapSecretStore, NoopConfigStore, NoopSecretStore, StubHttpClient, + build_services_with_config_and_secret, build_services_with_config_and_secret_and_client_ip, + build_services_with_secret_and_http_client, noop_services_with_client_ip, }; use crate::settings::Settings; @@ -664,6 +798,404 @@ mod tests { DataDomeIntegration::try_new(config).expect("should create integration") } + fn request_for_filter() -> Request { + request_builder() + .method(Method::GET.as_str()) + .uri("https://publisher.example/page") + .body(EdgeBody::empty()) + .expect("should build filter request") + } + + fn filter_marks_request( + config: DataDomeConfig, + services: &RuntimeServices, + ) -> Request { + filter_marks_request_with_geo(config, services, None) + } + + fn filter_marks_request_with_geo( + config: DataDomeConfig, + services: &RuntimeServices, + geo_info: Option<&GeoInfo>, + ) -> Request { + let integration = + DataDomeIntegration::try_new(config).expect("should create DataDome integration"); + let settings = Settings::default(); + let mut request = request_for_filter(); + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services, + request: &mut request, + geo_info, + is_integration_route: false, + }, + )); + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an excluded request should continue without a Protection API response" + ); + request + } + + fn has_client_tag_suppression_marker(request: &Request) -> bool { + request + .extensions() + .get::() + .is_some() + } + + #[test] + fn protection_test_bypass_skips_api_suppresses_tag_and_strips_header() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue without a challenge" + ); + assert!( + has_client_tag_suppression_marker(&request), + "the bypass should suppress the automatic DataDome client tag" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "the bypass credential must not reach the publisher origin" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + + #[test] + fn protection_test_bypass_wins_over_other_exclusions() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "staging-page-exclusion".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }], + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("temporary-test-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "a matching test credential should continue" + ); + assert!( + has_client_tag_suppression_marker(&request), + "a matching test credential should suppress the tag even on an excluded path" + ); + assert!( + http_client.recorded_backend_names().is_empty(), + "a matching test credential must not call the Protection API" + ); + } + + #[test] + fn protection_test_bypass_strips_invalid_credential_without_bypassing() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_test_bypass: Some(ProtectionTestBypassConfig { + enabled: true, + credential_secret_store: "ts_secrets".to_string(), + credential_secret_name: "datadome_test_bypass".to_string(), + }), + ..DataDomeConfig::default() + }; + let integration = DataDomeIntegration::try_new(config).expect("should create integration"); + let mut secrets = HashMap::new(); + secrets.insert( + "datadome_server_side_key".to_string(), + b"server-side-key".to_vec(), + ); + secrets.insert( + "datadome_test_bypass".to_string(), + b"temporary-test-credential".to_vec(), + ); + let http_client = Arc::new(StubHttpClient::new()); + http_client.push_response_with_headers( + 200, + Vec::new(), + vec![(HEADER_DATADOME_RESPONSE, "200")], + ); + let services = build_services_with_secret_and_http_client( + HashMapSecretStore::new(secrets), + http_client.clone(), + ); + let settings = Settings::default(); + let mut request = request_for_filter(); + request.headers_mut().insert( + super::super::HEADER_DATADOME_TEST_BYPASS, + edgezero_core::http::HeaderValue::from_static("wrong-credential"), + ); + + let decision = futures::executor::block_on(integration.filter_protection_request( + RequestFilterInput { + settings: &settings, + services: &services, + request: &mut request, + geo_info: None, + is_integration_route: false, + }, + )); + + assert!( + matches!(decision, RequestFilterDecision::Continue(_)), + "an allowed Protection API response should continue" + ); + assert!( + !has_client_tag_suppression_marker(&request), + "a non-matching credential must not suppress the DataDome client tag" + ); + assert!( + request + .headers() + .get(super::super::HEADER_DATADOME_TEST_BYPASS) + .is_none(), + "an invalid bypass credential must not reach the publisher origin" + ); + assert_eq!( + http_client.recorded_backend_names().len(), + 1, + "a non-matching credential must still call the Protection API" + ); + } + + #[test] + fn ip_exclusions_mark_requests_for_client_tag_suppression() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let mut inline = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], + ..DataDomeConfig::default() + }; + let inline_request = + filter_marks_request(inline.clone(), &noop_services_with_client_ip(ip)); + assert!( + has_client_tag_suppression_marker(&inline_request), + "inline IP exclusions should mark the request" + ); + + inline.protection_excluded_ip_cidrs.clear(); + inline.protection_excluded_ip_cidr_sources = + vec![super::super::ProtectionIpCidrSourceConfig { + config_store: "datadome-test-source".to_string(), + key: "inline-source".to_string(), + }]; + let mut source_values = HashMap::new(); + source_values.insert("inline-source".to_string(), "192.0.2.0/24".to_string()); + let source_services = build_services_with_config_and_secret_and_client_ip( + HashMapConfigStore::new(source_values), + NoopSecretStore, + ip, + ); + let source_request = filter_marks_request(inline, &source_services); + assert!( + has_client_tag_suppression_marker(&source_request), + "Config Store IP exclusions should mark the request" + ); + + let structured_ip = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "structured-ip".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidr { + cidrs: vec!["192.0.2.0/24".to_string()], + }, + }], + ..DataDomeConfig::default() + }; + let structured_request = + filter_marks_request(structured_ip, &noop_services_with_client_ip(ip)); + assert!( + has_client_tag_suppression_marker(&structured_request), + "structured IP exclusions should mark the request" + ); + + let structured_source = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "structured-ip-source".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::IpCidrSource { + config_store: "datadome-test-source".to_string(), + key: "structured-source".to_string(), + }, + }], + ..DataDomeConfig::default() + }; + let mut structured_values = HashMap::new(); + structured_values.insert("structured-source".to_string(), "192.0.2.0/24".to_string()); + let structured_services = build_services_with_config_and_secret_and_client_ip( + HashMapConfigStore::new(structured_values), + NoopSecretStore, + ip, + ); + let structured_source_request = + filter_marks_request(structured_source, &structured_services); + assert!( + has_client_tag_suppression_marker(&structured_source_request), + "structured Config Store IP exclusions should mark the request" + ); + } + + #[test] + fn non_ip_exclusions_do_not_mark_requests_for_client_tag_suppression() { + let ip = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10)); + let cases = [DataDomeConfig { + enabled: true, + enable_protection: true, + protection_exclusion_rules: vec![ProtectionExclusionRuleConfig { + id: "path".to_string(), + enabled: true, + methods: Vec::new(), + matcher: ProtectionMatcherConfig::PathExact { + paths: vec!["/page".to_string()], + }, + }], + ..DataDomeConfig::default() + }]; + + for config in cases { + let request = filter_marks_request(config, &noop_services_with_client_ip(ip)); + assert!( + !has_client_tag_suppression_marker(&request), + "non-IP exclusions should not mark the request" + ); + } + } + + #[test] + fn asn_exclusions_do_not_mark_requests_for_client_tag_suppression() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_asns: vec![64500], + ..DataDomeConfig::default() + }; + let geo_info = GeoInfo { + city: String::new(), + country: String::new(), + continent: String::new(), + latitude: 0.0, + longitude: 0.0, + metro_code: 0, + region: None, + asn: Some(64500), + }; + let request = filter_marks_request_with_geo( + config, + &noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(192, 0, 2, 10))), + Some(&geo_info), + ); + assert!( + !has_client_tag_suppression_marker(&request), + "ASN exclusions should not mark the request" + ); + } + + #[test] + fn non_matching_ip_does_not_mark_request_for_client_tag_suppression() { + let config = DataDomeConfig { + enabled: true, + enable_protection: true, + protection_excluded_ip_cidrs: vec!["192.0.2.0/24".to_string()], + ..DataDomeConfig::default() + }; + let request = filter_marks_request( + config, + &noop_services_with_client_ip(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 10))), + ); + assert!( + !has_client_tag_suppression_marker(&request), + "a non-matching IP should not mark the request" + ); + } + #[test] fn load_server_side_key_reads_secret_store() { let mut secrets = HashMap::new(); @@ -748,7 +1280,7 @@ mod tests { // the Protection API. let services = build_services_with_config_and_secret(NoopConfigStore, NoopSecretStore); let settings = Settings::default(); - let request = request_builder() + let mut request = request_builder() .method(Method::OPTIONS.as_str()) .uri("https://publisher.example/_ts/api/v1/identify") .body(EdgeBody::empty()) @@ -759,7 +1291,7 @@ mod tests { RequestFilterInput { settings: &settings, services: &services, - request: &request, + request: &mut request, geo_info: None, is_integration_route: false, }, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 0644fb522..291a54242 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -327,7 +327,7 @@ pub trait IntegrationProxy: Send + Sync { pub struct RequestFilterInput<'a> { pub settings: &'a Settings, pub services: &'a RuntimeServices, - pub request: &'a Request, + pub request: &'a mut Request, pub geo_info: Option<&'a GeoInfo>, /// Whether the request matches a registered integration proxy route. pub is_integration_route: bool, @@ -1345,6 +1345,8 @@ mod tests { } struct EnrichingRequestFilter; + #[derive(Clone, Copy)] + struct RequestAnnotation; #[async_trait(?Send)] impl IntegrationRequestFilter for EnrichingRequestFilter { @@ -1354,8 +1356,9 @@ mod tests { async fn filter_request( &self, - _input: RequestFilterInput<'_>, + input: RequestFilterInput<'_>, ) -> Result> { + input.request.extensions_mut().insert(RequestAnnotation); Ok(RequestFilterDecision::Continue(RequestFilterEffects { request_headers: vec![HeaderMutation::set("x-datadome-isbot", "1")], response_headers: vec![HeaderMutation::set("x-dd-b", "allowed")], @@ -1487,6 +1490,10 @@ mod tests { Some("1"), "should apply DataDome-style request enrichment before routing" ); + assert!( + req.extensions().get::().is_some(), + "should preserve private request annotations for downstream routing" + ); match outcome { RequestFilterRegistryOutcome::Continue(effects) => { assert_eq!( diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 94b42161c..13b6d70cd 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -631,6 +631,25 @@ pub(crate) fn build_services_with_config_and_secret( .build() } +pub(crate) fn build_services_with_config_and_secret_and_client_ip( + config_store: impl PlatformConfigStore + 'static, + secret_store: impl PlatformSecretStore + 'static, + client_ip: IpAddr, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(config_store)) + .secret_store(Arc::new(secret_store)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(NoopBackend)) + .http_client(Arc::new(NoopHttpClient)) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: Some(client_ip), + ..ClientInfo::default() + }) + .build() +} + pub(crate) fn build_request_signing_services() -> RuntimeServices { let signing_key = SigningKey::generate(&mut OsRng); let key_b64 = general_purpose::STANDARD.encode(signing_key.as_bytes()); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index d3410b4ed..5fc8dac0e 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -357,6 +357,7 @@ struct ProcessResponseParams<'a> { integration_registry: &'a IntegrationRegistry, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, + suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option<&'a crate::integrations::gpt_diagnostics::GptDiagnosticsRequestDecision>, } @@ -383,6 +384,7 @@ impl PublisherBodyProcessor { integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: Arc::clone(¶ms.ad_bids_state), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), })?) } else if is_rsc_flight { @@ -460,6 +462,7 @@ fn process_response_streaming( integration_registry: params.integration_registry, ad_slots_script: params.ad_slots_script.map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.cloned(), })?; StreamingPipeline::new(config, processor) @@ -944,6 +947,7 @@ struct HtmlStreamProcessorParams<'a> { integration_registry: &'a IntegrationRegistry, ad_slots_script: Option, ad_bids_state: Arc>>, + suppress_datadome_client_side_tag: bool, gpt_diagnostics: Option, } @@ -960,7 +964,8 @@ fn create_html_stream_processor( params.request_scheme, ) .with_ad_state(params.ad_slots_script, params.ad_bids_state) - .with_gpt_diagnostics(params.gpt_diagnostics); + .with_gpt_diagnostics(params.gpt_diagnostics) + .with_datadome_client_tag_suppression(params.suppress_datadome_client_side_tag); Ok(create_html_processor(config)) } @@ -1081,6 +1086,8 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, + /// Whether to omit Trusted Server's automatic `DataDome` client-side tag. + pub(crate) suppress_datadome_client_side_tag: bool, /// Request-scoped conditional diagnostics delivery decision. pub(crate) gpt_diagnostics: Option, @@ -1431,6 +1438,37 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } +/// Prevent shared caches from replaying tag-suppressed HTML to other clients. +fn apply_datadome_client_tag_cache_privacy( + response: &mut Response, + method: &Method, + suppress_datadome_client_side_tag: bool, + content_type: &str, +) { + if !suppress_datadome_client_side_tag + || !response_carries_body(method, response.status()) + || !is_html_content_type(content_type) + { + return; + } + + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|value| value.contains("private") || value.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } + for header_name in CDN_CACHE_HEADERS { + response.headers_mut().remove(*header_name); + } +} + /// Drop a bodiless response's body and correct its framing headers. /// /// The response keeps no body, and its `Content-Length` is corrected where the @@ -1547,6 +1585,7 @@ pub fn stream_publisher_body( integration_registry, ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: ¶ms.ad_bids_state, + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.as_ref(), }; process_response_streaming(body, output, &borrowed) @@ -1640,6 +1679,7 @@ pub async fn stream_publisher_body_async( integration_registry, ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), ad_bids_state: params.ad_bids_state.clone(), + suppress_datadome_client_side_tag: params.suppress_datadome_client_side_tag, gpt_diagnostics: params.gpt_diagnostics.clone(), }) { Ok(processor) => processor, @@ -2859,6 +2899,15 @@ pub async fn handle_publisher_request( // sets the flag unconditionally and tolerates buffered fallback): adapters // without streaming support may reject the flag outright rather than // silently buffering, which would fail every publisher fetch. + let request_method = req.method().clone(); + let suppress_datadome_client_side_tag = req + .extensions() + .get::() + .is_some(); + if suppress_datadome_client_side_tag { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + } let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -3051,6 +3100,12 @@ pub async fn handle_publisher_request( content_encoding ); + apply_datadome_client_tag_cache_privacy( + &mut response, + &request_method, + suppress_datadome_client_side_tag, + &content_type, + ); let body = std::mem::replace(response.body_mut(), EdgeBody::empty()); response.headers_mut().remove(header::CONTENT_LENGTH); @@ -3066,6 +3121,7 @@ pub async fn handle_publisher_request( content_type, ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), + suppress_datadome_client_side_tag, auction_observation, auction_request: auction_request_for_telemetry, dispatched_auction, @@ -4282,6 +4338,7 @@ mod tests { dispatched_auction: None, price_granularity: Default::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -5159,6 +5216,50 @@ mod tests { ); } + #[tokio::test] + async fn suppressed_publisher_request_removes_conditional_validators() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let mut req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .header(header::IF_NONE_MATCH, "\"cached-page\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build conditional request"); + req.extensions_mut() + .insert(crate::integrations::datadome::DataDomeClientTagSuppressed); + + let _response = run_publisher_proxy(&settings, &services, req).await; + + let headers = stub + .recorded_request_headers() + .into_iter() + .next() + .expect("should record one outbound request"); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_NONE_MATCH.as_str())), + "suppressed requests must not forward If-None-Match" + ); + assert!( + headers + .iter() + .all(|(name, _)| !name.eq_ignore_ascii_case(header::IF_MODIFIED_SINCE.as_str())), + "suppressed requests must not forward If-Modified-Since" + ); + } + #[tokio::test] async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { let settings = create_test_settings(); @@ -5274,6 +5375,154 @@ mod tests { ); } + #[test] + fn suppressed_datadome_tag_reaches_publisher_html_pipeline() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config( + "datadome", + &serde_json::json!({ + "enabled": true, + "client_side_key": "test-client-key", + }), + ) + .expect("should configure DataDome integration"); + let registry = IntegrationRegistry::new(&settings) + .expect("should create integration registry with DataDome"); + let mut params = make_stream_params(&settings, "identity"); + params.content_type = "text/html; charset=utf-8".to_string(); + params.suppress_datadome_client_side_tag = true; + let mut output = Vec::new(); + + stream_publisher_body( + EdgeBody::from(b"content".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should process suppressed HTML"); + + let html = String::from_utf8(output).expect("should produce UTF-8 HTML"); + assert!( + !html.contains("window.ddjskey"), + "publisher processing should omit the DataDome client configuration" + ); + assert!( + !html.contains("/integrations/datadome/tags.js"), + "publisher processing should omit the DataDome client tag URL" + ); + } + + #[test] + fn suppressed_datadome_html_is_private_and_not_shared_cached() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .header("fastly-surrogate-control", "max-age=600") + .header("cloudflare-cdn-cache-control", "max-age=600") + .header("cdn-cache-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable HTML response"); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, max-age=0"), + "suppressed HTML should be private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "suppressed HTML should not retain Surrogate-Control" + ); + assert!( + response.headers().get("fastly-surrogate-control").is_none(), + "suppressed HTML should not retain Fastly-Surrogate-Control" + ); + assert!( + response + .headers() + .get("cloudflare-cdn-cache-control") + .is_none(), + "suppressed HTML should not retain Cloudflare-CDN-Cache-Control" + ); + assert!( + response.headers().get("cdn-cache-control").is_none(), + "suppressed HTML should not retain CDN-Cache-Control" + ); + + let mut no_store_response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .body(EdgeBody::empty()) + .expect("should build no-store HTML response"); + super::apply_datadome_client_tag_cache_privacy( + &mut no_store_response, + &Method::GET, + true, + "text/html; charset=utf-8", + ); + assert_eq!( + no_store_response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("no-store"), + "suppressed HTML should preserve an existing no-store policy" + ); + } + + #[test] + fn datadome_cache_privacy_does_not_change_non_html_or_unsuppressed_responses() { + let mut response = Response::builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=600") + .header("surrogate-control", "max-age=600") + .body(EdgeBody::empty()) + .expect("should build cacheable response"); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + false, + "text/html; charset=utf-8", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "unsuppressed HTML should retain its existing cache policy" + ); + + super::apply_datadome_client_tag_cache_privacy( + &mut response, + &Method::GET, + true, + "text/css", + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("public, max-age=600"), + "non-HTML should retain its existing cache policy" + ); + } + #[test] fn response_carries_body_preserves_bodiless_metadata() { // A processable GET 200 buffers a body and recomputes Content-Length. @@ -6229,6 +6478,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6277,6 +6527,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -6314,6 +6565,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -6429,6 +6681,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -6482,6 +6735,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6538,6 +6792,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6594,6 +6849,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6650,6 +6906,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6694,6 +6951,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -6888,6 +7146,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -6952,6 +7211,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; // The `` that triggers bid injection lives in the SECOND gzip // member. `flate2::read::GzDecoder` decodes only the first member, so @@ -7015,6 +7275,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -7071,6 +7332,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7207,6 +7469,7 @@ mod tests { dispatched_auction, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } } @@ -7558,6 +7821,7 @@ mod tests { )), price_granularity: PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, } }; let make_stream_response = || PublisherResponse::Stream { @@ -7737,6 +8001,7 @@ mod tests { )), price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7804,6 +8069,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); @@ -7854,6 +8120,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -7962,6 +8229,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -8019,6 +8287,7 @@ mod tests { dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), gpt_diagnostics: None, + suppress_datadome_client_side_tag: false, }; let mut output = Vec::new(); diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index e23348211..2205ae892 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -22,6 +22,7 @@ pub const CDN_CACHE_HEADERS: &[&str] = &[ "fastly-surrogate-control", "cdn-cache-control", "cloudflare-cdn-cache-control", + "cdn-cache-control", ]; /// Forces cookie-bearing responses to stay private to shared caches. @@ -37,7 +38,7 @@ pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { if !response.headers().contains_key(header::SET_COOKIE) { return; } - // Surrogate cache headers must come off every cookie-bearing response, even + // Shared-cache control headers must come off every cookie-bearing response, even // one already carrying a stricter `no-store`/`private` directive — they are // independent of Cache-Control and would otherwise let a shared cache store // and replay one visitor's Set-Cookie. diff --git a/docs/guide/integrations/datadome.md b/docs/guide/integrations/datadome.md index 9c342679b..35eda5d3e 100644 --- a/docs/guide/integrations/datadome.md +++ b/docs/guide/integrations/datadome.md @@ -86,6 +86,7 @@ patterns = ["(?i)\\.(avi|flv|mka|mkv|mov|mp4|mpeg|mpg|mp3|flac|ogg|ogm|opus|wav| | `protection_excluded_ip_cidr_sources` | array | `[]` | Config Store sources containing dynamic client IP CIDR bypass lists | | `protection_ip_list_cache_ttl_seconds` | integer | `300` | Process-local cache TTL for Config Store-backed IP CIDR bypass lists | | `protection_exclusion_rules` | array | Static asset path regex | Structured method/path/query/IP/ASN exclusion rules | +| `protection_test_bypass` | object | omitted | Temporary static-header bypass for access-controlled staging tests | | `enable_graphql_support` | boolean | `false` | Reserved for future GraphQL body inspection; ignored in v1 | | `client_side_key` | string | `""` | DataDome client-side JavaScript key used for tag injection | | `inject_client_side_tag` | boolean | `true` | Auto-inject the browser tag when `client_side_key` is non-empty | @@ -168,11 +169,76 @@ A request is protected when all of the following are true: 5. The client IP does not match `protection_excluded_ip_cidrs` or any Config Store-backed CIDR source. 6. The client ASN is not listed in `protection_excluded_asns`. 7. No `protection_exclusion_rules` match. +8. The request does not contain a matching enabled `protection_test_bypass` credential. Static assets are excluded by default using a case-insensitive file-extension regex. Trusted Server internal routes such as `/static/tsjs=`, `/integrations/`, `/first-party/`, admin routes, discovery routes, and signature-verification routes are also excluded by default. Auction traffic at `/auction` is protected by default. +### Staging test bypass + +For short-lived browser automation on an access-controlled staging site, you +can configure a static header credential that skips only the server-side +Protection API: + +```toml +[integrations.datadome.protection_test_bypass] +enabled = true +credential_secret_store = "ts_secrets" +credential_secret_name = "datadome_test_bypass" +``` + +`protection_test_bypass` requires `enable_protection = true`; it is disabled +when omitted. Store the temporary credential in the configured Secret Store, +configure this section only while needed, protect the site with an outer access +control such as Basic Auth, and remove the section when testing finishes. Do not +enable it in production. + +The fixed `x-ts-datadome-bypass` header is compared in constant time, removed +before the request can reach DataDome or the publisher origin, and never +logged. Scope the header to the staging origin; do not attach it to every +request in a browser context because that can disclose the credential to +third-party origins. With Playwright: + +```ts +await context.route('https://staging.example.com/**', async (route) => { + const headers = { + ...route.request().headers(), + 'x-ts-datadome-bypass': process.env.DATADOME_TEST_BYPASS!, + } + await route.continue({ headers }) +}) +``` + +### Client-side tag suppression behavior + +On the Fastly adapter, a request that matches an IP-based DataDome exclusion +or the configured test-bypass credential also omits Trusted Server's +automatically injected client-side DataDome tag from processed HTML. This keeps +the client-side layer consistent with the server-side Protection API skip. + +This behavior applies to: + +- `protection_excluded_ip_cidrs`; +- `protection_excluded_ip_cidr_sources`; +- structured `ip_cidr` rules; +- structured `ip_cidr_source` rules; and +- a matching enabled `protection_test_bypass` credential. + +ASN, method, path, query-parameter, static-asset, and internal-route +exclusions do not automatically suppress the client-side tag. DataDome tags +already present in publisher HTML are not removed or changed by this behavior, +and `/integrations/datadome/tags.js` remains available when requested directly. + +Because the processed HTML differs by client IP or test credential, +tag-suppressed HTML is marked `private, max-age=0` and removed from shared +surrogate caches. The decision is reported in the existing protection log, for +example: + +```text +[datadome] protection decision=skipped rule=protection-test-bypass reason=test_bypass client_tag=omitted method=GET +``` + ### Structured exclusion rules Use structured rules for all DataDome protection exclusions. Each rule has an `id`, optional `methods`, and a typed matcher. The default configuration includes a `path_regex` rule for common static assets. diff --git a/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md new file mode 100644 index 000000000..0dfc923cf --- /dev/null +++ b/docs/superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md @@ -0,0 +1,475 @@ +# DataDome IP-excluded client tag suppression — Implementation Plan + +> **Status:** Approved for implementation +> +> **For implementers:** Work task by task and keep the workspace buildable. +> Follow `CLAUDE.md`: use target-matched Cargo aliases, do not use bare +> workspace tests, and do not add an internal HTTP header for request state. + +**Goal:** When Fastly's authoritative client IP matches a DataDome IP exclusion, +skip the Protection API call and omit only Trusted Server's automatically +injected DataDome client tag from every processed HTML response. + +**Issue:** #994 +**Design:** +`docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md` + +## Approved behavior + +| Request condition | Protection API | Trusted Server auto-injected tag | Publisher-originated tag | +| ------------------------------------------------------------- | --------------- | -------------------------------- | ------------------------ | +| Inline IP CIDR match | Skipped | Omitted | Unchanged | +| Config Store IP CIDR-source match | Skipped | Omitted | Unchanged | +| Structured `ip_cidr` match | Skipped | Omitted | Unchanged | +| Structured `ip_cidr_source` match | Skipped | Omitted | Unchanged | +| ASN, method, path, query, static, or internal-route exclusion | Skipped | Preserved | Unchanged | +| No exclusion match | Called normally | Preserved | Unchanged | +| Protection API fail-open | Continued | Preserved | Unchanged | + +The Fastly-only scope means that other adapters receive the default +non-suppressed value. Do not add a configuration option and do not modify their +request-filter wiring. + +## Runtime contracts + +1. **Trusted identity source:** determine exclusion from + `RuntimeServices::client_info().client_ip`, never a caller-provided header. +2. **Single evaluation:** use the existing `ProtectionScope` decision. Do not + evaluate CIDRs a second time while injecting HTML; this avoids diverging + Config Store/cache behavior. +3. **Private marker:** communicate the decision with a typed request extension, + never a request/response header. The marker cannot leak to the origin or + client. +4. **Precise scope:** tag suppression is keyed only on decision reasons + `client_ip`, `client_ip_source`, `ip_cidr`, and `ip_cidr_source`. +5. **Cache safety:** an HTML response with the tag omitted differs by client IP. + A suppressed processed HTML response must be `private, max-age=0` and have + `Surrogate-Control` and `Fastly-Surrogate-Control` removed. Do not alter + cache headers when the response is not processed HTML, because this feature + does not alter that body. +6. **No behavior drift:** DataDome proxy endpoints, response-header effects, + `rewrite_sdk`, and DataDome tags that were already in origin HTML retain + their current behavior. + +## File map + +| File | Change | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/registry.rs` | Permit filters to attach private typed request extensions while retaining header-effect semantics. Extend the HTML context with the propagated boolean. | +| `crates/trusted-server-core/src/integrations/datadome.rs` | Define the crate-private marker and have the head injector honor the HTML-context flag. | +| `crates/trusted-server-core/src/integrations/datadome/protection.rs` | Recognize IP scope skips, attach the marker, and add `client_tag=omitted` to the existing info log. | +| `crates/trusted-server-core/src/html_processor.rs` | Carry the per-response suppression boolean from config to all integration HTML contexts. | +| `crates/trusted-server-core/src/publisher.rs` | Snapshot the marker before origin dispatch, propagate it through every HTML streaming path, and apply cache privacy to suppressed processed HTML. | +| `docs/guide/integrations/datadome.md` | Document the Fastly IP-exclusion behavior and its limits. | +| `docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md` | Already updated with the cache-variance safeguard. | + +No changes are expected in `trusted-server.example.toml`, JavaScript bundles, +or non-Fastly adapters. + +--- + +## Task 1: Make the request-filter input capable of private annotations + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: its existing `#[cfg(test)]` module + +The current `RequestFilterInput` holds `&Request`. Change it to hold +`&mut Request` so a request filter can add a typed extension. This is +the narrowest safe transport because the registry already has exclusive mutable +access to the request while it invokes each filter. + +- [ ] **Step 1: Add a regression test for an extension-producing filter.** Create + a test-only zero-sized marker and filter that writes it to + `input.request.extensions_mut()`. Run `IntegrationRegistry::filter_request` + and assert the original mutable request has the marker afterward. In the + same test, verify normal `RequestFilterEffects` still apply their request + header mutation and return their response header mutation. +- [ ] **Step 2: Change `RequestFilterInput::request` to a mutable borrow.** Keep + the `IntegrationRequestFilter` method signature and `RequestFilterEffects` + unchanged. +- [ ] **Step 3: Update `IntegrationRegistry::filter_request`.** Pass its existing + `&mut Request` directly to each `RequestFilterInput`. Keep the ordering: + filter mutation first, then registry-applied request-header effects, then + the next filter. +- [ ] **Step 4: Update all direct filter tests and test filters.** Calls that build + `RequestFilterInput` must construct a mutable request and pass + `request: &mut request`. Read-only filters should continue to compile by + simply not mutating the request. +- [ ] **Step 5: Run focused tests.** + +```bash +cargo test-fastly integrations::registry +``` + +**Acceptance:** a filter can retain a typed marker for downstream route handling +without emitting a synthetic `x-*` header, and existing header effects retain +their behavior. + +--- + +## Task 2: Mark IP-based DataDome exclusions and log the outcome + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/datadome.rs` +- Modify: `crates/trusted-server-core/src/integrations/datadome/protection.rs` +- Test: `crates/trusted-server-core/src/integrations/datadome/protection.rs` +- Reuse: `crates/trusted-server-core/src/integrations/datadome/protection_scope.rs` + +- [ ] **Step 1: Add a crate-private marker in `datadome.rs`.** Define a + zero-sized type with a behavior-oriented name, such as + `DataDomeClientTagSuppressed`. It must be visible to `publisher.rs` and + `protection.rs` through `pub(crate)`, but must not be exported as public + integration configuration or API. +- [ ] **Step 2: Add an IP-reason predicate beside protection logging.** Centralize + the exact four eligible scope reasons in one helper: + +```rust +matches!(reason, "client_ip" | "client_ip_source" | "ip_cidr" | "ip_cidr_source") +``` + + Do not infer eligibility from rule ID: Config Store source rule IDs are + operator-configured strings. + +- [ ] **Step 3: Make `filter_protection_request` own a mutable input and pass it + mutably to `is_request_protected`.** In the existing + `ProtectionScopeDecision::Skip` arm: + + 1. determine whether the reason is IP-based; + 2. if so, insert the typed marker into `input.request.extensions_mut()`; + 3. call the updated skip logger with `client_tag_omitted = true`; and + 4. return `false` exactly as today so the Protection API is not called. + + Do not set the marker for the early method/integration/internal-route + returns. Do not set it when the API call returns a fail-open error. + +- [ ] **Step 4: Update `log_protection_skip`.** Keep IP exclusions at `info` and + non-IP exclusions at `debug`. For the IP branch, extend the existing + structured text after the reason with `client_tag=omitted`; retain rule, + reason, method, host, and path, but do not include the client IP. The + desired shape is: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET +``` + +- [ ] **Step 5: Add filter-level marker tests.** Add small helpers in the + protection test module to build `RuntimeServices` with a fixed client IP, + optional Config Store data, and a mutable request. For each case, call + `filter_protection_request`, assert it returns `Continue`, and inspect the + request extension: + + - inline `protection_excluded_ip_cidrs` match → marker present; + - `protection_excluded_ip_cidr_sources` match → marker present; + - structured `ProtectionMatcherConfig::IpCidr` match → marker present; + - structured `ProtectionMatcherConfig::IpCidrSource` match → marker present. + + Clear the process-global CIDR-source test cache before and after source + tests so cached values cannot affect another case. + +- [ ] **Step 6: Add negative filter-level tests.** Assert the marker is absent + for a non-matching IP, a configured ASN match, a structured path match, + a structured query match, an excluded method, and an internal/integration + route. Reuse the existing `ProtectionScope` unit tests for matching + semantics; these new tests verify only the new side effect. +- [ ] **Step 7: Preserve API-call behavior.** For an IP marker test, use an HTTP + client double that records calls or errors if called. Assert no Protection + API request is sent. This protects against accidentally marking a request + while still invoking DataDome. +- [ ] **Step 8: Run focused tests.** + +```bash +cargo test-fastly datadome::protection +cargo test-fastly datadome::protection_scope +``` + +**Acceptance:** only the four IP decision reasons add the private marker and +produce the augmented informational skip log; all other exclusion and fail-open +paths keep their current tag behavior. + +--- + +## Task 3: Thread suppression through publisher response processing + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/html_processor.rs` +- Modify: `crates/trusted-server-core/src/integrations/registry.rs` +- Test: `publisher.rs` and `html_processor.rs` test modules + +### Data flow to implement + +```text +DataDomeClientTagSuppressed request extension + -> bool captured by handle_publisher_request before origin dispatch + -> OwnedProcessResponseParams + -> ProcessResponseParams / HtmlStreamProcessorParams + -> HtmlProcessorConfig + -> IntegrationHtmlContext + -> DataDomeIntegration::head_inserts +``` + +- [ ] **Step 1: Capture the marker once in `handle_publisher_request`.** Read + `req.extensions().get::().is_some()` before + `req` is rewritten and moved into `PlatformHttpRequest`. Store the boolean + only in the `PublisherResponse::Stream` parameters, because that is the + only response route that passes through HTML injection. +- [ ] **Step 2: Add a boolean to the owned and borrowed publisher-processing + parameter structs.** Add a clearly named field such as + `suppress_datadome_client_side_tag` to: + + - `OwnedProcessResponseParams`; + - `ProcessResponseParams`; and + - `HtmlStreamProcessorParams`. + + Pass it through all three existing HTML construction sites: + + - `PublisherBodyProcessor::new` for async buffered processing; + - `process_response_streaming` for synchronous processing; and + - `stream_publisher_body_async` for the Fastly streaming auction-hold path. + + Every test fixture that constructs `OwnedProcessResponseParams` directly + must set `false` unless it explicitly exercises suppression. + +- [ ] **Step 3: Extend `HtmlProcessorConfig`.** Add the same boolean, default it + to `false` in `from_settings`, and add a narrow builder method used by + `create_html_stream_processor`. Update direct `HtmlProcessorConfig` + fixtures and the benchmark fixture to set `false` explicitly. +- [ ] **Step 4: Extend `IntegrationHtmlContext`.** Add the boolean as immutable + request-scoped context. Populate it at both construction sites in + `html_processor.rs`: + + - the streaming `` element handler; and + - `HtmlWithPostProcessing::process_chunk` for full-document post-processors. + + Update every test helper that constructs `IntegrationHtmlContext` to set + `false` by default. + +- [ ] **Step 5: Add plumbing tests.** + + - `HtmlProcessorConfig::from_settings` defaults to non-suppressed. + - A test head injector records the context flag and sees `true` when a config + is built with suppression. + - A `publisher.rs` route test inserts the DataDome marker into a request, + receives a processable HTML `PublisherResponse::Stream`, and verifies the + owned parameters carry `true`. + - A buffered and a streaming-body path both preserve `true` to head injection. + +- [ ] **Step 6: Run focused tests.** + +```bash +cargo test-fastly html_processor +cargo test-fastly publisher +``` + +**Acceptance:** the decision is read once from a private request extension and +is available to every head injector for every processed HTML response, including +Fastly's streaming path. + +--- + +## Task 4: Omit only Trusted Server's injected DataDome tag + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/datadome.rs` +- Test: `crates/trusted-server-core/src/integrations/datadome.rs` +- Test: `crates/trusted-server-core/src/html_processor.rs` or `publisher.rs` + +- [ ] **Step 1: Add a direct head-injector regression test.** With a client-side + key configured and `ctx.suppress_datadome_client_side_tag = true`, assert + `head_inserts()` returns an empty vector. The same config with `false` + must still return exactly one snippet containing both `window.ddjskey` and + the configured tag URL. +- [ ] **Step 2: Implement the guard as the first condition in + `DataDomeIntegration::head_inserts`.** Return an empty vector when the + context flag is true; otherwise retain all current serialization, + escaping, blank-key, and `inject_client_side_tag` behavior unchanged. +- [ ] **Step 3: Add an end-to-end HTML pipeline test.** Configure the DataDome + integration with a client-side key, process representative HTML with + suppression enabled, and assert the result contains neither: + +```text +window.ddjskey= +/integrations/datadome/tags.js +``` + + Repeat with suppression disabled and assert both appear. + +- [ ] **Step 4: Pin publisher-originated-tag behavior.** Feed origin HTML that + contains a DataDome `tags.js` element. With suppression enabled, assert + that element remains in output and is rewritten by `rewrite_sdk` exactly + as before. This distinguishes automatic injection from origin markup. +- [ ] **Step 5: Pin direct route behavior.** Retain or add a DataDome proxy test + showing that `GET /integrations/datadome/tags.js` remains registered and + fetches/proxies the SDK normally; suppression affects only HTML injection. +- [ ] **Step 6: Run focused tests.** + +```bash +cargo test-fastly datadome +``` + +**Acceptance:** suppression removes only the generated configuration/script +pair; nothing removes publisher markup or disables DataDome endpoints. + +--- + +## Task 5: Make tag-suppressed processed HTML private + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` + +The automatic tag makes the processed HTML vary by client IP. Cache privacy is +therefore a correctness and protection requirement, not an optional +optimization. + +- [ ] **Step 1: Add a failing cache-privacy test.** Build a `PublisherResponse` + with a processable HTML content type, suppression `true`, and cacheable + origin headers (`Cache-Control`, `Surrogate-Control`, and + `Fastly-Surrogate-Control`). Assert the stream response is: + + - `Cache-Control: private, max-age=0`; and + - missing both surrogate cache headers. + +- [ ] **Step 2: Apply privacy only in the `ResponseRoute::Stream` HTML arm.** + After response classification confirms a processable HTML stream, use the + existing per-user ad-stack policy as the model. Do not alter cache headers + for CSS, RSC, non-processable pass-through, unsupported encodings, HEAD, + 204/205/304, or responses without suppression: none has a body variation + created by this feature. +- [ ] **Step 3: Add non-regression cache tests.** Verify that: + + - non-suppressed processed HTML keeps its existing cache headers unless + another existing policy changes them; + - a suppressed CSS/non-HTML stream is not made private by this feature; and + - existing ad-stack privacy behavior remains unchanged when both features are + active. + +- [ ] **Step 4: Run focused tests.** + +```bash +cargo test-fastly publisher +``` + +**Acceptance:** a shared cache cannot replay an IP-excluded client's tagless +HTML to a non-excluded visitor, while unchanged responses retain their existing +cacheability. + +--- + +## Task 6: Add Fastly-path regression coverage + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` tests only if the + existing dispatch helpers can exercise the DataDome registry with a stubbed + publisher response. +- Otherwise, document the existing core filter + publisher pipeline tests as + the executable behavioral coverage; do not refactor Fastly production code + merely to enable a duplicate test. + +- [ ] **Step 1: Extend the existing Fastly request-filter dispatch regression + test or add a focused equivalent.** Configure a DataDome request filter, + insert trusted `ClientInfo` into the request extensions with a matching + IP, and confirm the filter runs before publisher routing. +- [ ] **Step 2: Assert that the routed request retains the private DataDome + marker.** The assertion must inspect request extensions or the processed + HTML result, not an HTTP header. +- [ ] **Step 3: Ensure no actual DataDome API call occurs for the matching IP.** + Use a recording/failing HTTP client or the existing Fastly test seam. +- [ ] **Step 4: Add the non-matching counterpart.** It must not receive the + marker and must continue to inject the configured tag when HTML is + processed. +- [ ] **Step 5: Run Fastly adapter tests.** + +```bash +cargo test-fastly +``` + +**Acceptance:** the production adapter's actual filter ordering preserves the +marker from authoritative Fastly client metadata through publisher HTML +processing. If the current test seam cannot stub a full origin response, retain +this as focused request-filter-order coverage and rely on Task 3's core +pipeline tests for body output rather than expanding adapter production code. + +--- + +## Task 7: Document operator-visible behavior + +**Files:** + +- Modify: `docs/guide/integrations/datadome.md` +- Do not modify: `trusted-server.example.toml` + +- [ ] **Step 1: Add a subsection adjacent to “Protected traffic” or “Client-side + setup.”** State that, on Fastly, an IP exclusion skips the Protection API + and suppresses only Trusted Server's automatic DataDome tag injection on + processed HTML. +- [ ] **Step 2: List the four covered IP sources.** Use the exact configuration + names and structured rule types. +- [ ] **Step 3: State the exclusions that do not suppress the client tag.** ASN, + method, path, query, static-asset, and internal-route exclusions retain + normal auto-injection. +- [ ] **Step 4: State the limits.** Publisher-originated/manual tags are not + removed; `/integrations/datadome/tags.js` remains available; no new + configuration is required; and tag-suppressed processed HTML is private + to prevent shared-cache replay. +- [ ] **Step 5: Add the diagnostic example.** Use an example-only host/IP and + include `client_tag=omitted` with rule and reason. +- [ ] **Step 6: Format-check the changed documentation.** + +```bash +cd docs +npx prettier --check guide/integrations/datadome.md \ + superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md \ + superpowers/plans/2026-08-03-datadome-ip-excluded-client-tag.md +``` + +**Acceptance:** operators can predict exactly when the tag will be omitted and +understand that this is an IP-based Fastly behavior, not a general exclusion +side effect. + +--- + +## Final verification + +- [ ] Confirm the working tree contains only the intended core, Fastly-test, + guide, spec, and plan changes. +- [ ] Run formatting. + +```bash +cargo fmt --all -- --check +``` + +- [ ] Run the relevant target-matched test suites. + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +``` + +- [ ] Run required lint suites. + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +``` + +- [ ] Run the docs check from Task 7. +- [ ] Review the diff for accidental exposure of the marker as a request or + response header, duplicate CIDR evaluation, unintended publisher-tag + removal, or shared-cacheable tag-suppressed HTML. + +## Deferred acceptance + +Do **not** perform live production/browser verification in this change. After +deployment, the separate testing workflow should verify that a matching +whitelisted IP receives processed HTML without Trusted Server's +`/integrations/datadome/tags.js` injection, while an unlisted IP retains it. diff --git a/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md new file mode 100644 index 000000000..d6d811780 --- /dev/null +++ b/docs/superpowers/specs/2026-08-03-datadome-ip-excluded-client-tag-design.md @@ -0,0 +1,339 @@ +# DataDome IP-excluded client tag suppression + +**Issue:** #994 +**Date:** 2026-08-03 +**Status:** Proposed + +## Problem + +Trusted Server has two DataDome protection layers: + +1. Server-side Protection API validation, which can be skipped for configured + client IP CIDRs. +2. Client-side tag auto-injection, which adds `window.ddjskey`, + `window.ddoptions`, and the configured `tags.js` script to processed HTML. + +When a request matches an IP-based server-side exclusion, the Protection API is +skipped, but the client-side tag is currently still injected. The browser can +therefore continue running client-side DataDome protection for a request that +was explicitly whitelisted at Trusted Server. + +The desired behavior is that Fastly requests skipped by an IP-based DataDome +exclusion also omit Trusted Server's automatically injected client-side tag. + +## Goals + +- Suppress Trusted Server's automatically injected DataDome client-side tag for + Fastly requests skipped by an IP-based protection exclusion. +- Reuse the existing authoritative protection-scope decision. +- Cover all supported IP-based exclusion mechanisms: + - `protection_excluded_ip_cidrs` + - `protection_excluded_ip_cidr_sources` + - structured `ip_cidr` rules + - structured `ip_cidr_source` rules +- Preserve current behavior for non-IP exclusions. +- Leave publisher-originated or manually configured DataDome tags untouched. +- Add an informational diagnostic indicating that the client tag is omitted. +- Keep the implementation independent of caller-supplied IP headers. +- Prevent a shared cache from replaying IP-specific, tag-suppressed HTML to + non-excluded visitors. + +## Non-goals + +- Do not add a configuration flag or make this behavior opt-in. +- Do not change Axum, Cloudflare, or Spin request-filter wiring. This behavior + is intentionally scoped to the Fastly adapter, where the DataDome server-side + request filter is currently run. +- Do not suppress DataDome tags that originate in publisher HTML. +- Do not remove or disable the `/integrations/datadome/tags.js` route. +- Do not change DataDome signal-collection proxy behavior. +- Do not change ASN, path, query-parameter, method, static-asset, or internal + route exclusions. +- Do not perform live production verification as part of implementation. + +## Confirmed decisions + +1. **Adapter scope:** Fastly only. +2. **IP scope:** all four IP-based exclusion mechanisms listed above. +3. **Tag scope:** Trusted Server's auto-injected tag only. +4. **HTML scope:** every HTML response that enters the existing HTML processing + pipeline. +5. **Logging:** enrich the existing IP-exclusion skip log with + `client_tag=omitted`, including the matched rule and reason. +6. **Live testing:** deferred until after implementation and deployment/testing + workflow review. + +## Current architecture + +### Server-side protection + +`DataDomeIntegration::is_request_protected()` in +`crates/trusted-server-core/src/integrations/datadome/protection.rs` evaluates +method, internal-route, ASN, IP, and structured exclusion conditions. It uses +the client IP from `RuntimeServices::client_info()`, which is populated from +trusted Fastly request metadata. It does not use a caller-supplied IP header. + +The current function reduces the protection-scope result to a boolean. For an +IP exclusion it logs the skip and returns `false`, causing the request filter to +continue without calling the Protection API. + +The Fastly EdgeZero fallback path runs this request filter before route +selection and publisher proxying. The request continues into +`handle_publisher_request()` after the filter returns a continue decision. + +### Client-side injection + +`DataDomeIntegration::head_inserts()` in +`crates/trusted-server-core/src/integrations/datadome.rs` emits the client-side +snippet when: + +- `inject_client_side_tag` is true; and +- `client_side_key` is non-empty. + +The injector currently receives `IntegrationHtmlContext`, which contains HTML +host/scheme and document state but no request IP or protection decision. + +The publisher response path carries request-specific values through: + +```text +Request + -> OwnedProcessResponseParams + -> HtmlStreamProcessorParams + -> HtmlProcessorConfig + -> IntegrationHtmlContext + -> IntegrationHeadInjector +``` + +The existing DataDome attribute rewriter separately rewrites DataDome URLs +found in publisher HTML. That behavior must remain unchanged. + +## Design + +### 1. Capture an IP-exclusion marker at the request filter + +The request filter must attach a typed, internal request-scoped marker when the +existing protection-scope evaluation returns a skip for one of these reasons: + +- `client_ip` +- `client_ip_source` +- `ip_cidr` +- `ip_cidr_source` + +The marker must be attached only after the existing scope decision confirms the +IP exclusion. It must not be inferred from request headers or recomputed later +in the HTML pipeline. + +The request-filter API currently exposes an immutable request view. Add the +smallest internal mechanism needed for a filter to attach a typed request +extension without introducing a caller-visible header. Header mutations should +continue to use `RequestFilterEffects` as they do today. + +The marker should be a zero-sized or otherwise minimal internal type. It only +needs to answer whether Trusted Server's DataDome client tag should be +suppressed; the existing skip log supplies the rule ID and reason. + +The marker must not be attached for: + +- `OPTIONS` or other excluded methods before scope evaluation; +- internal or integration routes; +- ASN exclusions; +- path, query, or other non-IP structured exclusions; +- unmatched IP rules; +- Protection API fail-open behavior; or +- requests where `enable_protection` is false and the request filter does not + run. + +### 2. Enrich the existing skip log + +For IP-based skips, extend the existing informational log with +`client_tag=omitted`: + +```text +[datadome] protection decision=skipped rule=excluded-ip-cidrs reason=client_ip client_tag=omitted method=GET +``` + +The existing rule ID, reason, and method remain part of the log. Host and path are omitted to avoid placing dynamic request data in the protection logs. +Client IP values are not included. Non-IP skip logs retain their current +behavior and level. + +This log represents the request policy decision. It may also apply to a +non-HTML response, for which no HTML tag would have been injected anyway. + +### 3. Propagate the marker into HTML processing + +Before the publisher request is moved into the platform HTTP client, snapshot +whether the request carries the marker. Carry that request-scoped boolean +through `OwnedProcessResponseParams`, `HtmlStreamProcessorParams`, and +`HtmlProcessorConfig`. + +The value should default to `false` in all existing constructors and direct +unit-test fixtures. Non-Fastly adapters will naturally retain the default +because they do not currently produce the Fastly request-filter marker. + +Expose the value to head injectors through the existing HTML processing context +or equivalent request-scoped integration context. The propagation must work for +both: + +- the normal buffered HTML path; and +- the streaming HTML path, including the auction-hold path. + +The value is irrelevant for non-HTML, RSC, pass-through, and unmodified +responses, which should retain their current processing. + +### 4. Keep IP-specific HTML out of shared cache + +A processed HTML response differs by client IP when the generated tag is +suppressed. In the `PublisherResponse::Stream` path, when suppression is active +and the response is HTML, set `Cache-Control: private, max-age=0` and remove +`Surrogate-Control` and `Fastly-Surrogate-Control` before the body is streamed. + +This matches the existing per-user ad-stack cache policy. It prevents Fastly or +another shared cache from replaying a tag-suppressed response to a visitor whose +IP does not match an exclusion. Do not change cache headers for non-HTML, +pass-through, or unmodified responses because their output does not vary by this +feature. + +### 5. Suppress only the generated DataDome snippet + +At the start of `DataDomeIntegration::head_inserts()`: + +1. Check the request-scoped suppression marker. +2. If present, return no DataDome head inserts. +3. Otherwise preserve the current `inject_client_side_tag` and + `client_side_key` checks and emit the existing snippet unchanged. + +When suppression is active, omit both: + +```html + + +``` + +Do not alter: + +- publisher-originated DataDome script tags; +- `rewrite_sdk` behavior; +- the DataDome SDK proxy route; +- the signal collection API proxy; +- DataDome configuration serialization for non-suppressed requests; or +- injection behavior for requests without the marker. + +## Testing plan + +### Protection-filter tests + +Add or extend tests in +`crates/trusted-server-core/src/integrations/datadome/protection.rs` to verify +that the marker is attached for: + +- a matching inline IPv4 CIDR; +- a matching Config Store-backed CIDR source; +- a matching structured `ip_cidr` rule; and +- a matching structured `ip_cidr_source` rule. + +Verify that the marker is absent for: + +- a non-matching IP; +- an ASN exclusion; +- a path exclusion; +- a query-parameter exclusion; +- an excluded method; and +- an internal or integration route. + +Verify the existing protection behavior remains unchanged: IP-matched requests +continue without a Protection API call. + +### Head-injector tests + +Add tests in +`crates/trusted-server-core/src/integrations/datadome.rs` verifying that: + +- a configured client tag is omitted when suppression is active; +- a configured client tag is emitted when suppression is inactive; +- a blank client-side key remains a no-op; and +- `inject_client_side_tag = false` remains a no-op. + +### HTML pipeline tests + +Add coverage for the request-scoped value flowing through the HTML processor, +including the streaming path. Confirm that a suppressed processed HTML response +contains neither the injected `window.ddjskey` configuration nor the configured +DataDome `tags.js` script. For a suppressed HTML stream, assert the response is +private and has no surrogate cache headers. Confirm a non-suppressed HTML stream +retains its origin cache behavior. + +Confirm that publisher-originated DataDome tags remain in the output and are +still rewritten according to the existing `rewrite_sdk` behavior. + +### Fastly dispatch tests + +Add a Fastly adapter dispatch test with: + +- DataDome protection enabled; +- a client IP matching an inline exclusion; +- a configured client-side key; and +- an HTML publisher response. + +The test should verify that the request continues without a Protection API +call, the response includes the `client_tag=omitted` decision log through the +existing test logging seam where available, and the generated tag is absent. + +Also cover a non-excluded request to confirm the generated tag remains present. + +## Documentation changes + +Update `docs/guide/integrations/datadome.md` to state that IP-excluded Fastly +requests skip both: + +- server-side Protection API validation; and +- Trusted Server's automatic client-side tag injection. + +Document that this does not remove or disable publisher-originated DataDome +tags, and that non-IP exclusions do not automatically suppress the client-side +tag. + +No configuration template changes are required because this behavior has no +new setting. + +## Files expected to change + +- `crates/trusted-server-core/src/integrations/registry.rs` + - Support the internal request-scoped annotation mechanism. +- `crates/trusted-server-core/src/integrations/datadome.rs` + - Define the marker and conditionally suppress head injection. +- `crates/trusted-server-core/src/integrations/datadome/protection.rs` + - Attach the marker for IP-based scope skips and enrich the skip log. +- `crates/trusted-server-core/src/integrations/registry.rs` or the relevant + HTML context definition + - Carry the suppression decision into head injection. +- `crates/trusted-server-core/src/html_processor.rs` + - Carry the request-scoped value into HTML integration context. +- `crates/trusted-server-core/src/publisher.rs` + - Snapshot and propagate the request marker through response processing. +- `docs/guide/integrations/datadome.md` + - Document the behavior. +- Relevant unit and Fastly adapter test modules. + +The exact split between registry request annotations and HTML context plumbing +should remain minimal and should not introduce a new public configuration API. + +## Verification + +Implementation verification should use the repository's target-matched +commands: + +```bash +cargo fmt --all -- --check +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +``` + +No live production validation is required for this implementation task. Live +browser verification will be performed later through the deployment/testing +workflow.