From a39138cd49061cc7cc5aa04679a40d1a97757361 Mon Sep 17 00:00:00 2001 From: MO Thibault <103271673+MO-Thibault@users.noreply.github.com> Date: Sat, 19 Sep 2026 14:45:24 -0400 Subject: [PATCH 1/2] analytics: report referrer and traffic source in both analytics versions Adds two fields to the auction payload built by lib/addons/prebid/analytics.ts and lib/addons/prototypes/analytics.js: - referrer: the referring hostname only, never the full referring URL, which can carry query strings with personal data. - trafficSource: direct, internal, paid, campaign, email, organic_search, social or referral, derived from the referrer plus the campaign and click-id parameters on the landing URL. Both are first-touch per tab. document.referrer becomes the site's own hostname after the first internal click, so the first non-internal result is cached in sessionStorage under optableTrafficSource and reused for the rest of the session. Without that, every auction after the first pageview would report internal. The logic lives in lib/core/traffic-source.ts for the typed path and is inlined in the prototype file, which is self-contained and imports no modules. Co-Authored-By: Claude Opus 5 (1M context) --- lib/addons/prebid/analytics.md | 12 +++- lib/addons/prebid/analytics.ts | 2 + lib/addons/prototypes/analytics.js | 86 +++++++++++++++++++++++ lib/core/traffic-source.ts | 105 +++++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 lib/core/traffic-source.ts diff --git a/lib/addons/prebid/analytics.md b/lib/addons/prebid/analytics.md index 3af8879e..cf85c541 100644 --- a/lib/addons/prebid/analytics.md +++ b/lib/addons/prebid/analytics.md @@ -129,5 +129,13 @@ Returns the `OptablePrebidAnalytics` instance, or `null` when no Prebid instance Each sampled auction sends an `optable.prebid.auction` event with, among others: `bidderRequests` (with per-bid `status`, `cpm`, `size`, `splitTestAssignment`), `optableMatchers`, `optableSources`, `optableTargetingDone`, `bidWon`, `missed`, -`url`, `tenant`, `prebidjsVersion`, `sessionDepth`, `pageAuctionsCount`, -`originSlug`, and the parsed `userAgent`/`device`. +`url`, `referrer`, `trafficSource`, `tenant`, `prebidjsVersion`, `sessionDepth`, +`pageAuctionsCount`, `originSlug`, and the parsed `userAgent`/`device`. + +`referrer` is the referring hostname only, never the full referring URL. +`trafficSource` classifies it as one of `direct`, `internal`, `paid`, +`campaign`, `email`, `organic_search`, `social` or `referral`. Both are +first-touch per tab: the first non-internal result is cached in `sessionStorage` +under `optableTrafficSource` and reused for the rest of the session, because +`document.referrer` becomes the site's own hostname after the first internal +click. diff --git a/lib/addons/prebid/analytics.ts b/lib/addons/prebid/analytics.ts index bec0197c..d2afdd27 100644 --- a/lib/addons/prebid/analytics.ts +++ b/lib/addons/prebid/analytics.ts @@ -3,6 +3,7 @@ import type { WitnessProperties } from "../../edge/witness"; import type OptableSDK from "../../sdk"; import { buildRequest } from "../../core/network"; +import { getTrafficSource } from "../../core/traffic-source"; import * as Bowser from "bowser"; @@ -485,6 +486,7 @@ class OptablePrebidAnalytics { })), missed, url: `${window.location.hostname}${window.location.pathname}`, + ...getTrafficSource(), tenant: this.optableInstance?.dcn?.node ?? "unknown", // eslint-disable-next-line no-undef optableWrapperVersion: SDK_WRAPPER_VERSION || "unknown", diff --git a/lib/addons/prototypes/analytics.js b/lib/addons/prototypes/analytics.js index fbdf4f1e..704ab731 100644 --- a/lib/addons/prototypes/analytics.js +++ b/lib/addons/prototypes/analytics.js @@ -5,6 +5,91 @@ const STATUS = { NO_BID: "NO_BID", TIMEOUT: "TIMEOUT", }; + +// Mirrors lib/core/traffic-source.ts. Inlined because this prototype bundle is +// self-contained and pulls in no modules. +const TRAFFIC_SOURCE_KEY = "optableTrafficSource"; +const SEARCH_SITES = ["google", "bing", "yahoo", "duckduckgo", "ecosia", "baidu", "yandex", "qwant", "startpage"]; +const SOCIAL_SITES = [ + "facebook", + "fb", + "instagram", + "twitter", + "x", + "t", + "linkedin", + "lnkd", + "reddit", + "pinterest", + "tiktok", + "threads", + "snapchat", + "youtube", +]; +const PAID_CLICK_IDS = ["gclid", "gbraid", "wbraid", "fbclid", "msclkid", "ttclid", "twclid", "dclid"]; +const PAID_MEDIUMS = ["cpc", "ppc", "paid", "paidsocial", "paid_social", "display", "cpm"]; +const TWO_PART_SUFFIXES = ["co.uk", "com.au", "co.jp", "co.in", "com.br", "co.nz", "com.mx", "co.za", "co.kr"]; + +function siteName(host) { + const labels = host.toLowerCase().split("."); + if (labels.length < 2) return labels[0] || ""; + + const index = TWO_PART_SUFFIXES.includes(labels.slice(-2).join(".")) ? labels.length - 3 : labels.length - 2; + return labels[index] || ""; +} + +function referrerHost() { + if (!document.referrer) return ""; + + try { + return new URL(document.referrer).hostname; + } catch { + return ""; + } +} + +function classifyTrafficSource(host) { + const params = new URLSearchParams(window.location.search); + const medium = (params.get("utm_medium") || "").toLowerCase(); + + if (PAID_CLICK_IDS.some((id) => params.has(id)) || PAID_MEDIUMS.includes(medium)) return "paid"; + if (medium === "email") return "email"; + if (params.has("utm_source")) return "campaign"; + if (!host) return "direct"; + if (host === window.location.hostname) return "internal"; + + const site = siteName(host); + if (SEARCH_SITES.includes(site)) return "organic_search"; + if (SOCIAL_SITES.includes(site)) return "social"; + + return "referral"; +} + +// First-touch per tab: document.referrer becomes our own hostname after the +// first internal click, so the first non-internal result is cached and reused. +// Only the referring hostname is kept, never the full referring URL. +function getTrafficSource() { + try { + const cached = window.sessionStorage?.getItem(TRAFFIC_SOURCE_KEY); + if (cached) return JSON.parse(cached); + } catch { + // sessionStorage unavailable or holding malformed JSON, recompute below. + } + + const referrer = referrerHost(); + const properties = { referrer, trafficSource: classifyTrafficSource(referrer) }; + + if (properties.trafficSource !== "internal") { + try { + window.sessionStorage?.setItem(TRAFFIC_SOURCE_KEY, JSON.stringify(properties)); + } catch { + // Storage blocked or full, the value is still returned for this event. + } + } + + return properties; +} + class OptablePrebidAnalytics { constructor(optableInstance, config = {}) { if (!optableInstance || typeof optableInstance.witness !== "function") { @@ -276,6 +361,7 @@ class OptablePrebidAnalytics { liveintentEIDs: Array.from(lSourcesSet), missed, url: `${window.location.hostname}${window.location.pathname}`, + ...getTrafficSource(), tenant: this.config.tenant, optableWrapperVersion: SDK_WRAPPER_VERSION, // eslint-disable-line no-undef prebidjsVersion: this.prebidInstance?.version || "unknown", diff --git a/lib/core/traffic-source.ts b/lib/core/traffic-source.ts new file mode 100644 index 00000000..06d29441 --- /dev/null +++ b/lib/core/traffic-source.ts @@ -0,0 +1,105 @@ +const TRAFFIC_SOURCE_KEY = "optableTrafficSource"; + +const SEARCH_SITES = ["google", "bing", "yahoo", "duckduckgo", "ecosia", "baidu", "yandex", "qwant", "startpage"]; +const SOCIAL_SITES = [ + "facebook", + "fb", + "instagram", + "twitter", + "x", + "t", + "linkedin", + "lnkd", + "reddit", + "pinterest", + "tiktok", + "threads", + "snapchat", + "youtube", +]; +const PAID_CLICK_IDS = ["gclid", "gbraid", "wbraid", "fbclid", "msclkid", "ttclid", "twclid", "dclid"]; +const PAID_MEDIUMS = ["cpc", "ppc", "paid", "paidsocial", "paid_social", "display", "cpm"]; + +// Suffixes that take two labels, so the registrable name is the third from the +// right (google.co.uk -> google) rather than the second (-> co). +const TWO_PART_SUFFIXES = ["co.uk", "com.au", "co.jp", "co.in", "com.br", "co.nz", "com.mx", "co.za", "co.kr"]; + +type TrafficSource = "direct" | "internal" | "paid" | "campaign" | "email" | "organic_search" | "social" | "referral"; + +type TrafficSourceProperties = { + referrer: string; + trafficSource: TrafficSource; +}; + +// Registrable name of a host: "news.google.co.uk" -> "google", "t.co" -> "t". +// Approximate, since resolving this exactly needs the public suffix list. +function siteName(host: string): string { + const labels = host.toLowerCase().split("."); + if (labels.length < 2) return labels[0] || ""; + + const index = TWO_PART_SUFFIXES.includes(labels.slice(-2).join(".")) ? labels.length - 3 : labels.length - 2; + return labels[index] || ""; +} + +function referrerHost(): string { + if (!document.referrer) return ""; + + try { + return new URL(document.referrer).hostname; + } catch { + return ""; + } +} + +function classify(host: string): TrafficSource { + const params = new URLSearchParams(window.location.search); + const medium = (params.get("utm_medium") || "").toLowerCase(); + + if (PAID_CLICK_IDS.some((id) => params.has(id)) || PAID_MEDIUMS.includes(medium)) return "paid"; + if (medium === "email") return "email"; + if (params.has("utm_source")) return "campaign"; + if (!host) return "direct"; + if (host === window.location.hostname) return "internal"; + + const site = siteName(host); + if (SEARCH_SITES.includes(site)) return "organic_search"; + if (SOCIAL_SITES.includes(site)) return "social"; + + return "referral"; +} + +/** + * Referring host and classified traffic source for the current page. + * + * The result is first-touch per tab: the first non-internal classification is + * cached in `sessionStorage` and reused for the rest of the session. Without + * that, `document.referrer` becomes the site's own hostname after the first + * internal click and every later event would report `internal`. + * + * Only the referring hostname is reported, never the full referring URL, which + * can carry query strings with personal data. + */ +function getTrafficSource(): TrafficSourceProperties { + try { + const cached = window.sessionStorage?.getItem(TRAFFIC_SOURCE_KEY); + if (cached) return JSON.parse(cached) as TrafficSourceProperties; + } catch { + // sessionStorage unavailable or holding malformed JSON, fall through and recompute. + } + + const referrer = referrerHost(); + const properties: TrafficSourceProperties = { referrer, trafficSource: classify(referrer) }; + + if (properties.trafficSource !== "internal") { + try { + window.sessionStorage?.setItem(TRAFFIC_SOURCE_KEY, JSON.stringify(properties)); + } catch { + // Storage blocked or full, the value is still returned for this event. + } + } + + return properties; +} + +export type { TrafficSource, TrafficSourceProperties }; +export { getTrafficSource, TRAFFIC_SOURCE_KEY }; From 9e729f006d0ac29eb47cd5e321db2e677b0eb897 Mon Sep 17 00:00:00 2001 From: MO Thibault <103271673+MO-Thibault@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:28:35 -0400 Subject: [PATCH 2/2] analytics: address traffic source review feedback - prototypes/analytics.js imports getTrafficSource from lib/core instead of carrying an inlined copy. Consumers import it from lib/dist through a bundler, where core/traffic-source.js is emitted alongside it. - Internal is checked first, so UTM-tagged links between the site's own pages are no longer reported as campaign. - Internal matches on the registrable name, so another subdomain of the same site is internal rather than referral. - A cached direct is replaced by a later non-internal source in the same tab. - Add traffic-source.test.ts. Co-Authored-By: Claude Opus 5.5 (1M context) --- lib/addons/prebid/analytics.md | 5 +- lib/addons/prototypes/analytics.js | 86 +------------------ lib/core/traffic-source.test.ts | 131 +++++++++++++++++++++++++++++ lib/core/traffic-source.ts | 30 ++++--- 4 files changed, 157 insertions(+), 95 deletions(-) create mode 100644 lib/core/traffic-source.test.ts diff --git a/lib/addons/prebid/analytics.md b/lib/addons/prebid/analytics.md index cf85c541..b0d43567 100644 --- a/lib/addons/prebid/analytics.md +++ b/lib/addons/prebid/analytics.md @@ -138,4 +138,7 @@ Each sampled auction sends an `optable.prebid.auction` event with, among others: first-touch per tab: the first non-internal result is cached in `sessionStorage` under `optableTrafficSource` and reused for the rest of the session, because `document.referrer` becomes the site's own hostname after the first internal -click. +click. A cached `direct` is the exception: a later non-internal result in the +same tab replaces it. A referrer counts as `internal` when its registrable name +matches the page's (`news.site.com` on `www.site.com`), and that check runs +before the UTM and click-id checks. diff --git a/lib/addons/prototypes/analytics.js b/lib/addons/prototypes/analytics.js index 704ab731..7dac7dc5 100644 --- a/lib/addons/prototypes/analytics.js +++ b/lib/addons/prototypes/analytics.js @@ -1,4 +1,6 @@ /* eslint-disable no-param-reassign */ +import { getTrafficSource } from "../../core/traffic-source"; + const STATUS = { REQUESTED: "REQUESTED", RECEIVED: "RECEIVED", @@ -6,90 +8,6 @@ const STATUS = { TIMEOUT: "TIMEOUT", }; -// Mirrors lib/core/traffic-source.ts. Inlined because this prototype bundle is -// self-contained and pulls in no modules. -const TRAFFIC_SOURCE_KEY = "optableTrafficSource"; -const SEARCH_SITES = ["google", "bing", "yahoo", "duckduckgo", "ecosia", "baidu", "yandex", "qwant", "startpage"]; -const SOCIAL_SITES = [ - "facebook", - "fb", - "instagram", - "twitter", - "x", - "t", - "linkedin", - "lnkd", - "reddit", - "pinterest", - "tiktok", - "threads", - "snapchat", - "youtube", -]; -const PAID_CLICK_IDS = ["gclid", "gbraid", "wbraid", "fbclid", "msclkid", "ttclid", "twclid", "dclid"]; -const PAID_MEDIUMS = ["cpc", "ppc", "paid", "paidsocial", "paid_social", "display", "cpm"]; -const TWO_PART_SUFFIXES = ["co.uk", "com.au", "co.jp", "co.in", "com.br", "co.nz", "com.mx", "co.za", "co.kr"]; - -function siteName(host) { - const labels = host.toLowerCase().split("."); - if (labels.length < 2) return labels[0] || ""; - - const index = TWO_PART_SUFFIXES.includes(labels.slice(-2).join(".")) ? labels.length - 3 : labels.length - 2; - return labels[index] || ""; -} - -function referrerHost() { - if (!document.referrer) return ""; - - try { - return new URL(document.referrer).hostname; - } catch { - return ""; - } -} - -function classifyTrafficSource(host) { - const params = new URLSearchParams(window.location.search); - const medium = (params.get("utm_medium") || "").toLowerCase(); - - if (PAID_CLICK_IDS.some((id) => params.has(id)) || PAID_MEDIUMS.includes(medium)) return "paid"; - if (medium === "email") return "email"; - if (params.has("utm_source")) return "campaign"; - if (!host) return "direct"; - if (host === window.location.hostname) return "internal"; - - const site = siteName(host); - if (SEARCH_SITES.includes(site)) return "organic_search"; - if (SOCIAL_SITES.includes(site)) return "social"; - - return "referral"; -} - -// First-touch per tab: document.referrer becomes our own hostname after the -// first internal click, so the first non-internal result is cached and reused. -// Only the referring hostname is kept, never the full referring URL. -function getTrafficSource() { - try { - const cached = window.sessionStorage?.getItem(TRAFFIC_SOURCE_KEY); - if (cached) return JSON.parse(cached); - } catch { - // sessionStorage unavailable or holding malformed JSON, recompute below. - } - - const referrer = referrerHost(); - const properties = { referrer, trafficSource: classifyTrafficSource(referrer) }; - - if (properties.trafficSource !== "internal") { - try { - window.sessionStorage?.setItem(TRAFFIC_SOURCE_KEY, JSON.stringify(properties)); - } catch { - // Storage blocked or full, the value is still returned for this event. - } - } - - return properties; -} - class OptablePrebidAnalytics { constructor(optableInstance, config = {}) { if (!optableInstance || typeof optableInstance.witness !== "function") { diff --git a/lib/core/traffic-source.test.ts b/lib/core/traffic-source.test.ts new file mode 100644 index 00000000..56cb5abe --- /dev/null +++ b/lib/core/traffic-source.test.ts @@ -0,0 +1,131 @@ +import { getTrafficSource, TRAFFIC_SOURCE_KEY } from "./traffic-source"; + +function visit(url: string, referrer = "") { + const { hostname, search } = new URL(url); + Object.defineProperty(window, "location", { + value: { hostname, search }, + writable: true, + configurable: true, + }); + Object.defineProperty(document, "referrer", { value: referrer, configurable: true }); +} + +beforeEach(() => { + sessionStorage.clear(); +}); + +describe("getTrafficSource - classification", () => { + it("reports direct when there is no referrer", () => { + visit("https://www.site.com/"); + expect(getTrafficSource()).toEqual({ referrer: "", trafficSource: "direct" }); + }); + + it("reports direct when the referrer is not a valid URL", () => { + visit("https://www.site.com/", "not a url"); + expect(getTrafficSource()).toEqual({ referrer: "", trafficSource: "direct" }); + }); + + it("reports internal for the same host", () => { + visit("https://www.site.com/b", "https://www.site.com/a"); + expect(getTrafficSource().trafficSource).toBe("internal"); + }); + + it("reports internal for another subdomain of the same site", () => { + visit("https://www.site.com/b", "https://news.site.com/a"); + expect(getTrafficSource().trafficSource).toBe("internal"); + }); + + it("reports internal for subdomains under a two-part suffix", () => { + visit("https://www.site.co.uk/b", "https://news.site.co.uk/a"); + expect(getTrafficSource().trafficSource).toBe("internal"); + }); + + it("reports internal before campaign when an internal link carries UTM params", () => { + visit("https://www.site.com/b?utm_source=newsletter&utm_medium=email", "https://www.site.com/a"); + expect(getTrafficSource().trafficSource).toBe("internal"); + }); + + it("reports paid for a click id", () => { + visit("https://www.site.com/?gclid=abc", "https://www.google.com/"); + expect(getTrafficSource()).toEqual({ referrer: "www.google.com", trafficSource: "paid" }); + }); + + it("reports paid for a paid utm_medium, case-insensitively", () => { + visit("https://www.site.com/?utm_source=fb&utm_medium=CPC"); + expect(getTrafficSource().trafficSource).toBe("paid"); + }); + + it("reports email for utm_medium=email", () => { + visit("https://www.site.com/?utm_source=newsletter&utm_medium=email"); + expect(getTrafficSource().trafficSource).toBe("email"); + }); + + it("reports campaign for utm_source without a paid or email medium", () => { + visit("https://www.site.com/?utm_source=partner", "https://partner.com/"); + expect(getTrafficSource().trafficSource).toBe("campaign"); + }); + + it("reports organic_search for a search engine", () => { + visit("https://www.site.com/", "https://www.google.co.uk/"); + expect(getTrafficSource()).toEqual({ referrer: "www.google.co.uk", trafficSource: "organic_search" }); + }); + + it("reports social for a social network", () => { + visit("https://www.site.com/", "https://t.co/xyz"); + expect(getTrafficSource()).toEqual({ referrer: "t.co", trafficSource: "social" }); + }); + + it("reports referral for any other site", () => { + visit("https://www.site.com/", "https://blog.example.org/post?email=a@b.c"); + expect(getTrafficSource()).toEqual({ referrer: "blog.example.org", trafficSource: "referral" }); + }); +}); + +describe("getTrafficSource - session caching", () => { + it("keeps the first external source across internal navigation", () => { + visit("https://www.site.com/a", "https://www.google.com/"); + expect(getTrafficSource().trafficSource).toBe("organic_search"); + + visit("https://www.site.com/b", "https://www.site.com/a"); + expect(getTrafficSource()).toEqual({ referrer: "www.google.com", trafficSource: "organic_search" }); + }); + + it("does not replace a cached external source with a later one", () => { + visit("https://www.site.com/a", "https://www.google.com/"); + getTrafficSource(); + + visit("https://www.site.com/b", "https://www.facebook.com/"); + expect(getTrafficSource().trafficSource).toBe("organic_search"); + }); + + it("does not cache internal", () => { + visit("https://www.site.com/b", "https://www.site.com/a"); + expect(getTrafficSource().trafficSource).toBe("internal"); + expect(sessionStorage.getItem(TRAFFIC_SOURCE_KEY)).toBeNull(); + }); + + it("keeps a cached direct across internal navigation", () => { + visit("https://www.site.com/a"); + getTrafficSource(); + + visit("https://www.site.com/b", "https://www.site.com/a"); + expect(getTrafficSource()).toEqual({ referrer: "", trafficSource: "direct" }); + }); + + it("replaces a cached direct with a later external source", () => { + visit("https://www.site.com/a"); + getTrafficSource(); + + visit("https://www.site.com/b", "https://www.facebook.com/"); + expect(getTrafficSource()).toEqual({ referrer: "www.facebook.com", trafficSource: "social" }); + + visit("https://www.site.com/c", "https://www.site.com/b"); + expect(getTrafficSource().trafficSource).toBe("social"); + }); + + it("recomputes when the cached value is malformed", () => { + sessionStorage.setItem(TRAFFIC_SOURCE_KEY, "{not json"); + visit("https://www.site.com/", "https://www.bing.com/"); + expect(getTrafficSource().trafficSource).toBe("organic_search"); + }); +}); diff --git a/lib/core/traffic-source.ts b/lib/core/traffic-source.ts index 06d29441..3fe26d6b 100644 --- a/lib/core/traffic-source.ts +++ b/lib/core/traffic-source.ts @@ -51,7 +51,13 @@ function referrerHost(): string { } } +// Internal is checked first so that UTM-tagged links between the site's own +// pages are not reported as campaign traffic. Matching on the registrable name +// rather than the exact host keeps www.site.com -> news.site.com internal. function classify(host: string): TrafficSource { + const site = siteName(host); + if (host && site === siteName(window.location.hostname)) return "internal"; + const params = new URLSearchParams(window.location.search); const medium = (params.get("utm_medium") || "").toLowerCase(); @@ -59,9 +65,7 @@ function classify(host: string): TrafficSource { if (medium === "email") return "email"; if (params.has("utm_source")) return "campaign"; if (!host) return "direct"; - if (host === window.location.hostname) return "internal"; - const site = siteName(host); if (SEARCH_SITES.includes(site)) return "organic_search"; if (SOCIAL_SITES.includes(site)) return "social"; @@ -76,26 +80,32 @@ function classify(host: string): TrafficSource { * that, `document.referrer` becomes the site's own hostname after the first * internal click and every later event would report `internal`. * + * A cached `direct` is the one exception. It is the weakest signal (in-app + * browsers and `no-referrer` policies also land there), so a later non-internal + * classification in the same tab replaces it. + * * Only the referring hostname is reported, never the full referring URL, which * can carry query strings with personal data. */ function getTrafficSource(): TrafficSourceProperties { + let cached: TrafficSourceProperties | null = null; try { - const cached = window.sessionStorage?.getItem(TRAFFIC_SOURCE_KEY); - if (cached) return JSON.parse(cached) as TrafficSourceProperties; + const stored = window.sessionStorage?.getItem(TRAFFIC_SOURCE_KEY); + if (stored) cached = JSON.parse(stored) as TrafficSourceProperties; } catch { // sessionStorage unavailable or holding malformed JSON, fall through and recompute. } + if (cached && cached.trafficSource !== "direct") return cached; const referrer = referrerHost(); const properties: TrafficSourceProperties = { referrer, trafficSource: classify(referrer) }; - if (properties.trafficSource !== "internal") { - try { - window.sessionStorage?.setItem(TRAFFIC_SOURCE_KEY, JSON.stringify(properties)); - } catch { - // Storage blocked or full, the value is still returned for this event. - } + if (properties.trafficSource === "internal") return cached ?? properties; + + try { + window.sessionStorage?.setItem(TRAFFIC_SOURCE_KEY, JSON.stringify(properties)); + } catch { + // Storage blocked or full, the value is still returned for this event. } return properties;