analytics: report referrer and traffic source in both analytics versions

lib/addons/prebid/analytics.md

@@ -128,7 +128,17 @@ Returns the `OptablePrebidAnalytics` instance, or `null` when no Prebid instance
 
 Each sampled auction sends an `optable.prebid.auction` event with, among others:
 `bidderRequests` (with per-bid `status`, `cpm`, `size`, `splitTestAssignment`),
-`optableMatchers`, `optableSources`, `optableTargetingDone`, `liveintentEIDs`
-(with per-bidder `hasLiEids` and `liSources`), `bidWon`, `missed`,
-`url`, `tenant`, `prebidjsVersion`, `sessionDepth`, `pageAuctionsCount`,
-`originSlug`, and the parsed `userAgent`/`device`.
+`optableMatchers`, `optableSources`, `optableTargetingDone`, `bidWon`, `missed`,
+`url`, `referrer`, `trafficSource`, `tenant`, `prebidjsVersion`, `sessionDepth`,
+`pageAuctionsCount`, `originSlug`, and the parsed `userAgent`/`device`.
+
+`referrer` is the referring hostname only, never the full referring URL.
+`trafficSource` classifies it as one of `direct`, `internal`, `paid`,
+`campaign`, `email`, `organic_search`, `social` or `referral`. Both are
+first-touch per tab: the first non-internal result is cached in `sessionStorage`
+under `optableTrafficSource` and reused for the rest of the session, because
+`document.referrer` becomes the site's own hostname after the first internal
+click. A cached `direct` is the exception: a later non-internal result in the
+same tab replaces it. A referrer counts as `internal` when its registrable name
+matches the page's (`news.site.com` on `www.site.com`), and that check runs
+before the UTM and click-id checks.

lib/addons/prebid/analytics.ts

@@ -3,6 +3,7 @@
 import type { WitnessProperties } from "../../edge/witness";
 import type OptableSDK from "../../sdk";
 import { buildRequest } from "../../core/network";
+import { getTrafficSource } from "../../core/traffic-source";
 
 import * as Bowser from "bowser";
 
@@ -494,6 +495,7 @@ class OptablePrebidAnalytics {
       })),
       missed,
       url: `${window.location.hostname}${window.location.pathname}`,
+      ...getTrafficSource(),
       tenant: this.optableInstance?.dcn?.node ?? "unknown",
       // eslint-disable-next-line no-undef
       optableWrapperVersion: SDK_WRAPPER_VERSION || "unknown",

lib/addons/prototypes/analytics.js

@@ -1,10 +1,13 @@
 /* eslint-disable no-param-reassign */
+import { getTrafficSource } from "../../core/traffic-source";
+
 const STATUS = {
   REQUESTED: "REQUESTED",
   RECEIVED: "RECEIVED",
   NO_BID: "NO_BID",
   TIMEOUT: "TIMEOUT",
 };
+
 class OptablePrebidAnalytics {
   constructor(optableInstance, config = {}) {
     if (!optableInstance || typeof optableInstance.witness !== "function") {
@@ -276,6 +279,7 @@ class OptablePrebidAnalytics {
         liveintentEIDs: Array.from(lSourcesSet),
         missed,
         url: `${window.location.hostname}${window.location.pathname}`,
+        ...getTrafficSource(),
         tenant: this.config.tenant,
         optableWrapperVersion: SDK_WRAPPER_VERSION, // eslint-disable-line no-undef
         prebidjsVersion: this.prebidInstance?.version || "unknown",

lib/core/traffic-source.test.ts

@@ -0,0 +1,131 @@
+import { getTrafficSource, TRAFFIC_SOURCE_KEY } from "./traffic-source";
+
+function visit(url: string, referrer = "") {
+  const { hostname, search } = new URL(url);
+  Object.defineProperty(window, "location", {
+    value: { hostname, search },
+    writable: true,
+    configurable: true,
+  });
+  Object.defineProperty(document, "referrer", { value: referrer, configurable: true });
+}
+
+beforeEach(() => {
+  sessionStorage.clear();
+});
+
+describe("getTrafficSource - classification", () => {
+  it("reports direct when there is no referrer", () => {
+    visit("https://www.site.com/");
+    expect(getTrafficSource()).toEqual({ referrer: "", trafficSource: "direct" });
+  });
+
+  it("reports direct when the referrer is not a valid URL", () => {
+    visit("https://www.site.com/", "not a url");
+    expect(getTrafficSource()).toEqual({ referrer: "", trafficSource: "direct" });
+  });
+
+  it("reports internal for the same host", () => {
+    visit("https://www.site.com/b", "https://www.site.com/a");
+    expect(getTrafficSource().trafficSource).toBe("internal");
+  });
+
+  it("reports internal for another subdomain of the same site", () => {
+    visit("https://www.site.com/b", "https://news.site.com/a");
+    expect(getTrafficSource().trafficSource).toBe("internal");
+  });
+
+  it("reports internal for subdomains under a two-part suffix", () => {
+    visit("https://www.site.co.uk/b", "https://news.site.co.uk/a");
+    expect(getTrafficSource().trafficSource).toBe("internal");
+  });
+
+  it("reports internal before campaign when an internal link carries UTM params", () => {
+    visit("https://www.site.com/b?utm_source=newsletter&utm_medium=email", "https://www.site.com/a");
+    expect(getTrafficSource().trafficSource).toBe("internal");
+  });
+
+  it("reports paid for a click id", () => {
+    visit("https://www.site.com/?gclid=abc", "https://www.google.com/");
+    expect(getTrafficSource()).toEqual({ referrer: "www.google.com", trafficSource: "paid" });
+  });
+
+  it("reports paid for a paid utm_medium, case-insensitively", () => {
+    visit("https://www.site.com/?utm_source=fb&utm_medium=CPC");
+    expect(getTrafficSource().trafficSource).toBe("paid");
+  });
+
+  it("reports email for utm_medium=email", () => {
+    visit("https://www.site.com/?utm_source=newsletter&utm_medium=email");
+    expect(getTrafficSource().trafficSource).toBe("email");
+  });
+
+  it("reports campaign for utm_source without a paid or email medium", () => {
+    visit("https://www.site.com/?utm_source=partner", "https://partner.com/");
+    expect(getTrafficSource().trafficSource).toBe("campaign");
+  });
+
+  it("reports organic_search for a search engine", () => {
+    visit("https://www.site.com/", "https://www.google.co.uk/");
+    expect(getTrafficSource()).toEqual({ referrer: "www.google.co.uk", trafficSource: "organic_search" });
+  });
+
+  it("reports social for a social network", () => {
+    visit("https://www.site.com/", "https://t.co/xyz");
+    expect(getTrafficSource()).toEqual({ referrer: "t.co", trafficSource: "social" });
+  });
+
+  it("reports referral for any other site", () => {
+    visit("https://www.site.com/", "https://blog.example.org/post?email=a@b.c");
+    expect(getTrafficSource()).toEqual({ referrer: "blog.example.org", trafficSource: "referral" });
+  });
+});
+
+describe("getTrafficSource - session caching", () => {
+  it("keeps the first external source across internal navigation", () => {
+    visit("https://www.site.com/a", "https://www.google.com/");
+    expect(getTrafficSource().trafficSource).toBe("organic_search");
+
+    visit("https://www.site.com/b", "https://www.site.com/a");
+    expect(getTrafficSource()).toEqual({ referrer: "www.google.com", trafficSource: "organic_search" });
+  });
+
+  it("does not replace a cached external source with a later one", () => {
+    visit("https://www.site.com/a", "https://www.google.com/");
+    getTrafficSource();
+
+    visit("https://www.site.com/b", "https://www.facebook.com/");
+    expect(getTrafficSource().trafficSource).toBe("organic_search");
+  });
+
+  it("does not cache internal", () => {
+    visit("https://www.site.com/b", "https://www.site.com/a");
+    expect(getTrafficSource().trafficSource).toBe("internal");
+    expect(sessionStorage.getItem(TRAFFIC_SOURCE_KEY)).toBeNull();
+  });
+
+  it("keeps a cached direct across internal navigation", () => {
+    visit("https://www.site.com/a");
+    getTrafficSource();
+
+    visit("https://www.site.com/b", "https://www.site.com/a");
+    expect(getTrafficSource()).toEqual({ referrer: "", trafficSource: "direct" });
+  });
+
+  it("replaces a cached direct with a later external source", () => {
+    visit("https://www.site.com/a");
+    getTrafficSource();
+
+    visit("https://www.site.com/b", "https://www.facebook.com/");
+    expect(getTrafficSource()).toEqual({ referrer: "www.facebook.com", trafficSource: "social" });
+
+    visit("https://www.site.com/c", "https://www.site.com/b");
+    expect(getTrafficSource().trafficSource).toBe("social");
+  });
+
+  it("recomputes when the cached value is malformed", () => {
+    sessionStorage.setItem(TRAFFIC_SOURCE_KEY, "{not json");
+    visit("https://www.site.com/", "https://www.bing.com/");
+    expect(getTrafficSource().trafficSource).toBe("organic_search");
+  });
+});

lib/core/traffic-source.ts

@@ -0,0 +1,115 @@
+const TRAFFIC_SOURCE_KEY = "optableTrafficSource";
+
+const SEARCH_SITES = ["google", "bing", "yahoo", "duckduckgo", "ecosia", "baidu", "yandex", "qwant", "startpage"];
+const SOCIAL_SITES = [
+  "facebook",
+  "fb",
+  "instagram",
+  "twitter",
+  "x",
+  "t",
+  "linkedin",
+  "lnkd",
+  "reddit",
+  "pinterest",
+  "tiktok",
+  "threads",
+  "snapchat",
+  "youtube",
+];
+const PAID_CLICK_IDS = ["gclid", "gbraid", "wbraid", "fbclid", "msclkid", "ttclid", "twclid", "dclid"];
+const PAID_MEDIUMS = ["cpc", "ppc", "paid", "paidsocial", "paid_social", "display", "cpm"];
+
+// Suffixes that take two labels, so the registrable name is the third from the
+// right (google.co.uk -> google) rather than the second (-> co).
+const TWO_PART_SUFFIXES = ["co.uk", "com.au", "co.jp", "co.in", "com.br", "co.nz", "com.mx", "co.za", "co.kr"];
+
+type TrafficSource = "direct" | "internal" | "paid" | "campaign" | "email" | "organic_search" | "social" | "referral";
+
+type TrafficSourceProperties = {
+  referrer: string;
+  trafficSource: TrafficSource;
+};
+
+// Registrable name of a host: "news.google.co.uk" -> "google", "t.co" -> "t".
+// Approximate, since resolving this exactly needs the public suffix list.
+function siteName(host: string): string {
+  const labels = host.toLowerCase().split(".");
+  if (labels.length < 2) return labels[0] || "";
+
+  const index = TWO_PART_SUFFIXES.includes(labels.slice(-2).join(".")) ? labels.length - 3 : labels.length - 2;
+  return labels[index] || "";
+}
+
+function referrerHost(): string {
+  if (!document.referrer) return "";
+
+  try {
+    return new URL(document.referrer).hostname;
+  } catch {
+    return "";
+  }
+}
+
+// Internal is checked first so that UTM-tagged links between the site's own
+// pages are not reported as campaign traffic. Matching on the registrable name
+// rather than the exact host keeps www.site.com -> news.site.com internal.
+function classify(host: string): TrafficSource {
+  const site = siteName(host);
+  if (host && site === siteName(window.location.hostname)) return "internal";
+
+  const params = new URLSearchParams(window.location.search);
+  const medium = (params.get("utm_medium") || "").toLowerCase();
+
+  if (PAID_CLICK_IDS.some((id) => params.has(id)) || PAID_MEDIUMS.includes(medium)) return "paid";
+  if (medium === "email") return "email";
+  if (params.has("utm_source")) return "campaign";
+  if (!host) return "direct";
+
+  if (SEARCH_SITES.includes(site)) return "organic_search";
+  if (SOCIAL_SITES.includes(site)) return "social";
+
+  return "referral";
+}
+
+/**
+ * Referring host and classified traffic source for the current page.
+ *
+ * The result is first-touch per tab: the first non-internal classification is
+ * cached in `sessionStorage` and reused for the rest of the session. Without
+ * that, `document.referrer` becomes the site's own hostname after the first
+ * internal click and every later event would report `internal`.
+ *
+ * A cached `direct` is the one exception. It is the weakest signal (in-app
+ * browsers and `no-referrer` policies also land there), so a later non-internal
+ * classification in the same tab replaces it.
+ *
+ * Only the referring hostname is reported, never the full referring URL, which
+ * can carry query strings with personal data.
+ */
+function getTrafficSource(): TrafficSourceProperties {
+  let cached: TrafficSourceProperties | null = null;
+  try {
+    const stored = window.sessionStorage?.getItem(TRAFFIC_SOURCE_KEY);
+    if (stored) cached = JSON.parse(stored) as TrafficSourceProperties;
+  } catch {
+    // sessionStorage unavailable or holding malformed JSON, fall through and recompute.
+  }
+  if (cached && cached.trafficSource !== "direct") return cached;
+
+  const referrer = referrerHost();
+  const properties: TrafficSourceProperties = { referrer, trafficSource: classify(referrer) };
+
+  if (properties.trafficSource === "internal") return cached ?? properties;
+
+  try {
+    window.sessionStorage?.setItem(TRAFFIC_SOURCE_KEY, JSON.stringify(properties));
+  } catch {
+    // Storage blocked or full, the value is still returned for this event.
+  }
+
+  return properties;
+}
+
+export type { TrafficSource, TrafficSourceProperties };
+export { getTrafficSource, TRAFFIC_SOURCE_KEY };