Submit 9lives sitemap URLs through IndexNow after production deploys

.github/workflows/indexnow.yml

@@ -0,0 +1,47 @@
+name: IndexNow
+
+on:
+  pull_request:
+    paths:
+      - 'website/**'
+      - '.github/workflows/indexnow.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'website/**'
+      - '.github/workflows/indexnow.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: indexnow-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22'
+      - run: node --test website/scripts/indexnow.check.mjs
+
+  submit:
+    if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
+    needs: test
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: '22'
+      - name: Wait for this production deployment and submit public URLs
+        run: node website/scripts/indexnow.mjs submit

docs/INDEXNOW.md

@@ -0,0 +1,33 @@
+# IndexNow
+
+Set a unique `INDEXNOW_KEY` in the production hosting environment for this
+site. Generate 32 random hexadecimal characters with a password manager; do
+not reuse another site's key or commit it. Preview/local builds can omit it.
+
+The build writes `/indexnow-key.txt` and `/indexnow-deployment.txt` into the
+public output. Both generated files are gitignored. The first is the UTF-8
+ownership proof required by IndexNow; the second identifies the deployed
+commit. Treat the key as a credential even though search engines must be able
+to fetch its verification file.
+
+After changes to the site merge to `main`, the IndexNow GitHub workflow waits
+for up to 15 minutes for the canonical production host to serve that exact
+commit. It retrieves the ownership proof without logging it, then submits
+the current flat sitemap's public URLs to `https://api.indexnow.org/indexnow`.
+There is no browser key or GitHub secret, and preview deployments never submit.
+The sitemap must contain only HTTPS URLs on this site's canonical host, without
+query strings or fragments. Keep private/authenticated pages out of it.
+
+The workflow can also be run manually on `main` after a hosting configuration
+change or redeployment. A missing key, failed deployment, or stale production
+revision makes submission fail without sending URLs. HTTP 200 means the URLs
+were received; 202 means ownership validation is pending. Neither guarantees
+indexing. Check Bing Webmaster Tools > IndexNow for receipt evidence.
+
+This small static-site integration submits the current sitemap after a site
+deploy. If removing a page, submit its deleted canonical URL separately before
+removing it from the sitemap. Sitemap indexes are not supported.
+
+Run the focused checks with `node --test website/scripts/indexnow.check.mjs`.
+
+Protocol: https://www.indexnow.org/documentation

website/.gitignore

@@ -36,3 +36,7 @@ next-env.d.ts
 /.wrangler/
 /outputs/
 /work/
+
+# Generated IndexNow ownership files (never commit credentials)
+/public/indexnow-key.txt
+/public/indexnow-deployment.txt

website/package.json

@@ -11,7 +11,9 @@
     "build:vercel": "next build",
     "start": "vinext start",
     "lint": "eslint . --ignore-pattern dist --ignore-pattern .next",
-    "test": "vitest run"
+    "test": "vitest run",
+    "prebuild": "node scripts/indexnow.mjs prepare",
+    "prebuild:vercel": "node scripts/indexnow.mjs prepare"
   },
   "dependencies": {
     "next": "^16.3.3",

website/scripts/indexnow.check.mjs

@@ -0,0 +1,101 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { randomBytes } from 'node:crypto';
+import { readFile, rm } from 'node:fs/promises';
+import { origin, validKey, sitemapUrls, prepare, submit } from './indexnow.mjs';
+
+const key = randomBytes(16).toString('hex');
+const revision = 'a'.repeat(40);
+const publicDir = new URL('../public/', import.meta.url);
+const xml = (urls) => `<urlset>${urls.map((url) => `<url><loc>${url}</loc></url>`).join('')}</urlset>`;
+
+test('canonical sitemap rejects foreign hosts, private query URLs, fragments and empty lists', () => {
+  assert.deepEqual(sitemapUrls(xml([`${origin}/`, `${origin}/`])), [`${origin}/`]);
+  for (const urls of [[], ['https://example.net/'], [`${origin}/?token=private`],
+    [`${origin}/#fragment`], [origin.replace('https:', 'http:')]]) {
+    assert.throws(() => sitemapUrls(xml(urls)));
+  }
+  assert.throws(() => sitemapUrls('<sitemapindex><loc>https://example.net/</loc></sitemapindex>'));
+  assert.throws(() => sitemapUrls('<!DOCTYPE urlset [<!ENTITY key SYSTEM "file:///etc/passwd">]><urlset/>'));
+  assert.throws(() => sitemapUrls(xml(Array.from({length: 10001}, (_, n) => `${origin}/${n}`))));
+});
+
+test('key validation rejects whitespace, unsafe characters and invalid lengths', () => {
+  assert.equal(validKey(key), true);
+  for (const candidate of [undefined, '', 'short', key + '\n', 'x'.repeat(129), 'a/b'.repeat(8)]) {
+    assert.equal(validKey(candidate), false);
+  }
+});
+
+test('build writes UTF-8 ownership files and clears stale files when disabled', async () => {
+  const keyFile = new URL('indexnow-key.txt', publicDir);
+  const revisionFile = new URL('indexnow-deployment.txt', publicDir);
+  try {
+    await prepare({ INDEXNOW_KEY: key, VERCEL_GIT_COMMIT_SHA: revision });
+    assert.equal(await readFile(keyFile, 'utf8'), key);
+    assert.equal(await readFile(revisionFile, 'utf8'), revision);
+    await prepare({});
+    await assert.rejects(readFile(keyFile, 'utf8'), { code: 'ENOENT' });
+    await assert.rejects(readFile(revisionFile, 'utf8'), { code: 'ENOENT' });
+    await assert.rejects(prepare({ INDEXNOW_KEY: 'bad' }), /Invalid INDEXNOW_KEY/);
+    await assert.rejects(prepare({ INDEXNOW_KEY: key, RAILWAY_GIT_COMMIT_SHA: 'wrong' }), /revision/);
+    await prepare({ INDEXNOW_KEY: key, RAILWAY_GIT_COMMIT_SHA: revision });
+    assert.equal(await readFile(revisionFile, 'utf8'), revision);
+  } finally {
+    await Promise.all([rm(keyFile, {force: true}), rm(revisionFile, {force: true})]);
+  }
+});
+
+test('waits for the production commit, then sends only sitemap URLs with matching ownership', async () => {
+  let markerReads = 0, posts = 0, waits = 0;
+  const fetcher = async (url, options) => {
+    assert.equal(options.redirect, 'error');
+    if (url.endsWith('indexnow-deployment.txt')) {
+      markerReads++;
+      return new Response(markerReads === 1 ? 'b'.repeat(40) : revision);
+    }
+    if (url.endsWith('indexnow-key.txt')) return new Response(key);
+    assert.equal(url, 'https://api.indexnow.org/indexnow');
+    assert.equal(options.method, 'POST');
+    posts++;
+    const payload = JSON.parse(options.body);
+    assert.equal(payload.host, new URL(origin).host);
+    assert.equal(payload.key, key);
+    assert.equal(payload.keyLocation, `${origin}/indexnow-key.txt`);
+    assert.deepEqual(payload.urlList, sitemapUrls(await readFile(new URL('sitemap.xml', publicDir), 'utf8')));
+    return new Response('', {status: 202});
+  };
+  assert.equal(await submit({ revision, fetcher, attempts: 2, wait: async () => waits++ }), 202);
+  assert.equal(markerReads, 2);
+  assert.equal(waits, 1);
+  assert.equal(posts, 1);
+});
+
+test('never POSTs when production is stale, inaccessible or returns an HTML key page', async () => {
+  for (const mode of ['stale', 'unavailable', 'html', 'network']) {
+    let posts = 0;
+    const fetcher = async (url, options) => {
+      if (options.method === 'POST') posts++;
+      if (mode === 'network') throw new Error('unavailable');
+      if (mode === 'unavailable') return new Response('', {status: 404});
+      if (url.endsWith('indexnow-deployment.txt')) return new Response(mode === 'stale' ? 'old' : revision);
+      return new Response('<html>Sign in</html>');
+    };
+    await assert.rejects(submit({revision, fetcher, attempts: 1}), /not ready/);
+    assert.equal(posts, 0);
+  }
+});
+
+test('HTTP receipts distinguish successful acceptance from rejected submissions', async () => {
+  for (const status of [200, 202, 400, 403, 422, 429, 500]) {
+    const fetcher = async (url) => {
+      if (url.endsWith('indexnow-deployment.txt')) return new Response(revision);
+      if (url.endsWith('indexnow-key.txt')) return new Response(key);
+      return new Response('', {status});
+    };
+    const result = submit({revision, fetcher, attempts: 1});
+    if ([200,202].includes(status)) assert.equal(await result, status);
+    else await assert.rejects(result, new RegExp(`HTTP ${status}`));
+  }
+  await assert.rejects(submit({revision: 'not-a-sha'}), /commit SHA/);
+});

website/scripts/indexnow.mjs

@@ -0,0 +1,100 @@
+import { readFile, writeFile, rm } from 'node:fs/promises';
+import { execFileSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { setTimeout as sleep } from 'node:timers/promises';
+
+export const origin = 'https://9lives.run';
+class IndexNowError extends Error {}
+const publicDir = new URL('../public/', import.meta.url);
+const keyFile = new URL('indexnow-key.txt', publicDir);
+const revisionFile = new URL('indexnow-deployment.txt', publicDir);
+
+export function validKey(key) {
+  return typeof key === 'string' && /^[a-zA-Z0-9-]{8,128}$/.test(key) && key.trim() === key;
+}
+
+// These sites use a flat, repository-owned urlset, not a sitemap index.
+export function sitemapUrls(xml) {
+  if (!/<urlset\b/.test(xml) || /<!DOCTYPE|<!ENTITY/i.test(xml)) {
+    throw new IndexNowError('Expected a flat sitemap urlset');
+  }
+  const urls = [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/g)].map((match) => {
+    const value = match[1].replaceAll('&amp;', '&');
+    const url = new URL(value);
+    if (url.origin !== origin || url.username || url.password || url.search || url.hash) {
+      throw new IndexNowError('Sitemap must contain canonical public URLs only');
+    }
+    return url.href;
+  });
+  const unique = [...new Set(urls)];
+  if (!unique.length || unique.length > 10000) throw new IndexNowError('Invalid sitemap URL count');
+  return unique;
+}
+
+export async function prepare(env = process.env) {
+  const key = env.INDEXNOW_KEY;
+  // Local and preview builds work without a production credential. Remove stale files.
+  if (!key) {
+    await Promise.all([rm(keyFile, { force: true }), rm(revisionFile, { force: true })]);
+    console.log('IndexNow is disabled: INDEXNOW_KEY is not configured');
+    return;
+  }
+  if (!validKey(key)) throw new IndexNowError('Invalid INDEXNOW_KEY format');
+  const revision = env.VERCEL_GIT_COMMIT_SHA || env.RAILWAY_GIT_COMMIT_SHA ||
+    execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
+  if (!/^[a-f0-9]{40}$/.test(revision)) throw new IndexNowError('Invalid deployment revision');
+  await writeFile(keyFile, key, 'utf8');
+  await writeFile(revisionFile, revision, 'utf8');
+  console.log('IndexNow verification files prepared');
+}
+
+async function getText(path, fetcher) {
+  const response = await fetcher(`${origin}/${path}`, {
+    redirect: 'error', cache: 'no-store', signal: AbortSignal.timeout(10000),
+  });
+  if (response.status !== 200) throw new IndexNowError('Production verification file unavailable');
+  return (await response.text()).trim();
+}
+
+export async function submit({ revision, fetcher = fetch, wait = sleep, attempts = 90 }) {
+  if (!/^[a-f0-9]{40}$/.test(revision ?? '')) throw new IndexNowError('Expected production commit SHA');
+  const urlList = sitemapUrls(await readFile(new URL('sitemap.xml', publicDir), 'utf8'));
+  let key;
+  const deadline = Date.now() + 15 * 60 * 1000;
+  for (let attempt = 0; attempt < attempts && Date.now() < deadline; attempt++) {
+    try {
+      if (await getText('indexnow-deployment.txt', fetcher) === revision) {
+        const candidate = await getText('indexnow-key.txt', fetcher);
+        if (validKey(candidate)) { key = candidate; break; }
+      }
+    } catch {
+      // A deployment may still be starting. Never log bodies, credentials, or fetch errors.
+    }
+    if (attempt + 1 < attempts) await wait(10000);
+  }
+  if (!key) throw new IndexNowError('Matching production deployment was not ready before timeout');
+  const response = await fetcher('https://api.indexnow.org/indexnow', {
+    method: 'POST', redirect: 'error', signal: AbortSignal.timeout(15000),
+    headers: { 'Content-Type': 'application/json; charset=utf-8' },
+    body: JSON.stringify({ host: new URL(origin).host, key,
+      keyLocation: `${origin}/indexnow-key.txt`, urlList }),
+  });
+  if (![200, 202].includes(response.status)) {
+    throw new IndexNowError(`IndexNow rejected submission (HTTP ${response.status})`);
+  }
+  console.log(`IndexNow received ${urlList.length} URLs (HTTP ${response.status})`);
+  if (response.status === 202) console.log('Ownership validation is pending');
+  return response.status;
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  try {
+    if (process.argv[2] === 'prepare') await prepare();
+    else if (process.argv[2] === 'submit') await submit({ revision: process.env.GITHUB_SHA });
+    else throw new IndexNowError('Use prepare or submit');
+  } catch (error) {
+    // Even network exceptions can contain request data. Emit only a safe status.
+    console.error(error instanceof IndexNowError ? error.message : 'IndexNow failed: network or file operation unavailable');
+    process.exitCode = 1;
+  }
+}