diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml new file mode 100644 index 0000000..14f1200 --- /dev/null +++ b/.github/workflows/indexnow.yml @@ -0,0 +1,47 @@ +name: IndexNow + +on: + pull_request: + paths: + - 'website/**' + - '.github/workflows/indexnow.yml' + push: + branches: [main] + paths: + - 'website/**' + - '.github/workflows/indexnow.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: indexnow-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + - run: node --test website/scripts/indexnow.check.mjs + + submit: + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + needs: test + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + - name: Wait for this production deployment and submit public URLs + run: node website/scripts/indexnow.mjs submit diff --git a/docs/INDEXNOW.md b/docs/INDEXNOW.md new file mode 100644 index 0000000..fd7bcae --- /dev/null +++ b/docs/INDEXNOW.md @@ -0,0 +1,33 @@ +# IndexNow + +Set a unique `INDEXNOW_KEY` in the production hosting environment for this +site. Generate 32 random hexadecimal characters with a password manager; do +not reuse another site's key or commit it. Preview/local builds can omit it. + +The build writes `/indexnow-key.txt` and `/indexnow-deployment.txt` into the +public output. Both generated files are gitignored. The first is the UTF-8 +ownership proof required by IndexNow; the second identifies the deployed +commit. Treat the key as a credential even though search engines must be able +to fetch its verification file. + +After changes to the site merge to `main`, the IndexNow GitHub workflow waits +for up to 15 minutes for the canonical production host to serve that exact +commit. It retrieves the ownership proof without logging it, then submits +the current flat sitemap's public URLs to `https://api.indexnow.org/indexnow`. +There is no browser key or GitHub secret, and preview deployments never submit. +The sitemap must contain only HTTPS URLs on this site's canonical host, without +query strings or fragments. Keep private/authenticated pages out of it. + +The workflow can also be run manually on `main` after a hosting configuration +change or redeployment. A missing key, failed deployment, or stale production +revision makes submission fail without sending URLs. HTTP 200 means the URLs +were received; 202 means ownership validation is pending. Neither guarantees +indexing. Check Bing Webmaster Tools > IndexNow for receipt evidence. + +This small static-site integration submits the current sitemap after a site +deploy. If removing a page, submit its deleted canonical URL separately before +removing it from the sitemap. Sitemap indexes are not supported. + +Run the focused checks with `node --test website/scripts/indexnow.check.mjs`. + +Protocol: https://www.indexnow.org/documentation diff --git a/website/.gitignore b/website/.gitignore index ef6f899..b3f7655 100644 --- a/website/.gitignore +++ b/website/.gitignore @@ -36,3 +36,7 @@ next-env.d.ts /.wrangler/ /outputs/ /work/ + +# Generated IndexNow ownership files (never commit credentials) +/public/indexnow-key.txt +/public/indexnow-deployment.txt diff --git a/website/package.json b/website/package.json index 831e3d5..2476c03 100644 --- a/website/package.json +++ b/website/package.json @@ -11,7 +11,9 @@ "build:vercel": "next build", "start": "vinext start", "lint": "eslint . --ignore-pattern dist --ignore-pattern .next", - "test": "vitest run" + "test": "vitest run", + "prebuild": "node scripts/indexnow.mjs prepare", + "prebuild:vercel": "node scripts/indexnow.mjs prepare" }, "dependencies": { "next": "^16.3.3", diff --git a/website/scripts/indexnow.check.mjs b/website/scripts/indexnow.check.mjs new file mode 100644 index 0000000..461afdf --- /dev/null +++ b/website/scripts/indexnow.check.mjs @@ -0,0 +1,101 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; +import { readFile, rm } from 'node:fs/promises'; +import { origin, validKey, sitemapUrls, prepare, submit } from './indexnow.mjs'; + +const key = randomBytes(16).toString('hex'); +const revision = 'a'.repeat(40); +const publicDir = new URL('../public/', import.meta.url); +const xml = (urls) => `${urls.map((url) => `${url}`).join('')}`; + +test('canonical sitemap rejects foreign hosts, private query URLs, fragments and empty lists', () => { + assert.deepEqual(sitemapUrls(xml([`${origin}/`, `${origin}/`])), [`${origin}/`]); + for (const urls of [[], ['https://example.net/'], [`${origin}/?token=private`], + [`${origin}/#fragment`], [origin.replace('https:', 'http:')]]) { + assert.throws(() => sitemapUrls(xml(urls))); + } + assert.throws(() => sitemapUrls('https://example.net/')); + assert.throws(() => sitemapUrls(']>')); + assert.throws(() => sitemapUrls(xml(Array.from({length: 10001}, (_, n) => `${origin}/${n}`)))); +}); + +test('key validation rejects whitespace, unsafe characters and invalid lengths', () => { + assert.equal(validKey(key), true); + for (const candidate of [undefined, '', 'short', key + '\n', 'x'.repeat(129), 'a/b'.repeat(8)]) { + assert.equal(validKey(candidate), false); + } +}); + +test('build writes UTF-8 ownership files and clears stale files when disabled', async () => { + const keyFile = new URL('indexnow-key.txt', publicDir); + const revisionFile = new URL('indexnow-deployment.txt', publicDir); + try { + await prepare({ INDEXNOW_KEY: key, VERCEL_GIT_COMMIT_SHA: revision }); + assert.equal(await readFile(keyFile, 'utf8'), key); + assert.equal(await readFile(revisionFile, 'utf8'), revision); + await prepare({}); + await assert.rejects(readFile(keyFile, 'utf8'), { code: 'ENOENT' }); + await assert.rejects(readFile(revisionFile, 'utf8'), { code: 'ENOENT' }); + await assert.rejects(prepare({ INDEXNOW_KEY: 'bad' }), /Invalid INDEXNOW_KEY/); + await assert.rejects(prepare({ INDEXNOW_KEY: key, RAILWAY_GIT_COMMIT_SHA: 'wrong' }), /revision/); + await prepare({ INDEXNOW_KEY: key, RAILWAY_GIT_COMMIT_SHA: revision }); + assert.equal(await readFile(revisionFile, 'utf8'), revision); + } finally { + await Promise.all([rm(keyFile, {force: true}), rm(revisionFile, {force: true})]); + } +}); + +test('waits for the production commit, then sends only sitemap URLs with matching ownership', async () => { + let markerReads = 0, posts = 0, waits = 0; + const fetcher = async (url, options) => { + assert.equal(options.redirect, 'error'); + if (url.endsWith('indexnow-deployment.txt')) { + markerReads++; + return new Response(markerReads === 1 ? 'b'.repeat(40) : revision); + } + if (url.endsWith('indexnow-key.txt')) return new Response(key); + assert.equal(url, 'https://api.indexnow.org/indexnow'); + assert.equal(options.method, 'POST'); + posts++; + const payload = JSON.parse(options.body); + assert.equal(payload.host, new URL(origin).host); + assert.equal(payload.key, key); + assert.equal(payload.keyLocation, `${origin}/indexnow-key.txt`); + assert.deepEqual(payload.urlList, sitemapUrls(await readFile(new URL('sitemap.xml', publicDir), 'utf8'))); + return new Response('', {status: 202}); + }; + assert.equal(await submit({ revision, fetcher, attempts: 2, wait: async () => waits++ }), 202); + assert.equal(markerReads, 2); + assert.equal(waits, 1); + assert.equal(posts, 1); +}); + +test('never POSTs when production is stale, inaccessible or returns an HTML key page', async () => { + for (const mode of ['stale', 'unavailable', 'html', 'network']) { + let posts = 0; + const fetcher = async (url, options) => { + if (options.method === 'POST') posts++; + if (mode === 'network') throw new Error('unavailable'); + if (mode === 'unavailable') return new Response('', {status: 404}); + if (url.endsWith('indexnow-deployment.txt')) return new Response(mode === 'stale' ? 'old' : revision); + return new Response('Sign in'); + }; + await assert.rejects(submit({revision, fetcher, attempts: 1}), /not ready/); + assert.equal(posts, 0); + } +}); + +test('HTTP receipts distinguish successful acceptance from rejected submissions', async () => { + for (const status of [200, 202, 400, 403, 422, 429, 500]) { + const fetcher = async (url) => { + if (url.endsWith('indexnow-deployment.txt')) return new Response(revision); + if (url.endsWith('indexnow-key.txt')) return new Response(key); + return new Response('', {status}); + }; + const result = submit({revision, fetcher, attempts: 1}); + if ([200,202].includes(status)) assert.equal(await result, status); + else await assert.rejects(result, new RegExp(`HTTP ${status}`)); + } + await assert.rejects(submit({revision: 'not-a-sha'}), /commit SHA/); +}); diff --git a/website/scripts/indexnow.mjs b/website/scripts/indexnow.mjs new file mode 100644 index 0000000..05ae4ae --- /dev/null +++ b/website/scripts/indexnow.mjs @@ -0,0 +1,100 @@ +import { readFile, writeFile, rm } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { setTimeout as sleep } from 'node:timers/promises'; + +export const origin = 'https://9lives.run'; +class IndexNowError extends Error {} +const publicDir = new URL('../public/', import.meta.url); +const keyFile = new URL('indexnow-key.txt', publicDir); +const revisionFile = new URL('indexnow-deployment.txt', publicDir); + +export function validKey(key) { + return typeof key === 'string' && /^[a-zA-Z0-9-]{8,128}$/.test(key) && key.trim() === key; +} + +// These sites use a flat, repository-owned urlset, not a sitemap index. +export function sitemapUrls(xml) { + if (!/\s*([^<]+?)\s*<\/loc>/g)].map((match) => { + const value = match[1].replaceAll('&', '&'); + const url = new URL(value); + if (url.origin !== origin || url.username || url.password || url.search || url.hash) { + throw new IndexNowError('Sitemap must contain canonical public URLs only'); + } + return url.href; + }); + const unique = [...new Set(urls)]; + if (!unique.length || unique.length > 10000) throw new IndexNowError('Invalid sitemap URL count'); + return unique; +} + +export async function prepare(env = process.env) { + const key = env.INDEXNOW_KEY; + // Local and preview builds work without a production credential. Remove stale files. + if (!key) { + await Promise.all([rm(keyFile, { force: true }), rm(revisionFile, { force: true })]); + console.log('IndexNow is disabled: INDEXNOW_KEY is not configured'); + return; + } + if (!validKey(key)) throw new IndexNowError('Invalid INDEXNOW_KEY format'); + const revision = env.VERCEL_GIT_COMMIT_SHA || env.RAILWAY_GIT_COMMIT_SHA || + execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); + if (!/^[a-f0-9]{40}$/.test(revision)) throw new IndexNowError('Invalid deployment revision'); + await writeFile(keyFile, key, 'utf8'); + await writeFile(revisionFile, revision, 'utf8'); + console.log('IndexNow verification files prepared'); +} + +async function getText(path, fetcher) { + const response = await fetcher(`${origin}/${path}`, { + redirect: 'error', cache: 'no-store', signal: AbortSignal.timeout(10000), + }); + if (response.status !== 200) throw new IndexNowError('Production verification file unavailable'); + return (await response.text()).trim(); +} + +export async function submit({ revision, fetcher = fetch, wait = sleep, attempts = 90 }) { + if (!/^[a-f0-9]{40}$/.test(revision ?? '')) throw new IndexNowError('Expected production commit SHA'); + const urlList = sitemapUrls(await readFile(new URL('sitemap.xml', publicDir), 'utf8')); + let key; + const deadline = Date.now() + 15 * 60 * 1000; + for (let attempt = 0; attempt < attempts && Date.now() < deadline; attempt++) { + try { + if (await getText('indexnow-deployment.txt', fetcher) === revision) { + const candidate = await getText('indexnow-key.txt', fetcher); + if (validKey(candidate)) { key = candidate; break; } + } + } catch { + // A deployment may still be starting. Never log bodies, credentials, or fetch errors. + } + if (attempt + 1 < attempts) await wait(10000); + } + if (!key) throw new IndexNowError('Matching production deployment was not ready before timeout'); + const response = await fetcher('https://api.indexnow.org/indexnow', { + method: 'POST', redirect: 'error', signal: AbortSignal.timeout(15000), + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + body: JSON.stringify({ host: new URL(origin).host, key, + keyLocation: `${origin}/indexnow-key.txt`, urlList }), + }); + if (![200, 202].includes(response.status)) { + throw new IndexNowError(`IndexNow rejected submission (HTTP ${response.status})`); + } + console.log(`IndexNow received ${urlList.length} URLs (HTTP ${response.status})`); + if (response.status === 202) console.log('Ownership validation is pending'); + return response.status; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + try { + if (process.argv[2] === 'prepare') await prepare(); + else if (process.argv[2] === 'submit') await submit({ revision: process.env.GITHUB_SHA }); + else throw new IndexNowError('Use prepare or submit'); + } catch (error) { + // Even network exceptions can contain request data. Emit only a safe status. + console.error(error instanceof IndexNowError ? error.message : 'IndexNow failed: network or file operation unavailable'); + process.exitCode = 1; + } +}