diff --git a/components/google_search_console/actions/compare-search-analytics/compare-search-analytics.mjs b/components/google_search_console/actions/compare-search-analytics/compare-search-analytics.mjs new file mode 100644 index 0000000000000..b6ae9d5ef9f7c --- /dev/null +++ b/components/google_search_console/actions/compare-search-analytics/compare-search-analytics.mjs @@ -0,0 +1,262 @@ +import googleSearchConsole from "../../google_search_console.app.mjs"; +import { + buildComparison, formatPctChange, +} from "../../common/compare.mjs"; +import { buildDimensionFilterGroups } from "../../common/filters.mjs"; +import { trimIfString } from "../../common/utils.mjs"; + +// Rows fetched per period before the join. Deltas need the full set on both sides, so this +// is deliberately much larger than the `rowLimit` the caller sees. +const INTERNAL_ROW_LIMIT = 5000; +const DEFAULT_ROW_LIMIT = 50; + +export default { + name: "Compare Search Analytics", + description: "Compare Google Search Console traffic between two date ranges for one property and return the deltas. Fetches both periods in parallel, joins the rows on their dimension keys, and reports per-row and total `clicks`, `impressions`, `ctr` and `position` change — the period-over-period arithmetic is done for you.\n\n" + + "**When to use:** any question that compares two date ranges — month over month, quarter over quarter, year over year, \"which queries gained or lost the most clicks\", \"did mobile grow\", \"did that update hurt us\". For a single date range use **Query Search Analytics** instead. This tool also cannot answer property-level vs page-level position questions: that needs two **Query Search Analytics** calls with different `aggregationType` values.\n\n" + + "**Returns:** `{ current_period, previous_period, totals, rows, row_count, has_more, truncated, note }`. `totals` carries `current`, `previous`, `delta` and `pct_change` for the whole period (computed from every fetched row, not just the returned ones). Each row is `{ keys, current, previous, delta, pct_change }`. A key present in only one period gets zeros for the other, so new and lost queries both show up — but its `delta.ctr` and `delta.position` are `null`, because the missing period has no CTR or position to compare against; `delta.clicks` and `delta.impressions` are still real numbers there. `truncated` is true when either period hit the internal 5000-row maximum, so rows and totals may be incomplete — narrow the date range or add a filter. `note` warns about anonymized queries when relevant, otherwise it is null.\n\n" + + "**Cross-references:** call **List Sites** first when the user names a site in prose rather than giving an exact property identifier. Use **Query Search Analytics** for anything about a single range, for paging past 5000 rows, or for `hour`/`searchAppearance` dimensions this tool does not accept.\n\n" + + "**Parameter guidance:**\n" + + "- All four dates are `YYYY-MM-DD`, **Pacific Time**, inclusive. \"The previous period\" means the same number of days immediately before the current period: for 2026-08-01..2026-08-28 (28 days) the previous period is 2026-07-04..2026-07-31. \"The same period last year\" means both dates shifted back one year: 2025-08-01..2025-08-28.\n" + + "- `dimensions` decides what the rows are. Leave it empty for one totals row per period (the right choice for \"how did traffic change overall\"). `hour` and `searchAppearance` are not supported here.\n" + + "- Up to **5000 rows per period** are fetched internally before the join; when a period hits that cap its tail is excluded and `truncated` comes back true, so narrow the range or add a filter. `has_more` is a different signal — it only means the join produced more rows than `rowLimit`.\n" + + "- `sortBy`: the `*_delta` options sort by the ABSOLUTE change, so the biggest gains and the biggest losses both surface at the top; under `ctr_delta` and `position_delta` rows with a `null` delta sort last, so `clicks_delta` is the sort that surfaces new and lost queries. `current_clicks` sorts by current-period clicks descending. `rowLimit` (default 50) caps rows AFTER the join.\n" + + "- Filtering: `filterValue` with `filterDimension`/`filterOperator` is the single-condition shortcut and is applied identically to both periods. Use `advancedDimensionFilters` for multi-condition filters; it is ignored whenever `filterValue` is set. (The equivalent prop on **Query Search Analytics** is named `subdomainFilter` for backwards compatibility — same meaning.)\n\n" + + "**Common mistakes:**\n" + + "- Grouping Discover by `query` — Discover has no `query` dimension and the API returns a 400.\n" + + "- Reading query-row sums as the property total. Google omits anonymized (rare) queries, so query rows always understate the real total; compare with no dimensions, or by `date`, for true totals.\n" + + "- Re-averaging `ctr` or `position` across rows. Both are impression-weighted, and the totals here already are: `ctr` is a 0-1 fraction (0.1428 means 14.3%), `position` is 1-indexed and lower is better — so a NEGATIVE position delta is an improvement.\n" + + "- Reading a position or CTR change for a query that is new or lost. Those deltas are `null` by design — there is no ranking on the missing side to subtract.\n" + + "- Expecting a percentage where the previous period had zero. `pct_change` is null in that case, not 0 and not infinity.\n" + + "- Comparing a range that ends today. Data is final only after about 2-3 days, so a fresh current period looks artificially low unless `dataState` is `all`.\n\n" + + "**Example:** `siteUrl=\"sc-domain:example.com\"`, `currentStartDate=\"2026-08-01\"`, `currentEndDate=\"2026-08-28\"`, `previousStartDate=\"2026-07-04\"`, `previousEndDate=\"2026-07-31\"`, `dimensions=[\"query\"]`, `sortBy=\"clicks_delta\"` returns `totals: { current: { clicks: 74, impressions: 612, ctr: 0.1209, position: 2.6 }, previous: { clicks: 68, ... }, delta: { clicks: 6, ... }, pct_change: { clicks: 0.0882, impressions: 0.0431 } }` and rows such as `{ keys: [\"example brand\"], current: { clicks: 41, impressions: 287, ctr: 0.1429, position: 2.4 }, previous: { clicks: 33, ... }, delta: { clicks: 8, ... }, pct_change: { clicks: 0.2424, impressions: 0.1 } }`.\n\n" + + "[See the documentation](https://developers.google.com/webmaster-tools/v1/searchanalytics/query)", + key: "google_search_console-compare-search-analytics", + version: "0.0.1", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, + type: "action", + ai: "optimized", + props: { + googleSearchConsole, + siteUrl: { + propDefinition: [ + googleSearchConsole, + "siteUrl", + ], + }, + currentStartDate: { + type: "string", + label: "Current Period Start Date (YYYY-MM-DD)", + description: "First day of the RECENT period, inclusive, in `YYYY-MM-DD` **Pacific Time** — e.g. `2026-08-01`.", + }, + currentEndDate: { + type: "string", + label: "Current Period End Date (YYYY-MM-DD)", + description: "Last day of the RECENT period, inclusive — e.g. `2026-08-28`. Data is final only after about 2-3 days, so avoid ending on today unless `dataState` is `all`.", + }, + previousStartDate: { + type: "string", + label: "Previous Period Start Date (YYYY-MM-DD)", + description: "First day of the BASELINE period, inclusive. For \"the previous period\", use the same number of days immediately before the current period — for 2026-08-01..2026-08-28 that is `2026-07-04`. For \"the same period last year\", shift the current start date back one year: `2025-08-01`.", + }, + previousEndDate: { + type: "string", + label: "Previous Period End Date (YYYY-MM-DD)", + description: "Last day of the BASELINE period, inclusive. For \"the previous period\" it is the day before `currentStartDate` — for a current period starting 2026-08-01 that is `2026-07-31`. For \"the same period last year\", shift `currentEndDate` back one year.", + }, + dimensions: { + type: "string[]", + label: "Dimensions", + optional: true, + description: "How to group the compared rows; each row's `keys` lines up positionally with this list. Leave empty to compare one totals row per period, which is what \"how did traffic change overall\" needs. Use `query` for \"which queries gained or lost\", `page` for page-level movement, `device` for mobile vs desktop, `country` for market shifts, `date` for a day-by-day pair-up. `hour` and `searchAppearance` are not supported here — use **Query Search Analytics** for those.", + options: [ + "query", + "page", + "country", + "device", + "date", + ], + }, + searchType: { + propDefinition: [ + googleSearchConsole, + "searchType", + ], + }, + dataState: { + type: "string", + label: "Data State", + description: "Which data to include, applied to both periods. `final` (default) returns only finalized data, which lags roughly 2-3 days behind today. `all` also includes the most recent, not-yet-final days — use it only when the user explicitly wants recent or partial numbers, and note it makes a current period that ends today look incomplete rather than absent.", + optional: true, + options: [ + "final", + "all", + ], + default: "final", + }, + filterDimension: { + propDefinition: [ + googleSearchConsole, + "filterDimension", + ], + }, + filterOperator: { + propDefinition: [ + googleSearchConsole, + "filterOperator", + ], + }, + filterValue: { + type: "string", + label: "Filter Value", + optional: true, + description: "The value to filter both periods on, combined with `filterDimension` and `filterOperator` into a single filter — e.g. `filterDimension: device`, `filterOperator: equals`, value `MOBILE`. `page` expressions match the FULL URL (scheme and host included), not just a path. String comparison is case-insensitive; regex operators use RE2. When this is set, `advancedDimensionFilters` is ignored.", + }, + advancedDimensionFilters: { + propDefinition: [ + googleSearchConsole, + "advancedDimensionFilters", + ], + }, + sortBy: { + type: "string", + label: "Sort By", + description: "How to order the returned rows. The `*_delta` options sort by the ABSOLUTE change, so the biggest gains and the biggest losses both appear at the top — read the sign of `delta` to tell them apart. Under `ctr_delta` and `position_delta`, rows whose delta is `null` (a key present in only one period) sort last, so use `clicks_delta` to surface new and lost queries. `current_clicks` sorts by current-period clicks descending, which is the right choice for \"top queries, with their change\". Defaults to `clicks_delta`.", + optional: true, + options: [ + "clicks_delta", + "impressions_delta", + "ctr_delta", + "position_delta", + "current_clicks", + ], + default: "clicks_delta", + }, + rowLimit: { + type: "integer", + label: "Max Rows", + description: "How many joined rows to return, after sorting. Defaults to 50. This caps the OUTPUT only — up to 5000 rows per period are always fetched, so the totals cover everything even when the row list is short, unless `truncated` is true. `has_more` is true when the join produced more rows than were returned.", + optional: true, + default: DEFAULT_ROW_LIMIT, + }, + }, + async run({ $ }) { + const { + googleSearchConsole, + siteUrl, + currentStartDate, + currentEndDate, + previousStartDate, + previousEndDate, + dimensions, + searchType, + dataState, + filterDimension, + filterOperator, + filterValue, + advancedDimensionFilters, + sortBy, + rowLimit, + } = this; + + const dimensionFilterGroups = buildDimensionFilterGroups({ + app: googleSearchConsole, + filterValue, + filterDimension, + filterOperator, + advancedDimensionFilters, + }); + + const groupBy = dimensions?.length + ? dimensions.map((dimension) => trimIfString(dimension)) + : undefined; + + // `type` is the supported request key; `searchType` is deprecated server-side. + const baseBody = { + dimensions: groupBy, + type: trimIfString(searchType), + dataState: trimIfString(dataState), + dimensionFilterGroups, + rowLimit: INTERNAL_ROW_LIMIT, + startRow: 0, + }; + + for (const key of Object.keys(baseBody)) { + if (baseBody[key] === undefined) { + delete baseBody[key]; + } + } + + const current = { + startDate: trimIfString(currentStartDate), + endDate: trimIfString(currentEndDate), + }; + const previous = { + startDate: trimIfString(previousStartDate), + endDate: trimIfString(previousEndDate), + }; + + const [ + currentResponse, + previousResponse, + ] = await Promise.all([ + googleSearchConsole.getSitePerformanceData({ + $, + url: siteUrl, + data: { + ...baseBody, + ...current, + }, + }), + googleSearchConsole.getSitePerformanceData({ + $, + url: siteUrl, + data: { + ...baseBody, + ...previous, + }, + }), + ]); + + // A period that comes back exactly at the cap almost certainly had more rows behind + // it, and there is no paging here — so say so rather than let totals quietly understate. + const truncated = (currentResponse?.rows?.length ?? 0) >= INTERNAL_ROW_LIMIT + || (previousResponse?.rows?.length ?? 0) >= INTERNAL_ROW_LIMIT; + + const comparison = buildComparison({ + currentRows: currentResponse?.rows ?? [], + previousRows: previousResponse?.rows ?? [], + sortBy: sortBy || "clicks_delta", + rowLimit: rowLimit ?? DEFAULT_ROW_LIMIT, + }); + + const note = groupBy?.includes("query") + ? "Google omits anonymized (rare) queries from query-dimension rows, so these row sums understate the property total. Compare with no dimensions, or by date, for true totals." + : null; + + const { + totals, rows, row_count: rowCount, has_more: hasMore, + } = comparison; + + const truncationNote = truncated + ? ` — truncated at ${INTERNAL_ROW_LIMIT} rows per period` + : ""; + + $.export("$summary", `Compared ${current.startDate}..${current.endDate} vs ${previous.startDate}..${previous.endDate}: clicks ${totals.previous.clicks} → ${totals.current.clicks} (${formatPctChange(totals.pct_change.clicks)}), ${rowCount} rows${truncationNote}`); + + return { + current_period: current, + previous_period: previous, + totals, + rows, + row_count: rowCount, + has_more: hasMore, + truncated, + note, + }; + }, +}; diff --git a/components/google_search_console/actions/delete-sitemap/delete-sitemap.mjs b/components/google_search_console/actions/delete-sitemap/delete-sitemap.mjs new file mode 100644 index 0000000000000..06340515921c9 --- /dev/null +++ b/components/google_search_console/actions/delete-sitemap/delete-sitemap.mjs @@ -0,0 +1,97 @@ +import googleSearchConsole from "../../google_search_console.app.mjs"; +import { trimIfString } from "../../common/utils.mjs"; + +export default { + name: "Delete Sitemap", + description: "Removes (unlists) a sitemap from a Google Search Console property." + + "\n\n**Purpose.** Tells Search Console to stop tracking a sitemap file. This is a destructive, " + + "non-reversible-by-this-tool operation on the property's configuration." + + "\n\n**When to use.** Only when the user has EXPLICITLY confirmed the exact sitemap URL to remove. " + + "If the user says something vague such as \"remove the old sitemap\", do NOT call this tool: call " + + "**List Sitemaps** first, show the candidate paths, and ask which one. If the user names a sitemap " + + "but has not confirmed the deletion, state the exact `sitemapUrl` you are about to delete and ask " + + "for confirmation before calling. Never treat the absence of an answer as consent: " + + "if you cannot obtain an explicit confirmation in this turn (for example a " + + "confirmation prompt is unavailable), end your turn by asking the question in plain " + + "text and do NOT call this tool. Never delete a sitemap as a side effect of another " + + "task, and never delete-and-resubmit to \"refresh\" a sitemap - **Submit Sitemap** on " + + "the existing path already does that." + + "\n\n**Returns.** `{ deleted: true, sitemapUrl }` on success. The API returns an empty body, so " + + "there is nothing else to report; to prove it is gone, call **List Sitemaps** WITHOUT " + + "`sitemapUrl` (list everything and check the `path` is absent) — asking for the deleted path " + + "directly returns 404 \"'' is not a submitted or a known sitemap.\", which reads as an " + + "error." + + "\n\n**What this does NOT do.** It only UNLISTS the sitemap from Search Console. It does not " + + "deindex, remove or hide the pages the sitemap contained - those URLs stay in Google's index and " + + "can still be crawled and discovered through links. Do not offer this tool as a way to remove " + + "content from Google. It also does not delete the sitemap file from the website." + + "\n\n**Cross-references.** Get the exact `siteUrl` from **List Sites**. Get the exact `path` to " + + "delete from **List Sitemaps**, and call it again afterwards (with no `sitemapUrl`) to verify " + + "the path is absent. Use **Submit Sitemap** to add a sitemap back or to ask Google to re-read " + + "one. Use **Inspect URLs** to check the index status of individual pages." + + "\n\n**Parameter guidance.** `siteUrl` is the property identifier, copied verbatim from **List " + + "Sites**. `sitemapUrl` is the full URL of the sitemap exactly as **List Sitemaps** reports it in " + + "`path` - copy it, do not retype or normalize it (do not add or drop `www.`, do not change the " + + "scheme). Both are required." + + "\n\n**Common mistakes.** Passing a path (`/sitemap.xml`) or a guessed URL instead of a listed " + + "`path`: an unknown path returns 404 \"'' is not a submitted or a known sitemap.\" Passing " + + "the `https://www.` variant when the property lists the `http://` one (or the other way round) " + + "produces that same 404 - the host and scheme are part of the identity. Requires `siteOwner` or " + + "`siteFullUser`; a `siteRestrictedUser` gets a 403." + + "\n\n**Example.** `siteUrl=\"sc-domain:example.com\"`, " + + "`sitemapUrl=\"https://www.example.com/sitemap-archive.xml\"` -> " + + "`{ deleted: true, sitemapUrl: \"https://www.example.com/sitemap-archive.xml\" }`, " + + "and a follow-up **List Sitemaps** call (no `sitemapUrl`) no longer shows that path." + + "\n\n[See the documentation](https://developers.google.com/webmaster-tools/v1/sitemaps/delete)", + key: "google_search_console-delete-sitemap", + version: "0.0.1", + annotations: { + destructiveHint: true, + openWorldHint: true, + readOnlyHint: false, + }, + type: "action", + ai: "optimized", + props: { + googleSearchConsole, + siteUrl: { + propDefinition: [ + googleSearchConsole, + "siteUrl", + ], + }, + sitemapUrl: { + propDefinition: [ + googleSearchConsole, + "sitemapUrl", + ], + description: "Full URL of the sitemap to unlist, copied verbatim from the `path` field returned " + + "by **List Sitemaps**, e.g. `https://www.example.com/sitemap.xml`. An unknown path " + + "returns 404 \"'' is not a submitted or a known sitemap.\" Confirm this exact URL with " + + "the user before calling.", + }, + }, + async run({ $ }) { + const { + googleSearchConsole, + siteUrl, + sitemapUrl, + } = this; + + const trimmedSiteUrl = trimIfString(siteUrl); + const trimmedSitemapUrl = trimIfString(sitemapUrl); + + await googleSearchConsole.deleteSitemap({ + $, + siteUrl: trimmedSiteUrl, + sitemapUrl: trimmedSitemapUrl, + }); + + $.export("$summary", `Deleted sitemap ${trimmedSitemapUrl} from ${trimmedSiteUrl}`); + + return { + deleted: true, + sitemapUrl: trimmedSitemapUrl, + }; + }, +}; diff --git a/components/google_search_console/actions/inspect-urls/inspect-urls.mjs b/components/google_search_console/actions/inspect-urls/inspect-urls.mjs new file mode 100644 index 0000000000000..bd5a192f3cb46 --- /dev/null +++ b/components/google_search_console/actions/inspect-urls/inspect-urls.mjs @@ -0,0 +1,248 @@ +import googleSearchConsole from "../../google_search_console.app.mjs"; +import { trimIfString } from "../../common/utils.mjs"; + +const MAX_URLS = 10; +const CONCURRENCY = 5; +const MAX_REFERRING_URLS = 5; + +/** + * Minimal inline worker pool. Runs `worker` over `items` with at most `limit` + * calls in flight at any moment and returns the results in input order. + */ +async function mapWithConcurrency(items, limit, worker) { + const results = new Array(items.length); + let cursor = 0; + + const runNext = async () => { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + results[index] = await worker(items[index], index); + } + }; + + const runners = []; + const poolSize = Math.min(limit, items.length); + for (let i = 0; i < poolSize; i++) { + runners.push(runNext()); + } + await Promise.all(runners); + + return results; +} + +function buildRow({ + url, inspectionResult, error, includeFullResult, +}) { + const indexStatus = inspectionResult?.indexStatusResult ?? {}; + const googleCanonical = indexStatus.googleCanonical ?? null; + const userCanonical = indexStatus.userCanonical ?? null; + const referringUrls = indexStatus.referringUrls ?? []; + + const row = { + url, + verdict: indexStatus.verdict ?? null, + coverageState: indexStatus.coverageState ?? null, + indexingState: indexStatus.indexingState ?? null, + robotsTxtState: indexStatus.robotsTxtState ?? null, + pageFetchState: indexStatus.pageFetchState ?? null, + lastCrawlTime: indexStatus.lastCrawlTime ?? null, + crawledAs: indexStatus.crawledAs ?? null, + googleCanonical, + userCanonical, + canonical_mismatch: (googleCanonical && userCanonical) + ? googleCanonical !== userCanonical + : null, + referring_url_count: referringUrls.length, + referringUrls: referringUrls.slice(0, MAX_REFERRING_URLS), + sitemaps: indexStatus.sitemap ?? [], + rich_results_verdict: inspectionResult?.richResultsResult?.verdict ?? null, + inspectionResultLink: inspectionResult?.inspectionResultLink ?? null, + error: error ?? null, + }; + + if (includeFullResult) { + row.full_result = inspectionResult ?? null; + } + + return row; +} + +export default { + name: "Inspect URLs", + description: "Returns Google's index status, canonical selection and crawl state for 1-10 URLs " + + "of one Search Console property, in a single call." + + "\n\n**Purpose.** This is the API behind the URL Inspection tool in the Search Console UI. " + + "For each URL it reports whether Google has indexed it, why or why not, when it was last " + + "crawled and with which crawler, the canonical Google selected versus the canonical the page " + + "declares, which sitemaps reference it, and whether rich results were detected." + + "\n\n**When to use.** \"Is this page indexed?\", \"when did Google last crawl it?\", \"does " + + "Google's canonical match the one I declared?\", \"why is this URL missing from search?\", and " + + "batch health checks after a deploy or a migration - pass every URL you care about in ONE call " + + "instead of calling the tool once per URL." + + "\n\n**Returns.** `{ results: [...], summary: { total, indexed, not_indexed, errors } }`. Each " + + "row is `{ url, verdict, coverageState, indexingState, robotsTxtState, pageFetchState, " + + "lastCrawlTime, crawledAs, googleCanonical, userCanonical, canonical_mismatch, " + + "referring_url_count, referringUrls, sitemaps, rich_results_verdict, inspectionResultLink, " + + "error }`, in the same order as `inspectionUrls`. `verdict` is `PASS` (indexed), `NEUTRAL` " + + "(known but not indexed, or unknown to Google), `FAIL` or `PARTIAL`. `canonical_mismatch` is " + + "`true` when both canonicals are present and differ, `false` when they match, and `null` when " + + "either is missing. `referringUrls` is truncated to the first 5 while `referring_url_count` is " + + "the full count. These are sample referrers Google used to find the page, not a backlink " + + "report. `error` is `null` on success and a message string when that single URL failed: " + + "one bad URL never aborts the batch. In `summary`, `indexed` counts `verdict: \"PASS\"` rows, " + + "`errors` counts rows with an `error`, and `not_indexed` is everything else. The raw API result " + + "is attached per row as `full_result` ONLY when `includeFullResult` is true - leave it off " + + "unless the user asks for the complete raw result, because it is large and mostly rich-results " + + "and AMP detail." + + "\n\n**Cross-references.** Get the exact `siteUrl` and confirm your permission level with " + + "**List Sites**. Use **Query Search Analytics** to find the pages worth inspecting (for example " + + "the pages with impressions but no clicks). Use **Submit Sitemap** to ask Google to re-read a " + + "sitemap - that is the supported way to nudge crawling, because there is no API to request " + + "indexing of a single ordinary page. If the user asks to \"request indexing\" or force a " + + "recrawl of an ordinary page, do not run this tool (or any other) on your own initiative: " + + "explain that no API does that, OFFER this index-status check or a sitemap resubmission, and " + + "wait for the user to choose." + + "\n\n**Parameter guidance.** `inspectionUrls` takes full absolute URLs (scheme included) that " + + "live under the property named in `siteUrl`; a path such as `/about` is not accepted. Send " + + "several URLs in one call rather than one call per URL. The limit is 10 URLs per call - split " + + "longer lists into batches. Each inspection takes Google roughly 5-10 seconds, so a 10-URL " + + "batch runs about 20 seconds — that is normal, not a hang. Quota is 2,000 inspections per " + + "day and 600 per minute per property, and quota errors do not say which of the two was hit. " + + "`languageCode` (BCP-47, default `en-US`) only changes the language of the human-readable " + + "strings in the result." + + "\n\n**Common mistakes.** Do NOT use this tool for backlinks, \"who links to my site\", or " + + "the Links report: Search Console's Links report has no API at all, and `referringUrls` here " + + "is only a small sample of pages Google happened to discover the URL from — not a backlink " + + "profile. When asked for backlinks, call no Search Console tool and say plainly that the " + + "links report is not available through the API. The result also does NOT include Core Web " + + "Vitals or page-experience data. A URL Google has never seen returns " + + "`verdict: \"NEUTRAL\"` with a `coverageState` like " + + "`\"URL is unknown to Google\"`; that is a valid answer, not an error. This tool requires " + + "`siteOwner` or `siteFullUser` permission - a `siteRestrictedUser` gets 403. A wrong or " + + "mismatched `siteUrl` (URLs that do not belong to that property, a missing trailing slash, or " + + "`sc-domain:` versus URL-prefix confusion) also returns 403 \"User does not have sufficient " + + "permission for site\", so copy the identifier verbatim from **List Sites**. " + + "`mobileUsabilityResult` is deprecated by Google and is not surfaced here." + + "\n\n**Example.** `siteUrl=\"sc-domain:example.com\"`, " + + "`inspectionUrls=[\"https://www.example.com/\"]` -> `results[0]` has " + + "`verdict: \"PASS\"`, `coverageState: \"Submitted and indexed\"`, " + + "`googleCanonical: \"https://www.example.com/\"`, " + + "`userCanonical: \"https://example.com/\"`, `canonical_mismatch: true` (Google indexed " + + "the www URL even though the page declares the non-www one), and a `lastCrawlTime` such as " + + "`\"2026-08-28T04:12:33Z\"`; `summary` is " + + "`{ total: 1, indexed: 1, not_indexed: 0, errors: 0 }`." + + "\n\n[See the documentation](https://developers.google.com/webmaster-tools/v1/urlInspection.index/inspect)", + key: "google_search_console-inspect-urls", + version: "0.0.1", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, + type: "action", + ai: "optimized", + props: { + googleSearchConsole, + siteUrl: { + propDefinition: [ + googleSearchConsole, + "siteUrl", + ], + description: "Exact property identifier as returned by **List Sites** — `sc-domain:example.com` for a domain property, or a URL-prefix such as `https://www.example.com/` (trailing slash; scheme and subdomain must match exactly). Copy it verbatim; never construct it. Every URL in `inspectionUrls` must belong to this property, otherwise the call returns 403 \"User does not have sufficient permission for site\". Inspection requires `siteOwner` or `siteFullUser` on the property.", + }, + inspectionUrls: { + type: "string[]", + label: "URLs to Inspect", + description: "1-10 full absolute URLs to inspect, e.g. `[\"https://www.example.com/\", \"https://www.example.com/pricing\"]`. Each must live under the property given in `siteUrl`; paths alone (`/pricing`) are rejected. Batch the URLs into this one call rather than calling the action once per URL. Quota is 2,000 inspections per day AND 600 per minute per property; a quota error does not say which of the two limits was hit, so on a quota failure wait a minute before retrying and only then assume the daily cap.", + }, + languageCode: { + type: "string", + label: "Language Code", + description: "BCP-47 language code (e.g. `en-US`, `fr`, `pt-BR`) for the human-readable strings in the result, such as `coverageState`. Defaults to `en-US`. It does not change the verdicts or any other data.", + optional: true, + default: "en-US", + }, + includeFullResult: { + type: "boolean", + label: "Include Full Result", + description: "When `true`, attach the untrimmed API `inspectionResult` for each URL as `full_result` (rich results detail, AMP result, and everything else Google returns). Defaults to `false` because that payload is large and mostly rich-results and AMP detail; the curated fields already answer index-status, canonical and crawl questions. Set it to `true` only when the user asks for the complete or raw inspection result.", + optional: true, + default: false, + }, + }, + async run({ $ }) { + const { + siteUrl, inspectionUrls, languageCode, includeFullResult, + } = this; + + const trimmedSiteUrl = trimIfString(siteUrl); + + const urls = (Array.isArray(inspectionUrls) + ? inspectionUrls + : [ + inspectionUrls, + ]) + .map(trimIfString) + .filter((url) => typeof url === "string" && url !== ""); + + if (urls.length === 0) { + throw new Error("No URLs to inspect. Pass 1-10 full absolute URLs in `inspectionUrls`, e.g. [\"https://www.example.com/\"]."); + } + + if (urls.length > MAX_URLS) { + throw new Error(`Too many URLs: ${urls.length} given, but this action inspects at most ${MAX_URLS} per call. Split the list into batches of ${MAX_URLS} or fewer.`); + } + + const results = await mapWithConcurrency(urls, CONCURRENCY, async (url) => { + try { + const response = await this.googleSearchConsole.inspectUrl({ + $, + data: { + inspectionUrl: url, + siteUrl: trimmedSiteUrl, + languageCode, + }, + }); + return buildRow({ + url, + inspectionResult: response?.inspectionResult, + includeFullResult, + }); + } catch (error) { + // The contract is one row per URL: a single failure is reported in that + // row's `error` and must never abort the rest of the batch. + return buildRow({ + url, + error: error.response?.data?.error?.message || error.message, + includeFullResult, + }); + } + }); + + const total = results.length; + const indexed = results.filter((row) => row.verdict === "PASS").length; + const errors = results.filter((row) => row.error !== null).length; + const notIndexed = total - indexed - errors; + + const errorSuffix = errors > 0 + ? `, ${errors} error${errors === 1 + ? "" + : "s"}` + : ""; + + $.export("$summary", `Inspected ${total} URL${total === 1 + ? "" + : "s"}: ${indexed} indexed, ${notIndexed} not indexed${errorSuffix}`); + + return { + results, + summary: { + total, + indexed, + not_indexed: notIndexed, + errors, + }, + }; + }, +}; diff --git a/components/google_search_console/actions/list-sitemaps/list-sitemaps.mjs b/components/google_search_console/actions/list-sitemaps/list-sitemaps.mjs new file mode 100644 index 0000000000000..082a0210b9f1f --- /dev/null +++ b/components/google_search_console/actions/list-sitemaps/list-sitemaps.mjs @@ -0,0 +1,162 @@ +import googleSearchConsole from "../../google_search_console.app.mjs"; +import { trimIfString } from "../../common/utils.mjs"; + +export default { + name: "List Sitemaps", + description: "Lists the sitemaps Google Search Console knows about for a property, with their " + + "error, warning and freshness state normalized to numbers." + + "\n\n**Purpose.** Answers \"which sitemaps are submitted?\", \"do any of them have errors or " + + "warnings?\", \"when did Google last download this sitemap?\" and \"how many URLs does it " + + "contain?\" in a single call." + + "\n\n**When to use.** Use it for any sitemap question, and always before **Delete Sitemap** when " + + "the user has not named an exact sitemap URL. Use it after **Submit Sitemap** to confirm a " + + "submission landed, and after **Delete Sitemap** to confirm the path is gone — in both " + + "cases list everything (leave `sitemapUrl` empty) and look for the `path`; a single-path " + + "lookup of a deleted or unknown sitemap returns 404. Note that this tool reports what " + + "Google recorded about a sitemap; it does not fetch or parse the XML itself, and it does " + + "not tell you whether the individual URLs are indexed - use **Inspect URLs** for that." + + "\n\n**Returns.** `{ sitemaps: [{ path, type, isSitemapsIndex, isPending, lastSubmitted, " + + "lastDownloaded, warnings, errors, submitted_urls, contents }], count, summary: { with_errors, " + + "with_warnings, pending, never_downloaded } }`. `path` is the full sitemap URL and is the exact " + + "string to pass to **Submit Sitemap** or **Delete Sitemap**. `warnings`, `errors` and " + + "`submitted_urls` are integers here (the raw API returns them as strings); `submitted_urls` is the " + + "sum of `contents[].submitted`. `isPending: true` means Google has accepted the sitemap but has " + + "not fetched it yet. A sitemap with no `lastDownloaded` has never been downloaded and is counted " + + "in `summary.never_downloaded`. `contents` carries the per-type breakdown " + + "(`[{ type, submitted, indexed }]`) straight from the API." + + "\n\n**Cross-references.** Get `siteUrl` from **List Sites**. Submit or resubmit with **Submit " + + "Sitemap**. Unlist with **Delete Sitemap**. Check whether the pages inside a sitemap are actually " + + "indexed with **Inspect URLs**." + + "\n\n**Parameter guidance.** `siteUrl` is required and must be the exact property identifier. " + + "Leave `sitemapUrl` and `sitemapIndex` empty to list every sitemap for the property - that is the " + + "right call for \"list my sitemaps\" and for \"flag any with errors\". Set `sitemapUrl` to return " + + "just that one sitemap when the user names it. Set `sitemapIndex` to the URL of a sitemap index to " + + "list only the child sitemaps contained in that index. Do not set both." + + "\n\n**Common mistakes.** Do not guess a sitemap URL: an unknown path returns 404 " + + "\"'' is not a submitted or a known sitemap.\" - list first, then use a `path` from the " + + "result. A sitemap URL is a full URL (`https://www.example.com/sitemap.xml`), not a path " + + "(`/sitemap.xml`), and it must sit under the property. `lastDownloaded` can be years old while the " + + "sitemap is still valid; report the date rather than treating it as an error. `errors` and " + + "`warnings` count sitemap-parsing problems, not indexing problems." + + "\n\n**Example.** `siteUrl=\"sc-domain:example.com\"` (no other input) -> " + + "`{ count: 3, summary: { with_errors: 1, with_warnings: 1, pending: 0, never_downloaded: 0 }, " + + "sitemaps: [{ path: \"https://www.example.com/sitemap.xml\", type: \"sitemap\", " + + "isSitemapsIndex: false, isPending: false, lastSubmitted: \"2018-05-05T20:11:42.000Z\", " + + "lastDownloaded: \"2018-05-06T02:44:10.000Z\", errors: 1, warnings: 1, submitted_urls: 2, " + + "contents: [{ type: \"web\", submitted: \"2\", indexed: \"0\" }] }, ...] }` - so the answer to " + + "\"when did Google last download the sitemap and how many URLs does it have?\" is 6 May 2018 and " + + "2 URLs." + + "\n\n[See the documentation](https://developers.google.com/webmaster-tools/v1/sitemaps/list)", + key: "google_search_console-list-sitemaps", + version: "0.0.1", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, + type: "action", + ai: "optimized", + props: { + googleSearchConsole, + siteUrl: { + propDefinition: [ + googleSearchConsole, + "siteUrl", + ], + }, + sitemapUrl: { + propDefinition: [ + googleSearchConsole, + "sitemapUrl", + ], + optional: true, + description: "Optional. Full URL of a single sitemap to return just this one sitemap, e.g. " + + "`https://www.example.com/sitemap.xml`. Leave empty to list every sitemap submitted " + + "for the property. An unknown URL returns 404 \"'' is not a submitted or a known " + + "sitemap.\" Do not use this to check whether a sitemap exists or was deleted — list " + + "everything instead and look for the `path`.", + }, + sitemapIndex: { + type: "string", + label: "Sitemap Index URL", + description: "Optional. Full URL of a sitemap index, e.g. " + + "`https://www.example.com/sitemap_index.xml`. When set, only the child sitemaps " + + "contained in that index are listed. Leave empty to list all sitemaps for the property, and " + + "do not set it together with the single Sitemap URL (the call fails if both are set). To find " + + "an index URL, call this action with no filters first and take the `path` of an entry whose " + + "`isSitemapsIndex` is `true`.", + optional: true, + }, + }, + async run({ $ }) { + const { + googleSearchConsole, + siteUrl, + sitemapUrl, + sitemapIndex, + } = this; + + const trimmedSiteUrl = trimIfString(siteUrl); + const trimmedSitemapUrl = trimIfString(sitemapUrl); + const trimmedSitemapIndex = trimIfString(sitemapIndex); + + if (trimmedSitemapUrl && trimmedSitemapIndex) { + throw new Error("Set either `sitemapUrl` (return one sitemap) or `sitemapIndex` (list the children of an index), not both."); + } + + let records; + + if (trimmedSitemapUrl) { + const record = await googleSearchConsole.getSitemap({ + $, + siteUrl: trimmedSiteUrl, + sitemapUrl: trimmedSitemapUrl, + }); + records = [ + record, + ]; + } else { + const params = {}; + if (trimmedSitemapIndex) { + params.sitemapIndex = trimmedSitemapIndex; + } + const response = await googleSearchConsole.listSitemaps({ + $, + siteUrl: trimmedSiteUrl, + params, + }); + records = response?.sitemap ?? []; + } + + const sitemaps = records.map((record) => { + const contents = record?.contents ?? []; + return { + path: record?.path ?? null, + type: record?.type ?? null, + isSitemapsIndex: record?.isSitemapsIndex ?? false, + isPending: record?.isPending ?? false, + lastSubmitted: record?.lastSubmitted ?? null, + lastDownloaded: record?.lastDownloaded ?? null, + warnings: Number(record?.warnings ?? 0), + errors: Number(record?.errors ?? 0), + submitted_urls: contents.reduce((total, entry) => total + Number(entry?.submitted ?? 0), 0), + contents, + }; + }); + + const summary = { + with_errors: sitemaps.filter((sitemap) => sitemap.errors > 0).length, + with_warnings: sitemaps.filter((sitemap) => sitemap.warnings > 0).length, + pending: sitemaps.filter((sitemap) => sitemap.isPending).length, + never_downloaded: sitemaps.filter((sitemap) => !sitemap.lastDownloaded).length, + }; + + $.export("$summary", `Found ${sitemaps.length} sitemap(s) for ${trimmedSiteUrl}: ${summary.with_errors} with errors, ${summary.with_warnings} with warnings`); + + return { + sitemaps, + count: sitemaps.length, + summary, + }; + }, +}; diff --git a/components/google_search_console/actions/list-sites/list-sites.mjs b/components/google_search_console/actions/list-sites/list-sites.mjs new file mode 100644 index 0000000000000..8089ac280e121 --- /dev/null +++ b/components/google_search_console/actions/list-sites/list-sites.mjs @@ -0,0 +1,102 @@ +import googleSearchConsole from "../../google_search_console.app.mjs"; + +const DOMAIN_PREFIX = "sc-domain:"; + +export default { + name: "List Sites", + description: "Lists every Google Search Console property the connected Google account can access, " + + "together with the email address of that account." + + "\n\n**Purpose.** Property identifiers are opaque strings that every other Search Console tool " + + "requires byte-for-byte. This tool is where they come from, and it is also the only tool that " + + "reports which Google account is connected." + + "\n\n**When to use.** Call this FIRST on any per-site task, before running traffic, sitemap or " + + "index-status tools, unless the user already gave you an exact identifier such as " + + "`sc-domain:example.com`. Call it again after any 403 to see what the account really has. " + + "Do NOT call it when Search Console cannot do the task at all - backlinks or the Links " + + "report, requesting indexing of an ordinary page, adding or removing property owners: say " + + "that first, and call this only if the user then asks for something the tools can do." + + "\n\n**Returns.** `{ account_email, sites: [{ siteUrl, permissionLevel, property_type }], count }`. " + + "`property_type` is `\"domain\"` when `siteUrl` starts with `sc-domain:` (that property covers " + + "every subdomain and both http and https) and `\"url_prefix\"` otherwise (an exact scheme + host " + + "+ path prefix, trailing slash included). Domain properties are listed first, then alphabetically. " + + "The list is complete: the underlying API has no pagination. `account_email` is `null` when " + + "the account's email could not be read; the site list is still returned." + + "\n\n**Permission levels.** `siteOwner` - full access; can submit and delete sitemaps and inspect " + + "URLs. `siteFullUser` - same data and sitemap access as an owner, but cannot manage users. " + + "`siteRestrictedUser` - read-only on most reports; **cannot submit sitemaps and cannot inspect " + + "URLs** (those calls return 403). `siteUnverifiedUser` - verification was never completed, so no " + + "data is available. Check the level here before promising a write." + + "\n\n**Cross-references.** Pass the `siteUrl` you find here into **Query Search Analytics** " + + "(traffic, queries, pages), **Compare Search Analytics** (period-over-period deltas), **Inspect " + + "URLs** (index status of individual pages), **List Sitemaps** (sitemap health) and **Submit " + + "Sitemap** (submit or resubmit a sitemap)." + + "\n\n**Parameter guidance.** None. This action takes no parameters." + + "\n\n**Common mistakes.** Never construct a property identifier by hand; copy `siteUrl` verbatim " + + "from this list. When several variants of one host are present (for example " + + "`sc-domain:example.com`, `https://www.example.com/` and `http://example.com/`), prefer the " + + "`sc-domain:` property for traffic questions because it aggregates every subdomain and scheme - " + + "unless the user names a specific prefix, in which case use exactly that one. A URL-prefix " + + "identifier missing its trailing slash, or with the wrong scheme or subdomain, returns 403 " + + "\"User does not have sufficient permission for site\" even when the account is authorized." + + "\n\n**Example.** No inputs -> " + + "`{ account_email: \"owner@example.com\", count: 5, sites: [" + + "{ siteUrl: \"sc-domain:example.com\", permissionLevel: \"siteOwner\", property_type: \"domain\" }, " + + "{ siteUrl: \"http://example.com/\", permissionLevel: \"siteOwner\", property_type: \"url_prefix\" }, " + + "{ siteUrl: \"https://www.example.com/\", permissionLevel: \"siteOwner\", property_type: \"url_prefix\" }] }`. " + + "For \"how did example.com do last month?\" you would then call **Query Search Analytics** " + + "with `siteUrl=\"sc-domain:example.com\"`." + + "\n\n[See the documentation](https://developers.google.com/webmaster-tools/v1/sites/list)", + key: "google_search_console-list-sites", + version: "0.0.1", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, + type: "action", + ai: "optimized", + props: { + googleSearchConsole, + }, + async run({ $ }) { + const [ + sitesResponse, + userInfo, + ] = await Promise.all([ + this.googleSearchConsole.getSites({ + $, + }), + this.googleSearchConsole.getUserInfo({ + $, + }) + .catch(() => null), + ]); + + const sites = (sitesResponse?.siteEntry ?? []).map((entry) => ({ + siteUrl: entry?.siteUrl, + permissionLevel: entry?.permissionLevel, + property_type: String(entry?.siteUrl ?? "").startsWith(DOMAIN_PREFIX) + ? "domain" + : "url_prefix", + })); + + sites.sort((a, b) => { + if (a.property_type !== b.property_type) { + return a.property_type === "domain" + ? -1 + : 1; + } + return String(a.siteUrl).localeCompare(String(b.siteUrl)); + }); + + const accountEmail = userInfo?.email ?? null; + + $.export("$summary", `Listed ${sites.length} Search Console properties for ${accountEmail ?? "the connected account"}`); + + return { + account_email: accountEmail, + sites, + count: sites.length, + }; + }, +}; diff --git a/components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs b/components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs index 0f5c8bc5bb125..2f0f3659ea810 100644 --- a/components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs +++ b/components/google_search_console/actions/retrieve-site-performance-data/retrieve-site-performance-data.mjs @@ -1,17 +1,37 @@ import googleSearchConsole from "../../google_search_console.app.mjs"; +import { buildDimensionFilterGroups } from "../../common/filters.mjs"; import { trimIfString } from "../../common/utils.mjs"; +const DEFAULT_ROW_LIMIT = 50; + export default { - name: "Retrieve Site Performance Data", - description: "Fetches search analytics (clicks, impressions, CTR, position) from Google Search Console for a verified site. Use it to pull traffic metrics, optionally broken down by dimensions like page or query. Filter to a subset of pages with Subdomain Filter (a simple contains-style path/subdomain match), or with Advanced Dimension Filters (a JSON array of filter groups per the Search Console API, used only when Subdomain Filter is empty). Max Rows caps how many rows come back per call; use Start Row to page through more. [See the documentation](https://developers.google.com/webmaster-tools/v1/searchanalytics/query)", + name: "Query Search Analytics", + description: "Query Google Search Console search analytics for one property and one date range: clicks, impressions, CTR and average position, optionally grouped by dimensions and filtered. This is the main traffic-reporting tool for a site.\n\n" + + "**When to use:** any single-date-range question about how a site performs in Google Search — top queries, top pages, country or device splits, daily or hourly trends, the CTR or average position of a term. Use **Compare Search Analytics** instead for period-over-period questions (month over month, quarter over quarter, year over year, \"did the update hurt us\") — it fetches both ranges and joins them for you. Use **Inspect URLs** for index status, canonicals and crawl state; this tool only reports traffic.\n\n" + + "**Returns:** the API response unchanged — `rows` (each `{ keys, clicks, impressions, ctr, position }`, where `keys` lines up positionally with `dimensions`), `responseAggregationType` and `metadata` — plus four added fields: `row_count`; `has_more` (true when the page came back full, so more rows probably exist); `next_start_row` (pass it back as `startRow` for the next page); and `returned_totals` (`clicks` and `impressions` summed over the RETURNED rows only).\n\n" + + "**Cross-references:** call **List Sites** first when the user names a site in prose rather than giving an exact property identifier — it also reports your permission level. **Compare Search Analytics** answers two-period questions. **Inspect URLs** tells you whether a page reported here is actually indexed.\n\n" + + "**Parameter guidance:**\n" + + "- `startDate`/`endDate` are `YYYY-MM-DD`, **Pacific Time**, and inclusive. Data is final only after about 2-3 days, and retention is 16 months — a `startDate` older than that returns a 400.\n" + + "- `dimensions` groups the rows. `searchAppearance` cannot be combined with any other dimension (two-pass pattern: fetch the appearance types alone, then filter by one). `hour` cannot be combined with `date`, requires `dataState: hourly_all`, and spans at most 10 days — combining them returns 400 \"Request cannot be grouped by both date and hour\". Discover has no `query` dimension: 400 \"Request for DISCOVER cannot be grouped by query\" — when the user asks for Discover queries, do not stop to ask; report Discover pages instead (`searchType: discover`, `dimensions: [\"page\"]`) and say why. With no dimensions you get a single totals row. With `date` (and `hour`) the API returns a row for EVERY day in the range, including days with 0 impressions — when counting active days, count rows with `impressions > 0`, not rows.\n" + + "- `aggregationType: byProperty` is rejected whenever a `page` dimension or a page filter is present: 400 \"'BY_PROPERTY' is not a valid aggregation type in the context of the request.\" Average position under `byPage` is not the same number as under `byProperty` — the property-level average position is the `byProperty` one.\n" + + "- `rowLimit` defaults to 50 and maxes at 25000, but keep it at 200 or below per call: 100 rows is about 13k characters of output and 400 rows already exceeds the output cap and gets spilled to a file you cannot read. For long `date` sweeps use `rowLimit: 200` and page with `startRow`. Page with `startRow` while `has_more` is true; there is no page token. The last page is the one where `rows.length < rowLimit`, and an empty final page is normal.\n" + + "- Filtering: set `subdomainFilter` (the filter VALUE, despite the legacy name) with `filterDimension`/`filterOperator` for the common single-condition case, or `advancedDimensionFilters` for multi-condition filtering. `subdomainFilter` wins — `advancedDimensionFilters` is ignored when it is set.\n\n" + + "**Common mistakes:**\n" + + "- Constructing the property identifier instead of copying it from **List Sites**. `sc-domain:example.com` and `https://www.example.com/` are different properties, and the wrong string returns 403.\n" + + "- Treating `returned_totals` as the property total. It is not, especially when grouping by `query`: Google omits anonymized (rare) queries, so the sum of query rows is materially LESS than the same range grouped by `date`. For a true total, query with no dimensions or with `date`.\n" + + "- Re-averaging `ctr` or `position` across rows. Both are impression-weighted; a plain mean is wrong. `ctr` is a 0-1 fraction (0.1428 means 14.3%) and `position` is a 1-indexed float where lower is better.\n" + + "- Answering from a truncated first page. Check `has_more` before reporting a count or a \"top N\".\n\n" + + "**Example:** `siteUrl=\"sc-domain:example.com\"`, `startDate=\"2025-09-01\"`, `endDate=\"2026-08-31\"`, `dimensions=[\"query\"]`, `rowLimit=10` returns rows such as `{ keys: [\"example brand\"], clicks: 41, impressions: 287, ctr: 0.1429, position: 2.4 }` plus `row_count: 10`, `has_more: true`, `next_start_row: 10` and `returned_totals: { clicks: 41, impressions: 294 }`.\n\n" + + "There is no `fields` parameter — rows are already minimal, so `rowLimit` plus `has_more`/`next_start_row` is the payload lever. Quota: 1,200 queries per minute per site. [See the documentation](https://developers.google.com/webmaster-tools/v1/searchanalytics/query)", key: "google_search_console-retrieve-site-performance-data", - version: "1.0.0", + version: "1.1.0", annotations: { destructiveHint: false, openWorldHint: true, readOnlyHint: true, }, type: "action", + ai: "optimized", props: { googleSearchConsole, siteUrl: { @@ -19,118 +39,96 @@ export default { googleSearchConsole, "siteUrl", ], - description: "Select a verified site from your Google Search Console. For subdomains, select the domain property and use dimension filters.", }, startDate: { type: "string", label: "Start Date (YYYY-MM-DD)", - description: "Start date of the range for which to retrieve site performance data", + description: "First day of the range, inclusive, in `YYYY-MM-DD` **Pacific Time** — e.g. `2025-09-01`. Search Console keeps 16 months of data; a start date outside that window returns a 400. Data for the most recent 2-3 days is not final yet, so it is omitted unless `dataState` is `all`.", }, endDate: { type: "string", label: "End Date (YYYY-MM-DD)", - description: "End date of the range for which to retrieve site performance data", + description: "Last day of the range, inclusive, in `YYYY-MM-DD` **Pacific Time** — e.g. `2026-08-31`. Because reporting lags by about 2-3 days, an end date of today usually returns nothing for the final days unless `dataState` is `all`.", }, dimensions: { type: "string[]", label: "Dimensions", optional: true, - description: "e.g. ['query', 'page', 'country', 'device']", + description: "How to group the rows. Each row's `keys` array lines up positionally with this list, so `[\"query\",\"device\"]` yields `keys: [\"example brand\",\"MOBILE\"]`. Leave empty for a single totals row. Rules: `searchAppearance` cannot be combined with ANY other dimension (fetch the appearance types alone, then filter by one with `filterDimension: searchAppearance`); `hour` cannot be combined with `date`, requires `dataState: hourly_all`, and covers at most 10 days — otherwise the API returns 400 \"Request cannot be grouped by both date and hour\"; Discover (`searchType: discover`) has no `query` dimension and returns 400 \"Request for DISCOVER cannot be grouped by query\". `date` returns one row per calendar day in the range, zero-impression days included.", options: [ + "query", + "page", "country", "device", - "page", - "query", "searchAppearance", "date", + "hour", ], }, searchType: { - type: "string", - label: "Search Type", - description: "The type of search to use", - optional: true, - options: [ - "web", - "image", - "video", - "news", - "googleNews", - "discover", + propDefinition: [ + googleSearchConsole, + "searchType", ], - default: "web", }, aggregationType: { type: "string", label: "Aggregation Type", - description: "The aggregation type to use", + description: "How Google aggregates the metrics. `auto` (the default behaviour) lets Google choose — by page when grouping by page, by property otherwise. `byPage` aggregates by URI; `byProperty` aggregates across the whole property; `byNewsShowcasePanel` is for News Showcase reporting. `byProperty` is rejected whenever a `page` dimension or a page filter is present: the API returns 400 \"'BY_PROPERTY' is not a valid aggregation type in the context of the request.\" Average position differs between the two: the property-level average position is the `byProperty` number, and it does not equal the average of the `byPage` rows.", optional: true, options: [ "auto", "byPage", + "byProperty", + "byNewsShowcasePanel", ], }, rowLimit: { type: "integer", label: "Max Rows", - description: "Max number of rows to return", - default: 10, + description: "Rows to return in this call. Defaults to 50; the API maximum is 25000. Size it to the task — 5-10 for a \"top query\" answer, more for a sweep — and keep it at 200 or below per call (100 rows ≈ 13k characters; 400 rows exceeds the output cap and is spilled to a file you cannot read). When `has_more` comes back true, request the next page with `startRow: next_start_row`; there is no page token. Stop when a page returns fewer rows than `rowLimit` (an empty final page is normal).", + default: DEFAULT_ROW_LIMIT, optional: true, }, startRow: { type: "integer", label: "Start Row", - description: "Start row (for pagination)", + description: "Zero-based index of the first row to return. Omit for the first page, then pass the `next_start_row` value from the previous response to page forward.", optional: true, }, subdomainFilter: { type: "string", - label: "Subdomain Filter", + label: "Filter Value", optional: true, - description: "Filter results to a specific subdomain when using a domain property (e.g., `https://subdomain.example.com`). This will include all subpages of the subdomain.", + description: "The value to filter on, for ANY dimension — not just subdomains (the prop key is legacy). It is combined with `filterDimension` and `filterOperator` into a single filter: for example `filterDimension: page`, `filterOperator: contains`, value `https://www.example.com/blog/`. `page` expressions match the FULL URL (scheme and host included), not just a path. String comparison is case-insensitive; regex operators use RE2. When this is set, `advancedDimensionFilters` is ignored.", }, filterDimension: { - type: "string", - label: "Filter Dimension", - optional: true, - description: "Dimension to filter by (defaults to page when subdomain filter is used). Using 'page' will match the subdomain and all its subpages.", - options: [ - "country", - "device", - "page", - "query", + propDefinition: [ + googleSearchConsole, + "filterDimension", ], - default: "page", }, filterOperator: { - type: "string", - label: "Filter Operator", - optional: true, - description: "Operator to use for filtering (defaults to contains when subdomain filter is used)", - options: [ - "contains", - "equals", - "notContains", - "notEquals", - "includingRegex", - "excludingRegex", + propDefinition: [ + googleSearchConsole, + "filterOperator", ], - default: "contains", }, advancedDimensionFilters: { - type: "string", - label: "Advanced Dimension Filters", - optional: true, - description: "A JSON-encoded array of dimension filter groups, following the Search Console API structure — it must be an array, even for a single group. Example: `[{\"groupType\":\"and\",\"filters\":[{\"dimension\":\"page\",\"operator\":\"contains\",\"expression\":\"https://www.example.com/docs\"}]}]`. Used only when Subdomain Filter is empty.", + propDefinition: [ + googleSearchConsole, + "advancedDimensionFilters", + ], }, dataState: { type: "string", label: "Data State", - description: "The data state to use", + description: "Which data to include. `final` (default) returns only finalized data, which lags roughly 2-3 days behind today. `all` also includes the most recent, not-yet-final days and adds `metadata.firstIncompleteDate` to the response — use it when the user explicitly wants recent or partial numbers. `hourly_all` is required by the `hour` dimension.", optional: true, options: [ - "all", "final", + "all", + "hourly_all", ], default: "final", }, @@ -139,45 +137,51 @@ export default { const { googleSearchConsole, siteUrl, + startDate, + endDate, + dimensions, + searchType, + aggregationType, + rowLimit, + startRow, subdomainFilter, filterDimension, filterOperator, advancedDimensionFilters, - ...fields + dataState, } = this; - const body = Object.entries(fields).reduce((acc, [ - key, - value, - ]) => { - acc[key] = trimIfString(value); - return acc; - }, {}); + const dimensionFilterGroups = buildDimensionFilterGroups({ + app: googleSearchConsole, + filterValue: subdomainFilter, + filterDimension, + filterOperator, + advancedDimensionFilters, + }); - // Build dimension filters based on user input - let dimensionFilterGroups; + // The API's deprecated key is `searchType`; the supported one is `type`. The body is + // assembled explicitly so no prop name leaks into the request by accident. + const effectiveRowLimit = rowLimit ?? DEFAULT_ROW_LIMIT; + const effectiveStartRow = startRow ?? 0; - // Normalized once so whitespace-only input is treated as absent, leaving - // advancedDimensionFilters eligible instead of sending an empty-string filter. - const trimmedSubdomainFilter = trimIfString(subdomainFilter); + const data = { + startDate: trimIfString(startDate), + endDate: trimIfString(endDate), + dimensions: dimensions?.length + ? dimensions.map((dimension) => trimIfString(dimension)) + : undefined, + type: trimIfString(searchType), + aggregationType: trimIfString(aggregationType), + rowLimit: effectiveRowLimit, + startRow: effectiveStartRow, + dataState: trimIfString(dataState), + dimensionFilterGroups, + }; - if (trimmedSubdomainFilter) { - // If user provided a subdomain filter, create the filter structure - dimensionFilterGroups = [ - { - groupType: "and", - filters: [ - { - dimension: filterDimension || "page", - operator: filterOperator || "contains", - expression: trimmedSubdomainFilter, - }, - ], - }, - ]; - } else if (advancedDimensionFilters) { - // If user provided advanced filters, use those - dimensionFilterGroups = googleSearchConsole.parseIfJsonString(advancedDimensionFilters); + for (const key of Object.keys(data)) { + if (data[key] === undefined) { + delete data[key]; + } } let response; @@ -185,28 +189,54 @@ export default { response = await googleSearchConsole.getSitePerformanceData({ $, url: siteUrl, - data: { - ...body, - dimensionFilterGroups, - }, + data, }); } catch (error) { - // Identify if the error was thrown by internal validation or by the API call - const thrower = googleSearchConsole.checkWhoThrewError(error); - - // Add more helpful error messages for common 403 errors + // The only permitted error rewrite. Google returns the same 403 for four unrelated + // causes, so name the properties this account really has. if (error.response?.status === 403) { - const message = "Access denied. If you're trying to access a subdomain, select the domain property (sc-domain:example.com) and use the subdomain filter to filter for your subdomain."; - throw new Error(`Failed to fetch data: ${message}`); + // If the token or scope is what failed, listing the sites fails too — surface the + // original Search Console error rather than the lookup's rejection. + let list; + try { + const sites = await googleSearchConsole.getSites({ + $, + }); + list = (sites?.siteEntry ?? []) + .map((site) => `${site.siteUrl} (${site.permissionLevel})`) + .join(", "); + } catch { + throw error; + } + throw new Error(`Access denied for "${trimIfString(siteUrl)}". Properties this account can access: ${list}. Use the exact string from List Sites (domain properties look like sc-domain:example.com; URL-prefix properties need the trailing slash).`); } - - throw new Error(`Failed to fetch data (${thrower.whoThrew} error): ${error.message}`); + throw error; } - const rowCount = response.rows?.length || 0; - $.export("$summary", `Fetched ${rowCount} ${rowCount === 1 - ? "row" - : "rows"} of data.`); - return response; + const rows = response?.rows ?? []; + const returnedTotals = rows.reduce((acc, row) => ({ + clicks: acc.clicks + (row.clicks || 0), + impressions: acc.impressions + (row.impressions || 0), + }), { + clicks: 0, + impressions: 0, + }); + + const hasMore = rows.length === effectiveRowLimit; + const nextStartRow = effectiveStartRow + rows.length; + + $.export("$summary", hasMore + ? `Fetched ${rows.length} rows (more available — next startRow ${nextStartRow})` + : `Fetched ${rows.length} ${rows.length === 1 + ? "row" + : "rows"}`); + + return { + ...response, + row_count: rows.length, + has_more: hasMore, + next_start_row: nextStartRow, + returned_totals: returnedTotals, + }; }, }; diff --git a/components/google_search_console/actions/submit-sitemap/submit-sitemap.mjs b/components/google_search_console/actions/submit-sitemap/submit-sitemap.mjs new file mode 100644 index 0000000000000..0b24fba9baad2 --- /dev/null +++ b/components/google_search_console/actions/submit-sitemap/submit-sitemap.mjs @@ -0,0 +1,124 @@ +import googleSearchConsole from "../../google_search_console.app.mjs"; +import { trimIfString } from "../../common/utils.mjs"; + +export default { + name: "Submit Sitemap", + description: "Submits a sitemap (or resubmits one that is already listed) to Google Search Console " + + "for a property, then reads the stored record back." + + "\n\n**Purpose.** Tells Google where a sitemap lives and asks it to fetch it. Resubmitting a " + + "sitemap that is already listed is the SUPPORTED way to ask Google to re-read it, and the call is " + + "idempotent: the re-submit does not create a duplicate, it updates `lastSubmitted` and sets " + + "`isPending: true`." + + "\n\n**When to use.** Use it when the user adds a new sitemap, or when the user wants Google to " + + "pick up new or changed pages on a site. **There is no API to request indexing of a single " + + "ordinary page** - the \"Request indexing\" button in the Search Console UI has no API equivalent, " + + "and the Indexing API behind **Submit URL for Indexing** is only for JobPosting and BroadcastEvent " + + "pages. So the two legitimate options are this tool (have Google re-read the sitemap that contains " + + "the page) and **Inspect URLs** (check the page's current index status). Say that plainly rather " + + "than implying a page can be force-indexed - and when the user asks to request indexing or " + + "force a recrawl, OFFER these two options and wait for the user to pick one; do not run " + + "either unasked." + + "\n\n**Returns.** `{ submitted: true, sitemap, previous_last_submitted }`. `sitemap` is the record " + + "read back from Google immediately after submission (`path`, `type`, `isPending`, " + + "`isSitemapsIndex`, `lastSubmitted`, `lastDownloaded`, `warnings`, `errors`, `contents`) - the " + + "submit call itself returns an empty body, so this read-back is how you confirm it landed. " + + "`previous_last_submitted` is the `lastSubmitted` timestamp the sitemap had before this call, or " + + "`null` for a first submission; use it to tell a fresh submission from a resubmission. Right after " + + "a submit `isPending` is normally `true` and `lastDownloaded` still holds the old date (or is " + + "absent): Google fetches the file asynchronously, usually within minutes to days. `isPending` stays " + + "true until Google fetches it, so do not report a sitemap as \"processed\" or \"indexed\" on the " + + "strength of a successful submit." + + "\n\n**Cross-references.** Get the exact `siteUrl` from **List Sites**, and check there that the " + + "account is `siteOwner` or `siteFullUser` first - a `siteRestrictedUser` cannot submit sitemaps and " + + "gets a 403. Use **List Sitemaps** to find the exact `path` of an existing sitemap before " + + "resubmitting it, and again afterwards to confirm the new state. Use **Inspect URLs** to check " + + "whether specific pages are indexed. Use **Delete Sitemap** to unlist one." + + "\n\n**Parameter guidance.** `siteUrl` is the property identifier, copied verbatim from **List " + + "Sites**. `sitemapUrl` is the full URL of the sitemap file or sitemap index " + + "(`https://www.example.com/sitemap.xml`), and it must live under the property: for a domain " + + "property any subdomain and either scheme qualifies, but for a URL-prefix property the scheme, " + + "host and path prefix must match. Both are required; if the user has not given you a sitemap URL, " + + "ask for it rather than guessing a conventional path." + + "\n\n**Common mistakes.** Do not pass a page URL, a path (`/sitemap.xml`) or a property identifier " + + "as `sitemapUrl` - it must be an absolute URL to the sitemap file. A sitemap URL outside the " + + "property is rejected. Submitting is not the same as indexing: Google may still choose not to index " + + "the URLs it finds. Do not delete and resubmit a sitemap to \"refresh\" it - just resubmit." + + "\n\n**Example.** `siteUrl=\"sc-domain:example.com\"`, " + + "`sitemapUrl=\"https://www.example.com/sitemap.xml\"` -> " + + "`{ submitted: true, previous_last_submitted: \"2018-05-05T20:11:42.000Z\", sitemap: " + + "{ path: \"https://www.example.com/sitemap.xml\", type: \"sitemap\", isPending: true, " + + "isSitemapsIndex: false, lastSubmitted: \"2026-09-02T14:03:11.000Z\", " + + "lastDownloaded: \"2018-05-06T02:44:10.000Z\", errors: \"1\", warnings: \"1\" } }` - a " + + "resubmission (there was a previous `lastSubmitted`) that is now pending a fresh fetch by Google." + + "\n\n[See the documentation](https://developers.google.com/webmaster-tools/v1/sitemaps/submit)", + key: "google_search_console-submit-sitemap", + version: "0.0.1", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: false, + }, + type: "action", + ai: "optimized", + props: { + googleSearchConsole, + siteUrl: { + propDefinition: [ + googleSearchConsole, + "siteUrl", + ], + }, + sitemapUrl: { + propDefinition: [ + googleSearchConsole, + "sitemapUrl", + ], + }, + }, + async run({ $ }) { + const { + googleSearchConsole, + siteUrl, + sitemapUrl, + } = this; + + const trimmedSiteUrl = trimIfString(siteUrl); + const trimmedSitemapUrl = trimIfString(sitemapUrl); + + // Read the current list first: a GET on a path Google does not know about 404s, so the list is + // the only safe way to learn whether this is a first submission or a resubmission. + const existing = await googleSearchConsole.listSitemaps({ + $, + siteUrl: trimmedSiteUrl, + }); + + const priorRecord = (existing?.sitemap ?? []) + .find((entry) => entry?.path === trimmedSitemapUrl); + + await googleSearchConsole.submitSitemap({ + $, + siteUrl: trimmedSiteUrl, + sitemapUrl: trimmedSitemapUrl, + }); + + const sitemap = await googleSearchConsole.getSitemap({ + $, + siteUrl: trimmedSiteUrl, + sitemapUrl: trimmedSitemapUrl, + }); + + const previousLastSubmitted = priorRecord?.lastSubmitted ?? null; + + const verb = previousLastSubmitted + ? "Resubmitted" + : "Submitted"; + + $.export("$summary", `${verb} sitemap ${trimmedSitemapUrl} for ${trimmedSiteUrl}`); + + return { + submitted: true, + sitemap, + previous_last_submitted: previousLastSubmitted, + }; + }, +}; diff --git a/components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs b/components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs index 8a53e104d75ae..d17a19673ad92 100644 --- a/components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs +++ b/components/google_search_console/actions/submit-url-for-indexing/submit-url-for-indexing.mjs @@ -3,21 +3,68 @@ import { trimIfString } from "../../common/utils.mjs"; export default { name: "Submit URL for Indexing", - description: "Sends a URL update notification to the Google Indexing API", + description: "Sends a `URL_UPDATED` or `URL_DELETED` notification for one page to Google's " + + "**Indexing API**." + + "\n\n**Purpose.** The Indexing API is a SEPARATE API from Search Console " + + "(`indexing.googleapis.com`, not the Search Console reporting API). It tells Google that a " + + "specific page has been published, updated or removed." + + "\n\n**When to use.** Google supports it ONLY for pages that carry **JobPosting** or " + + "**BroadcastEvent** (livestream `VideoObject`) structured data. Use it for a job listing that was " + + "posted, changed or filled, or for a livestream page going live or ending. The default quota is " + + "**200 notifications per day** per project, and the connected Google account must be a **verified " + + "owner** of the site the URL belongs to." + + "\n\n**Do NOT use it to \"request indexing\" for an ordinary page - no API does that.** The " + + "\"Request indexing\" button in the Search Console UI has no API equivalent, and calling this tool " + + "for a normal page does not get it crawled sooner; Google ignores or rejects notifications for " + + "pages without the supported structured data. When a user asks you to request indexing, recrawl or " + + "\"push\" an ordinary page, say that no API can do it and offer the two real options instead: " + + "**Submit Sitemap** to have Google re-read the sitemap that contains the page, and **Inspect URLs** " + + "to check the page's current index status and last crawl time." + + "\n\n**Returns.** Google's `urlNotificationMetadata` for the URL: the notified `url` plus " + + "`latestUpdate` / `latestRemove` objects carrying `type` and `notifyTime`. A successful response " + + "means the notification was accepted, NOT that the page was crawled or indexed." + + "\n\n**Cross-references.** **Submit Sitemap** (ask Google to re-read a sitemap - the correct tool " + + "for ordinary pages), **Inspect URLs** (index status, canonical and last crawl time for up to 10 " + + "URLs), **List Sitemaps** (which sitemaps exist and whether they have errors), **List Sites** (to " + + "confirm the account owns the property)." + + "\n\n**Parameter guidance.** `URL for indexing` (the `siteUrl` prop) is the **page URL to notify " + + "about**, not a property identifier - the prop name is misleading and is kept only for backwards " + + "compatibility with existing workflows. Pass a full canonical page URL such as " + + "`https://www.example.com/jobs/paleobotanist`; never pass `sc-domain:example.com` or a bare " + + "property prefix. `Notification Type` is `URL_UPDATED` when the page was added or changed and " + + "`URL_DELETED` when the page has been taken down (only send `URL_DELETED` after the page actually " + + "returns 404 or 410)." + + "\n\n**Common mistakes.** Passing a property identifier or a site root instead of the page URL; " + + "using it as a general \"index this page\" button; expecting it to work on a page without " + + "JobPosting or BroadcastEvent markup; assuming acceptance means the page is indexed; and burning " + + "the 200/day quota on ordinary pages." + + "\n\n**Example.** `siteUrl=\"https://www.example.com/jobs/velociraptor-handler\"`, " + + "`notificationType=\"URL_UPDATED\"` -> `{ urlNotificationMetadata: { url: " + + "\"https://www.example.com/jobs/velociraptor-handler\", latestUpdate: { url: \"...\", " + + "type: \"URL_UPDATED\", notifyTime: \"2026-09-02T14:03:11.000Z\" } } }`. For " + + "`https://www.example.com/` - an ordinary page with no JobPosting or BroadcastEvent markup " + + "- do not call this tool at all; use **Submit Sitemap** or **Inspect URLs**." + + "\n\n[See the documentation](https://developers.google.com/search/apis/indexing-api/v3/using-api)", key: "google_search_console-submit-url-for-indexing", - version: "0.0.5", + version: "0.0.6", annotations: { - destructiveHint: true, + destructiveHint: false, openWorldHint: true, readOnlyHint: false, }, type: "action", + ai: "optimized", props: { googleSearchConsole, siteUrl: { type: "string", label: "URL for indexing", - description: "URL to be submitted for indexing (must be a canonical URL that's verified in Google Search Console)", + description: "The full page URL to notify Google about, e.g. " + + "`https://www.example.com/jobs/paleobotanist`. This is a PAGE URL, not a Search Console " + + "property identifier - never pass `sc-domain:example.com` or a bare property prefix here " + + "(the prop name is kept for backwards compatibility). It must be the canonical URL of a page " + + "on a site the connected account is a verified owner of, and the page must carry JobPosting " + + "or BroadcastEvent structured data.", }, notificationType: { type: "string", @@ -49,29 +96,13 @@ export default { warnings.push(...urlCheck.warnings); } - let response; - try { - response = await this.googleSearchConsole.submitUrlForIndexing({ - $, - data: { - url: trimmedUrl, - type: notificationType, - }, - }); - } catch (error) { - const thrower = this.googleSearchConsole.checkWhoThrewError(error); - - // Add more helpful error messages for common errors - if (error.response?.status === 403) { - throw new Error("Access denied. Make sure the URL belongs to a property you have access to in Google Search Console."); - } - - if (error.response?.status === 400) { - throw new Error("Invalid request. Ensure the URL is canonical and belongs to a verified property."); - } - - throw new Error(`Failed to submit URL (${thrower.whoThrew} error): ${error.message}`); - } + const response = await this.googleSearchConsole.submitUrlForIndexing({ + $, + data: { + url: trimmedUrl, + type: notificationType, + }, + }); // Format warnings string if any warnings exist const warningsString = warnings.length > 0 diff --git a/components/google_search_console/common/compare.mjs b/components/google_search_console/common/compare.mjs new file mode 100644 index 0000000000000..28ee66d3c0886 --- /dev/null +++ b/components/google_search_console/common/compare.mjs @@ -0,0 +1,218 @@ +const ZERO_METRICS = { + clicks: 0, + impressions: 0, + ctr: 0, + position: 0, +}; + +const SORT_FIELDS = { + clicks_delta: "clicks", + impressions_delta: "impressions", + ctr_delta: "ctr", + position_delta: "position", +}; + +function round(value, digits) { + if (typeof value !== "number" || !Number.isFinite(value)) { + return value; + } + const factor = 10 ** digits; + return Math.round(value * factor) / factor; +} + +function metricsOf(row = {}) { + return { + clicks: row.clicks || 0, + impressions: row.impressions || 0, + ctr: row.ctr || 0, + position: row.position || 0, + }; +} + +function roundMetrics(metrics) { + return { + clicks: metrics.clicks, + impressions: metrics.impressions, + ctr: round(metrics.ctr, 4), + position: round(metrics.position, 2), + }; +} + +/** + * `ctr` and `position` deltas are null when either side has no impressions: the zero + * placeholder for an absent key is not a real rank, so subtracting it would report a + * brand-new query at position 47 as a 47-place drop. Clicks and impressions are true + * zeros there and keep their numeric delta. + */ +function deltaOf(current, previous) { + const comparable = current.impressions !== 0 && previous.impressions !== 0; + return { + clicks: current.clicks - previous.clicks, + impressions: current.impressions - previous.impressions, + ctr: comparable + ? current.ctr - previous.ctr + : null, + position: comparable + ? current.position - previous.position + : null, + }; +} + +// null (not zero, not Infinity) when the previous period had nothing to grow from. +function pctOf(current, previous) { + if (!previous) { + return null; + } + return round((current - previous) / previous, 4); +} + +function pctChangeOf(current, previous) { + return { + clicks: pctOf(current.clicks, previous.clicks), + impressions: pctOf(current.impressions, previous.impressions), + }; +} + +/** + * Sums a period's rows. `ctr` is recomputed from the summed clicks/impressions and + * `position` is an impression-weighted average — neither may be a plain mean of the + * per-row values. + */ +export function summarizeRows(rows = []) { + let clicks = 0; + let impressions = 0; + let weightedPosition = 0; + + for (const row of rows) { + const metrics = metricsOf(row); + clicks += metrics.clicks; + impressions += metrics.impressions; + weightedPosition += metrics.position * metrics.impressions; + } + + return { + clicks, + impressions, + ctr: impressions + ? clicks / impressions + : 0, + position: impressions + ? weightedPosition / impressions + : 0, + }; +} + +function sortComparedRows(rows, sortBy) { + if (sortBy === "current_clicks") { + return rows.sort((a, b) => b.current.clicks - a.current.clicks); + } + // Absolute delta so the biggest gains AND the biggest losses surface first. A null + // delta has no magnitude to rank, so those rows go last rather than sorting as zero. + const field = SORT_FIELDS[sortBy] || "clicks"; + return rows.sort((a, b) => { + const aDelta = a.delta[field]; + const bDelta = b.delta[field]; + if (aDelta === null) { + return bDelta === null + ? 0 + : 1; + } + if (bDelta === null) { + return -1; + } + return Math.abs(bDelta) - Math.abs(aDelta); + }); +} + +/** + * Joins two periods of search-analytics rows on their `keys` and computes per-row and + * total deltas. Pure — no I/O — so the join, the sort and the totals are unit-testable. + * + * A key present in only one period gets `delta.ctr`/`delta.position` of null (there is no + * rank on the missing side to compare against), and those rows sort last under + * `ctr_delta`/`position_delta`. `delta.clicks`/`delta.impressions` stay numeric. + */ +export function buildComparison({ + currentRows = [], + previousRows = [], + sortBy = "clicks_delta", + rowLimit = 50, +}) { + const joined = new Map(); + + for (const row of currentRows) { + const keys = row.keys || []; + joined.set(JSON.stringify(keys), { + keys, + current: metricsOf(row), + previous: { + ...ZERO_METRICS, + }, + }); + } + + for (const row of previousRows) { + const keys = row.keys || []; + const id = JSON.stringify(keys); + const entry = joined.get(id); + if (entry) { + entry.previous = metricsOf(row); + } else { + joined.set(id, { + keys, + current: { + ...ZERO_METRICS, + }, + previous: metricsOf(row), + }); + } + } + + const all = [ + ...joined.values(), + ].map((entry) => ({ + keys: entry.keys, + current: entry.current, + previous: entry.previous, + delta: deltaOf(entry.current, entry.previous), + pct_change: pctChangeOf(entry.current, entry.previous), + })); + + sortComparedRows(all, sortBy); + + const rows = all + .slice(0, rowLimit) + .map((row) => ({ + keys: row.keys, + current: roundMetrics(row.current), + previous: roundMetrics(row.previous), + delta: roundMetrics(row.delta), + pct_change: row.pct_change, + })); + + const currentTotals = summarizeRows(currentRows); + const previousTotals = summarizeRows(previousRows); + + return { + totals: { + current: roundMetrics(currentTotals), + previous: roundMetrics(previousTotals), + delta: roundMetrics(deltaOf(currentTotals, previousTotals)), + pct_change: pctChangeOf(currentTotals, previousTotals), + }, + rows, + row_count: rows.length, + has_more: all.length > rows.length, + }; +} + +/** Formats a 0-1 fraction as a signed percentage for the run summary. */ +export function formatPctChange(value) { + if (value === null || value === undefined) { + return "n/a"; + } + const pct = round(value * 100, 1); + return pct >= 0 + ? `+${pct}%` + : `${pct}%`; +} diff --git a/components/google_search_console/common/filters.mjs b/components/google_search_console/common/filters.mjs new file mode 100644 index 0000000000000..64765a5f4af4d --- /dev/null +++ b/components/google_search_console/common/filters.mjs @@ -0,0 +1,74 @@ +import { trimIfString } from "./utils.mjs"; + +/** + * Builds the Search Console `dimensionFilterGroups` request field from either the + * single-filter shortcut or the advanced JSON input. Shared by + * `retrieve-site-performance-data` (whose shortcut value prop is the legacy + * `subdomainFilter`) and `compare-search-analytics` (whose shortcut value prop is + * `filterValue`), so the accepted input shapes stay identical across both tools. + * + * Accepted `advancedDimensionFilters` shapes (string or already-parsed): + * - bare filter array: [{ dimension, operator, expression }, ...] + * - raw API groups: [{ groupType: "and", filters: [...] }, ...] + * - a single object of either shape + * + * @returns {Array|undefined} the `dimensionFilterGroups` array, or undefined when no filter applies + */ +export function buildDimensionFilterGroups({ + app, + filterValue, + filterDimension, + filterOperator, + advancedDimensionFilters, +}) { + // Normalized so whitespace-only input counts as absent, leaving + // advancedDimensionFilters eligible instead of sending an empty-string filter. + const expression = trimIfString(filterValue); + + if (expression) { + return [ + { + groupType: "and", + filters: [ + { + dimension: filterDimension || "page", + operator: filterOperator || "contains", + expression, + }, + ], + }, + ]; + } + + const advanced = trimIfString(advancedDimensionFilters); + if (!advanced) { + return undefined; + } + + const parsed = app.parseIfJsonString(advanced, "Advanced Dimension Filters"); + const entries = Array.isArray(parsed) + ? parsed + : [ + parsed, + ]; + + if (!entries.length) { + return undefined; + } + + const [ + first, + ] = entries; + + // A bare filter array is ANDed into a single group; raw groups pass through. + if (first && typeof first === "object" && "dimension" in first) { + return [ + { + groupType: "and", + filters: entries, + }, + ]; + } + + return entries; +} diff --git a/components/google_search_console/common/methods.mjs b/components/google_search_console/common/methods.mjs index 21efd535c8d73..88db8e02b5626 100644 --- a/components/google_search_console/common/methods.mjs +++ b/components/google_search_console/common/methods.mjs @@ -10,21 +10,6 @@ export default { throw err; }, - /* ============================================================================================ - Determines whether an error originated from your own validation code or from the API request. - Useful for debugging and crafting more helpful error messages. -=============================================================================================== */ - - // ===================================================================== - checkWhoThrewError(error) { - return { - whoThrew: error?.response?.status - ? "API response" - : "Internal Code", - error, - }; - }, - /* ========================================================================================== Throws if the input is not a string or is a blank string (only whitespace, tabs, newlines, etc.). diff --git a/components/google_search_console/google_search_console.app.mjs b/components/google_search_console/google_search_console.app.mjs index 228dbf7fb1bf0..2d33c15f2edcb 100644 --- a/components/google_search_console/google_search_console.app.mjs +++ b/components/google_search_console/google_search_console.app.mjs @@ -1,5 +1,9 @@ import { axios } from "@pipedream/platform"; import methods from "./common/methods.mjs"; +import { trimIfString } from "./common/utils.mjs"; + +const SEARCH_CONSOLE_V3 = "https://searchconsole.googleapis.com/webmasters/v3"; +const URL_INSPECTION_V1 = "https://searchconsole.googleapis.com/v1"; export default { type: "app", @@ -7,12 +11,63 @@ export default { propDefinitions: { siteUrl: { type: "string", - label: "Site", - description: "Select a verified site from your Search Console", - async options({ prevContext }) { - const { nextPageToken } = prevContext || {}; - return this.listSiteOptions(nextPageToken); - }, + label: "Property (siteUrl)", + description: "Exact property identifier as returned by **List Sites** — `sc-domain:example.com` for a domain property, or a URL-prefix such as `https://www.example.com/` (trailing slash; scheme and subdomain must match exactly). Copy it verbatim; never construct it. For traffic questions prefer the domain property when one exists (it covers all subdomains and protocols).", + }, + sitemapUrl: { + type: "string", + label: "Sitemap URL", + description: "Full URL of the sitemap or sitemap index, e.g. `https://www.example.com/sitemap.xml`. It must live under the property given in `siteUrl` (for a domain property, any subdomain or scheme of that domain qualifies). Use **List Sitemaps** to see the exact paths Search Console already knows about.", + }, + searchType: { + type: "string", + label: "Search Type", + description: "Which Google surface to report on. `web` (default) is normal Google Search; `discover` is the Discover feed (has no `query` dimension); `googleNews` is the news.google.com surface, `news` is the News tab of Google Search. Sent to the API as the `type` field.", + optional: true, + options: [ + "web", + "image", + "video", + "news", + "discover", + "googleNews", + ], + default: "web", + }, + filterDimension: { + type: "string", + label: "Filter Dimension", + description: "Dimension the single-filter shortcut applies to. Filtering does not require grouping by the same dimension — you can filter by `page` while grouping by `query`. `page` expressions match the FULL URL (including scheme and host), not a path. Default `page`.", + optional: true, + options: [ + "country", + "device", + "page", + "query", + "searchAppearance", + ], + default: "page", + }, + filterOperator: { + type: "string", + label: "Filter Operator", + description: "How the filter value is compared. String comparison is case-insensitive. `includingRegex`/`excludingRegex` use RE2 syntax (no lookahead/lookbehind). Default `contains`.", + optional: true, + options: [ + "equals", + "notEquals", + "contains", + "notContains", + "includingRegex", + "excludingRegex", + ], + default: "contains", + }, + advancedDimensionFilters: { + type: "string", + label: "Advanced Dimension Filters", + description: "JSON for multi-condition filtering, used only when the single-filter shortcut is empty. Accepts either a bare array of filters, which is ANDed into one group — e.g. `[{\"dimension\":\"country\",\"operator\":\"equals\",\"expression\":\"usa\"},{\"dimension\":\"device\",\"operator\":\"equals\",\"expression\":\"MOBILE\"}]` — or the raw API `dimensionFilterGroups` array, e.g. `[{\"groupType\":\"and\",\"filters\":[{\"dimension\":\"page\",\"operator\":\"contains\",\"expression\":\"/blog/\"}]}]`. The API only supports `groupType: \"and\"`; there is no OR. Regex operators use RE2.", + optional: true, }, }, methods: { @@ -34,38 +89,66 @@ export default { async getSites(params = {}) { return this._makeRequest({ method: "GET", - url: "https://searchconsole.googleapis.com/webmasters/v3/sites", + url: `${SEARCH_CONSOLE_V3}/sites`, ...params, }); }, - async listSiteOptions(pageToken) { - const params = {}; - if (pageToken) { - params.pageToken = pageToken; - } - - const { - siteEntry = [], nextPageToken, - } = await this.getSites({ - params, + getUserInfo(opts = {}) { + return this._makeRequest({ + method: "GET", + url: "https://www.googleapis.com/oauth2/v3/userinfo", + ...opts, }); - - return { - options: siteEntry.map((site) => ({ - label: site.siteUrl, - value: site.siteUrl, - })), - context: { - nextPageToken, - }, - }; }, getSitePerformanceData({ url, ...opts }) { return this._makeRequest({ method: "POST", - url: `https://searchconsole.googleapis.com/webmasters/v3/sites/${encodeURIComponent(url)}/searchAnalytics/query`, + url: `${SEARCH_CONSOLE_V3}/sites/${encodeURIComponent(trimIfString(url))}/searchAnalytics/query`, + ...opts, + }); + }, + listSitemaps({ + siteUrl, ...opts + }) { + return this._makeRequest({ + method: "GET", + url: `${SEARCH_CONSOLE_V3}/sites/${encodeURIComponent(trimIfString(siteUrl))}/sitemaps`, + ...opts, + }); + }, + getSitemap({ + siteUrl, sitemapUrl, ...opts + }) { + return this._makeRequest({ + method: "GET", + url: `${SEARCH_CONSOLE_V3}/sites/${encodeURIComponent(trimIfString(siteUrl))}/sitemaps/${encodeURIComponent(trimIfString(sitemapUrl))}`, + ...opts, + }); + }, + submitSitemap({ + siteUrl, sitemapUrl, ...opts + }) { + return this._makeRequest({ + method: "PUT", + url: `${SEARCH_CONSOLE_V3}/sites/${encodeURIComponent(trimIfString(siteUrl))}/sitemaps/${encodeURIComponent(trimIfString(sitemapUrl))}`, + ...opts, + }); + }, + deleteSitemap({ + siteUrl, sitemapUrl, ...opts + }) { + return this._makeRequest({ + method: "DELETE", + url: `${SEARCH_CONSOLE_V3}/sites/${encodeURIComponent(trimIfString(siteUrl))}/sitemaps/${encodeURIComponent(trimIfString(sitemapUrl))}`, + ...opts, + }); + }, + inspectUrl(opts = {}) { + return this._makeRequest({ + method: "POST", + url: `${URL_INSPECTION_V1}/urlInspection/index:inspect`, ...opts, }); }, diff --git a/components/google_search_console/package.json b/components/google_search_console/package.json index b97afd6abaea6..ce594f104fa86 100644 --- a/components/google_search_console/package.json +++ b/components/google_search_console/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/google_search_console", - "version": "1.0.0", + "version": "1.1.0", "description": "Pipedream google_search_console Components", "main": "google_search_console.app.mjs", "keywords": [