Report failing setup checks in the weekly stuck-users mail

deploy/cloud/stuck-users-report.sh

@@ -25,6 +25,10 @@
 #   5. Etsy apps waiting for Etsy's approval: orgs whose app keys Etsy
 #      refused as not active yet this week, approved this week (a successful
 #      Etsy call after that error), still pending.
+#   6. setup checks that fail ("Check and save" in the guided setup), per
+#      adapter: workspaces refused this week that have not got through since,
+#      with the workspaces and errors listed when the error may be the
+#      adapter's (400, 404, 5xx) rather than a wrong credential.
 #
 # Read-only: every query runs in a read-only transaction (PGOPTIONS below)
 # through `docker exec` into the Postgres container, as the operators do by
@@ -448,6 +452,85 @@ from res;
 SQL
 ) || fail "Etsy pending query"
 
+# ── 6. Setup checks that fail ────────────────────────────────────────────────
+# The guided setup's "Check and save" tries the values with a real call before
+# anything works; a refusal is recorded as setup_verify_failed (adapter, kind,
+# HTTP status, the provider's words). Such a refusal never becomes a tool call,
+# so sections 1 and 2 cannot see it: on 2026-11-08 a wrong login body in the
+# Éxito adapter blocked a customer for a day without a trace here. Etsy has its
+# own section above. A workspace is through when, after its first failed check
+# on that adapter, it completed the setup verified or made a successful call
+# through a connector of that adapter.
+# Kinds that point at the customer's side (auth_failed, rate_limited,
+# invalid_input, unreachable: a wrong key, a typo in the address) are counted;
+# any other kind (bad_request, not_found, upstream_error, error…) may be the
+# adapter's fault, so those workspaces are listed with their error.
+CRED_KINDS="'auth_failed', 'rate_limited', 'invalid_input', 'unreachable'"
+setup=$(q <<SQL
+with f as (
+  select pe.organization_id as org_id, pe.metadata->>'adapterSlug' as slug,
+         coalesce(nullif(pe.metadata->>'kind', ''), 'error') as kind,
+         coalesce(pe.metadata->>'status', '') as st,
+         pe.metadata->>'error' as err, pe.created_at
+  from product_events pe
+  where pe.event = 'setup_verify_failed' and pe.created_at >= $WIN
+    and pe.organization_id is not null
+    and coalesce(pe.metadata->>'adapterSlug', '') not in ('', 'etsy')
+),
+po as (
+  select org_id, slug, min(created_at) as first_at, max(created_at) as last_at, count(*) as n,
+         bool_or(kind not in ($CRED_KINDS)) as suspect
+  from f group by 1, 2
+),
+st as (
+  select po.*,
+    exists (select 1 from product_events e
+            where e.organization_id = po.org_id and e.event = 'setup_completed'
+              and e.metadata->>'adapterSlug' = po.slug and e.created_at > po.first_at
+              and coalesce(e.metadata->>'verified', 'true') <> 'false')
+    or exists (select 1 from tool_invocations ti join connectors c on c.id = ti.connector_id
+               where ti.organization_id = po.org_id and c.config->>'adapterSlug' = po.slug
+                 and ti.status = 'SUCCESS' and ti.created_at > po.first_at) as through
+  from po
+),
+kinds as (
+  select slug, string_agg(k || ' ×' || n, ', ' order by n desc, k) as kinds from (
+    select slug, kind || case when st <> '' then ' ' || st else '' end as k, count(*) as n
+    from f group by 1, 2
+  ) x group by 1
+),
+errs as (
+  select slug, string_agg(n || '× ' || e, E'\x1e' order by n desc, e) as errs from (
+    select slug, e, n, row_number() over (partition by slug order by n desc, e) as rn from (
+      select f.slug, $(clean f.err) as e, count(*) as n
+      from f join st s on s.org_id = f.org_id and s.slug = f.slug
+      where not s.through and f.kind not in ($CRED_KINDS) group by 1, 2
+    ) x
+  ) y where rn <= 3 group by slug
+),
+people as (
+  select s.org_id, s.slug,
+         coalesce((select string_agg(u.email, ', ' order by u.created_at) from users u
+                   where u.organization_id = s.org_id), '(no member)') as emails
+  from st s where not s.through and s.suspect
+),
+orgl as (
+  select s.slug, string_agg($(oneline p.emails) || ' — ' || s.n || ' failed, last ' || $(fmt_ts s.last_at),
+                            E'\x1e' order by s.last_at desc) as orgs
+  from st s join people p on p.org_id = s.org_id and p.slug = s.slug group by 1
+),
+agg as (
+  select slug, count(*) as orgs, count(*) filter (where not through) as stuck,
+         count(*) filter (where not through and suspect) as stuck_suspect, sum(n) as fails
+  from st group by 1
+)
+select a.slug, a.orgs, a.stuck, a.stuck_suspect, a.fails, coalesce(k.kinds, ''),
+       coalesce(e.errs, ''), coalesce(o.orgs, '')
+from agg a left join kinds k using (slug) left join errs e using (slug) left join orgl o using (slug)
+order by (a.stuck_suspect > 0) desc, a.stuck_suspect desc, a.stuck desc, a.slug;
+SQL
+) || fail "setup checks query"
+
 # ── Render ───────────────────────────────────────────────────────────────────
 NL=$'\n'
 CAUSES="credentials config quota unreachable other"
@@ -502,6 +585,30 @@ while IFS="$US" read -r key label calls orgs ok calls30 ok30 lastok errs; do
 done <<< "$conns"
 n_conn=$((n_dead + n_weak))
 
+s6=""; n_setup_orgs=0; n_setup_stuck=0; n_setup_suspect=0; n_setup_adapters=0
+while IFS="$US" read -r slug orgs stuck suspect fails kinds errs orgl; do
+  [ -n "$slug" ] || continue
+  n_setup_orgs=$((n_setup_orgs + orgs)); n_setup_stuck=$((n_setup_stuck + stuck)); n_setup_suspect=$((n_setup_suspect + suspect))
+  # Adapters whose workspaces all got through count in the totals only.
+  [ "$stuck" -gt 0 ] || continue
+  n_setup_adapters=$((n_setup_adapters + 1))
+  [ "$n_setup_adapters" -le "$LIST_MAX" ] || continue
+  s6+="  $slug: $orgs workspace(s) failed the check, $stuck not through since; $fails failed checks: $kinds$NL"
+  if [ -n "$errs" ]; then
+    s6+="    errors that may be the adapter's (workspaces not through):$NL"
+    while IFS= read -r line; do s6+="      $line$NL"; done <<< "$(printf '%s' "$errs" | tr "$RS" '\n')"
+  fi
+  if [ -n "$orgl" ]; then
+    i=0; more=0
+    while IFS= read -r line; do
+      i=$((i + 1)); if [ "$i" -le 5 ]; then s6+="      $line$NL"; else more=$((more + 1)); fi
+    done <<< "$(printf '%s' "$orgl" | tr "$RS" '\n')"
+    [ "$more" -gt 0 ] && s6+="      … and $more more$NL"
+  fi
+  s6+="$NL"
+done <<< "$setup"
+[ "$n_setup_adapters" -gt "$LIST_MAX" ] && s6+="  … $((n_setup_adapters - LIST_MAX)) more adapter(s), REPORT_LIST_MAX=$LIST_MAX.$NL$NL"
+
 w_total=0; m_total=0
 w1=0; w2=0; w3=0; w4=0; m1=0; m2=0; m3=0; m4=0
 list1=""; list2=""
@@ -520,6 +627,7 @@ summary="$(plural "$n_new" "new stuck org") ($n_still still stuck)"
 summary+=", $(plural "$n_dead" "connector") failing for everyone"
 [ "$n_weak" -gt 0 ] && summary+=" (+$n_weak under ${MAX_RATE} %)"
 summary+=", $w1 of $(plural "$w_total" "signup") without a connector"
+[ "$n_setup_stuck" -gt 0 ] && summary+=", $(plural "$n_setup_stuck" "workspace") stuck at a setup check ($n_setup_suspect possibly the adapter)"
 
 out="AnythingMCP Cloud — weekly stuck-users report$NL"
 out+="Window: $WIN_FROM to $WIN_TO ($REPORT_TZ), last $WINDOW_DAYS days; context: last 30 days.$NL$NL"
@@ -588,6 +696,13 @@ out+="  ${etsy_wait:-0} still pending (error in the last 30 days, no successful
 out+="  ${etsy_mailed:-0} approval e-mail(s) sent this week (etsy_app_approved_notified)$NL"
 out+="  A shared secret that does not belong to the keystring gets the same 403, so \"pending\" includes it.$NL$NL"
 
+out+="6. SETUP CHECKS THAT FAIL (\"Check and save\" in the guided setup, this week; Etsy is in section 5)$NL"
+out+="   These never become tool calls, so sections 1 and 2 do not see them. Through = a verified setup$NL"
+out+="   or a successful call on that adapter after the first failed check. Workspaces are listed only$NL"
+out+="   when an error may be the adapter's (400, 404, 5xx, other); wrong credentials or addresses are counted.$NL"
+out+="   $n_setup_stuck of $n_setup_orgs workspace(s) with a failed check are not through; $n_setup_suspect of them with an error that may be ours.$NL$NL"
+[ -n "$s6" ] && out+="$s6" || out+="  None.$NL$NL"
+
 out+="--$NL"
 out+="Generated by deploy/cloud/stuck-users-report.sh on $(hostname) for ${DOMAIN}.$NL"
 out+="Contains customer e-mail addresses: keep it inside the team. Error texts are cut to 150$NL"

deploy/cloud/stuck-users-report.test.sh

@@ -9,7 +9,7 @@
 #   KEEP_MAIL=1 bash deploy/cloud/stuck-users-report.test.sh   # print the mail
 #
 # Scenarios:
-#   1. --print renders all five sections, marks every org NEW on a first run,
+#   1. --print renders all six sections, marks every org NEW on a first run,
 #      groups stuck orgs by cause and lists only "other" and company-address
 #      orgs, counts the funnel of this week's and last week's cohorts and the
 #      Etsy apps waiting for approval, scrubs error texts, and touches neither
@@ -179,7 +179,13 @@ from t0, (values
   ('etsy_app_approved_notified', 'org_etsy_ok', '{"connectorId":"c_k_etsy"}', interval '1 day'),
   -- not the pending answer: another status, another adapter
   ('setup_verify_failed', 'org_happy',   '{"adapterSlug":"etsy","kind":"invalid_input","status":400}', interval '1 day'),
-  ('setup_verify_failed', 'org_happy',   '{"adapterSlug":"shopify","kind":"auth_failed","status":403}', interval '1 day')
+  ('setup_verify_failed', 'org_happy',   '{"adapterSlug":"shopify","kind":"auth_failed","status":403}', interval '1 day'),
+  -- setup checks (section 6): an adapter-side 500 not through, a 400 fixed by a
+  -- verified setup after it, a 404 on a connector that has called since
+  ('setup_verify_failed', 'org_stuck_e', '{"adapterSlug":"odoo","kind":"upstream_error","status":500,"error":"The target API returned a server error (500). (Invalid field customer_rank on res.partner, see https://odoo.example/web?db=SEEKRIT-DB)"}', interval '2 days'),
+  ('setup_verify_failed', 'org_stuck_e', '{"adapterSlug":"odoo","kind":"upstream_error","status":500,"error":"The target API returned a server error (500). (Invalid field customer_rank on res.partner, see https://odoo.example/web?db=SEEKRIT-DB)"}', interval '1 day'),
+  ('setup_verify_failed', 'org_stuck_a', '{"adapterSlug":"reddit","kind":"bad_request","status":400,"error":"Bad body"}', interval '4 days'),
+  ('setup_verify_failed', 'org_happy',   '{"adapterSlug":"github","kind":"not_found","status":404,"error":"Not Found"}', interval '6 days')
 ) v(event, org, meta, age);
 SQL
 
@@ -191,7 +197,18 @@ echo "1. --print on a first run"
 printf 'DOMAIN=cloud.example.test\n' > "$ENVF"
 run --print > "$WORK/r1.txt" 2>/dev/null
 R="$WORK/r1.txt"
-has   "$R" "SUMMARY: 6 new stuck orgs (0 still stuck), 1 connector failing for everyone (+1 under 20 %), 1 of 3 signups without a connector." "summary line"
+has   "$R" "SUMMARY: 6 new stuck orgs (0 still stuck), 1 connector failing for everyone (+1 under 20 %), 1 of 3 signups without a connector, 2 workspaces stuck at a setup check (1 possibly the adapter)." "summary line"
+# Section 6: setup checks. odoo (500, judy) not through and listed; reddit
+# fixed by the verified setup after it; github fixed by a successful call; shopify 403
+# counted, not listed; etsy left to section 5.
+has   "$R" "6. SETUP CHECKS THAT FAIL" "setup checks section"
+has   "$R" "2 of 4 workspace(s) with a failed check are not through; 1 of them with an error that may be ours." "setup checks totals"
+has   "$R" "odoo: 1 workspace(s) failed the check, 1 not through since; 2 failed checks: upstream_error 500 ×2" "adapter-side failure per adapter"
+has   "$R" "2× The target API returned a server error (500). (Invalid field customer_rank on res.partner, see https://odoo.example/web?…" "provider words kept, query string scrubbed"
+has   "$R" "judy@gmail.com — 2 failed, last" "workspace listed even on free mail when the error may be ours"
+has   "$R" "shopify: 1 workspace(s) failed the check, 1 not through since; 1 failed checks: auth_failed 403 ×1" "credential failure counted"
+if sed -n '/^6\. /,/^--$/p' "$R" | grep -q "carol@example.net"; then bad "credential failure not listed by workspace"; else ok "credential failure not listed by workspace"; fi
+if sed -n '/^6\. /,/^--$/p' "$R" | grep -qE "^  (reddit|github|etsy):"; then bad "fixed setups and Etsy not in section 6"; else ok "fixed setups and Etsy not in section 6"; fi
 has   "$R" "[NEW] alice@example.com [opt-out]" "stuck org A, NEW, opt-out flagged"
 has   "$R" "[NEW] bob@example.org" "stuck org B, NEW"
 has   "$R" "Active this week: 6 (6 new, 0 still stuck). Active in the last 30 days: 7. All time: 7." "30-day and all-time context"

packages/backend/src/audit/product-event.service.ts

@@ -595,6 +595,12 @@ export function sanitizeSearchQuery(value: unknown): string | null {
  * `http://` of `http://http://…` (CodeQL js/polynomial-redos).
  */
 const SCRUB_INPUT_LIMIT = 2000;
+/**
+ * Long enough to keep the provider's own words after our hint: a 500 from
+ * Odoo starts with the classifier's sentence, and its "Invalid field 'x'"
+ * came after the old 160-character cut.
+ */
+const PROVIDER_MESSAGE_MAX = 400;
 
 export function scrubProviderMessage(message: unknown, secrets: unknown[] = []): string | null {
   if (typeof message !== 'string' || !message.trim()) return null;
@@ -611,7 +617,7 @@ export function scrubProviderMessage(message: unknown, secrets: unknown[] = []):
   });
   out = out.replace(/[A-Za-z0-9_\-.+/=]{32,}/g, '***');
   out = out.replace(/\s+/g, ' ').trim();
-  return out.length > 160 ? `${out.slice(0, 159)}…` : out;
+  return out.length > PROVIDER_MESSAGE_MAX ? `${out.slice(0, PROVIDER_MESSAGE_MAX - 1)}…` : out;
 }
 
 /** The attribution pair, re-sanitized here so no caller can store anything else. */

packages/backend/src/audit/product-event.spec.ts

@@ -155,10 +155,16 @@ describe('scrubProviderMessage', () => {
     expect(scrubProviderMessage('GET https://x.example/api?appid=abc failed', [])).toBe('GET https://x.example/api failed');
     expect(scrubProviderMessage('bad key s3cr3t-value', ['s3cr3t-value'])).toBe('bad key ***');
     expect(scrubProviderMessage(`token ${'a1'.repeat(20)} expired`)).toBe('token *** expired');
-    expect(scrubProviderMessage('x'.repeat(30) + ' ' + 'word '.repeat(60))!.length).toBe(160);
+    expect(scrubProviderMessage('x'.repeat(30) + ' ' + 'word '.repeat(100))!.length).toBe(400);
     expect(scrubProviderMessage('   ')).toBeNull();
   });
 
+  it("keeps the provider's words that follow our hint", () => {
+    const hint = 'The target API returned a server error (500). This is on their side, not your configuration.';
+    const odoo = `${hint} (Request failed with status code 500: {"name":"builtins.ValueError","message":"Invalid field 'customer_rank' on 'res.partner'"})`;
+    expect(scrubProviderMessage(odoo)).toContain("Invalid field 'customer_rank' on 'res.partner'");
+  });
+
   it('cuts a URL at its first ? or #, wherever it sits in the text', () => {
     expect(scrubProviderMessage('see "https://a.example/x#frag" and http://b.example/y?k=v&z=1 now')).toBe(
       'see "https://a.example/x" and http://b.example/y now',
@@ -177,7 +183,7 @@ describe('scrubProviderMessage', () => {
       const started = Date.now();
       const out = scrubProviderMessage(input);
       expect(Date.now() - started).toBeLessThan(200);
-      expect(out!.length).toBeLessThanOrEqual(160);
+      expect(out!.length).toBeLessThanOrEqual(400);
     }
   });
 });