diff --git a/deploy/cloud/stuck-users-report.sh b/deploy/cloud/stuck-users-report.sh index ee3005b48..fef15a90c 100755 --- a/deploy/cloud/stuck-users-report.sh +++ b/deploy/cloud/stuck-users-report.sh @@ -25,6 +25,10 @@ # 5. Etsy apps waiting for Etsy's approval: orgs whose app keys Etsy # refused as not active yet this week, approved this week (a successful # Etsy call after that error), still pending. +# 6. setup checks that fail ("Check and save" in the guided setup), per +# adapter: workspaces refused this week that have not got through since, +# with the workspaces and errors listed when the error may be the +# adapter's (400, 404, 5xx) rather than a wrong credential. # # Read-only: every query runs in a read-only transaction (PGOPTIONS below) # through `docker exec` into the Postgres container, as the operators do by @@ -448,6 +452,85 @@ from res; SQL ) || fail "Etsy pending query" +# ── 6. Setup checks that fail ──────────────────────────────────────────────── +# The guided setup's "Check and save" tries the values with a real call before +# anything works; a refusal is recorded as setup_verify_failed (adapter, kind, +# HTTP status, the provider's words). Such a refusal never becomes a tool call, +# so sections 1 and 2 cannot see it: on 2026-11-08 a wrong login body in the +# Éxito adapter blocked a customer for a day without a trace here. Etsy has its +# own section above. A workspace is through when, after its first failed check +# on that adapter, it completed the setup verified or made a successful call +# through a connector of that adapter. +# Kinds that point at the customer's side (auth_failed, rate_limited, +# invalid_input, unreachable: a wrong key, a typo in the address) are counted; +# any other kind (bad_request, not_found, upstream_error, error…) may be the +# adapter's fault, so those workspaces are listed with their error. +CRED_KINDS="'auth_failed', 'rate_limited', 'invalid_input', 'unreachable'" +setup=$(q <>'adapterSlug' as slug, + coalesce(nullif(pe.metadata->>'kind', ''), 'error') as kind, + coalesce(pe.metadata->>'status', '') as st, + pe.metadata->>'error' as err, pe.created_at + from product_events pe + where pe.event = 'setup_verify_failed' and pe.created_at >= $WIN + and pe.organization_id is not null + and coalesce(pe.metadata->>'adapterSlug', '') not in ('', 'etsy') +), +po as ( + select org_id, slug, min(created_at) as first_at, max(created_at) as last_at, count(*) as n, + bool_or(kind not in ($CRED_KINDS)) as suspect + from f group by 1, 2 +), +st as ( + select po.*, + exists (select 1 from product_events e + where e.organization_id = po.org_id and e.event = 'setup_completed' + and e.metadata->>'adapterSlug' = po.slug and e.created_at > po.first_at + and coalesce(e.metadata->>'verified', 'true') <> 'false') + or exists (select 1 from tool_invocations ti join connectors c on c.id = ti.connector_id + where ti.organization_id = po.org_id and c.config->>'adapterSlug' = po.slug + and ti.status = 'SUCCESS' and ti.created_at > po.first_at) as through + from po +), +kinds as ( + select slug, string_agg(k || ' ×' || n, ', ' order by n desc, k) as kinds from ( + select slug, kind || case when st <> '' then ' ' || st else '' end as k, count(*) as n + from f group by 1, 2 + ) x group by 1 +), +errs as ( + select slug, string_agg(n || '× ' || e, E'\x1e' order by n desc, e) as errs from ( + select slug, e, n, row_number() over (partition by slug order by n desc, e) as rn from ( + select f.slug, $(clean f.err) as e, count(*) as n + from f join st s on s.org_id = f.org_id and s.slug = f.slug + where not s.through and f.kind not in ($CRED_KINDS) group by 1, 2 + ) x + ) y where rn <= 3 group by slug +), +people as ( + select s.org_id, s.slug, + coalesce((select string_agg(u.email, ', ' order by u.created_at) from users u + where u.organization_id = s.org_id), '(no member)') as emails + from st s where not s.through and s.suspect +), +orgl as ( + select s.slug, string_agg($(oneline p.emails) || ' — ' || s.n || ' failed, last ' || $(fmt_ts s.last_at), + E'\x1e' order by s.last_at desc) as orgs + from st s join people p on p.org_id = s.org_id and p.slug = s.slug group by 1 +), +agg as ( + select slug, count(*) as orgs, count(*) filter (where not through) as stuck, + count(*) filter (where not through and suspect) as stuck_suspect, sum(n) as fails + from st group by 1 +) +select a.slug, a.orgs, a.stuck, a.stuck_suspect, a.fails, coalesce(k.kinds, ''), + coalesce(e.errs, ''), coalesce(o.orgs, '') +from agg a left join kinds k using (slug) left join errs e using (slug) left join orgl o using (slug) +order by (a.stuck_suspect > 0) desc, a.stuck_suspect desc, a.stuck desc, a.slug; +SQL +) || fail "setup checks query" + # ── Render ─────────────────────────────────────────────────────────────────── NL=$'\n' CAUSES="credentials config quota unreachable other" @@ -502,6 +585,30 @@ while IFS="$US" read -r key label calls orgs ok calls30 ok30 lastok errs; do done <<< "$conns" n_conn=$((n_dead + n_weak)) +s6=""; n_setup_orgs=0; n_setup_stuck=0; n_setup_suspect=0; n_setup_adapters=0 +while IFS="$US" read -r slug orgs stuck suspect fails kinds errs orgl; do + [ -n "$slug" ] || continue + n_setup_orgs=$((n_setup_orgs + orgs)); n_setup_stuck=$((n_setup_stuck + stuck)); n_setup_suspect=$((n_setup_suspect + suspect)) + # Adapters whose workspaces all got through count in the totals only. + [ "$stuck" -gt 0 ] || continue + n_setup_adapters=$((n_setup_adapters + 1)) + [ "$n_setup_adapters" -le "$LIST_MAX" ] || continue + s6+=" $slug: $orgs workspace(s) failed the check, $stuck not through since; $fails failed checks: $kinds$NL" + if [ -n "$errs" ]; then + s6+=" errors that may be the adapter's (workspaces not through):$NL" + while IFS= read -r line; do s6+=" $line$NL"; done <<< "$(printf '%s' "$errs" | tr "$RS" '\n')" + fi + if [ -n "$orgl" ]; then + i=0; more=0 + while IFS= read -r line; do + i=$((i + 1)); if [ "$i" -le 5 ]; then s6+=" $line$NL"; else more=$((more + 1)); fi + done <<< "$(printf '%s' "$orgl" | tr "$RS" '\n')" + [ "$more" -gt 0 ] && s6+=" … and $more more$NL" + fi + s6+="$NL" +done <<< "$setup" +[ "$n_setup_adapters" -gt "$LIST_MAX" ] && s6+=" … $((n_setup_adapters - LIST_MAX)) more adapter(s), REPORT_LIST_MAX=$LIST_MAX.$NL$NL" + w_total=0; m_total=0 w1=0; w2=0; w3=0; w4=0; m1=0; m2=0; m3=0; m4=0 list1=""; list2="" @@ -520,6 +627,7 @@ summary="$(plural "$n_new" "new stuck org") ($n_still still stuck)" summary+=", $(plural "$n_dead" "connector") failing for everyone" [ "$n_weak" -gt 0 ] && summary+=" (+$n_weak under ${MAX_RATE} %)" summary+=", $w1 of $(plural "$w_total" "signup") without a connector" +[ "$n_setup_stuck" -gt 0 ] && summary+=", $(plural "$n_setup_stuck" "workspace") stuck at a setup check ($n_setup_suspect possibly the adapter)" out="AnythingMCP Cloud — weekly stuck-users report$NL" out+="Window: $WIN_FROM to $WIN_TO ($REPORT_TZ), last $WINDOW_DAYS days; context: last 30 days.$NL$NL" @@ -588,6 +696,13 @@ out+=" ${etsy_wait:-0} still pending (error in the last 30 days, no successful out+=" ${etsy_mailed:-0} approval e-mail(s) sent this week (etsy_app_approved_notified)$NL" out+=" A shared secret that does not belong to the keystring gets the same 403, so \"pending\" includes it.$NL$NL" +out+="6. SETUP CHECKS THAT FAIL (\"Check and save\" in the guided setup, this week; Etsy is in section 5)$NL" +out+=" These never become tool calls, so sections 1 and 2 do not see them. Through = a verified setup$NL" +out+=" or a successful call on that adapter after the first failed check. Workspaces are listed only$NL" +out+=" when an error may be the adapter's (400, 404, 5xx, other); wrong credentials or addresses are counted.$NL" +out+=" $n_setup_stuck of $n_setup_orgs workspace(s) with a failed check are not through; $n_setup_suspect of them with an error that may be ours.$NL$NL" +[ -n "$s6" ] && out+="$s6" || out+=" None.$NL$NL" + out+="--$NL" out+="Generated by deploy/cloud/stuck-users-report.sh on $(hostname) for ${DOMAIN}.$NL" out+="Contains customer e-mail addresses: keep it inside the team. Error texts are cut to 150$NL" diff --git a/deploy/cloud/stuck-users-report.test.sh b/deploy/cloud/stuck-users-report.test.sh index d6261881d..53d5d4ec3 100644 --- a/deploy/cloud/stuck-users-report.test.sh +++ b/deploy/cloud/stuck-users-report.test.sh @@ -9,7 +9,7 @@ # KEEP_MAIL=1 bash deploy/cloud/stuck-users-report.test.sh # print the mail # # Scenarios: -# 1. --print renders all five sections, marks every org NEW on a first run, +# 1. --print renders all six sections, marks every org NEW on a first run, # groups stuck orgs by cause and lists only "other" and company-address # orgs, counts the funnel of this week's and last week's cohorts and the # Etsy apps waiting for approval, scrubs error texts, and touches neither @@ -179,7 +179,13 @@ from t0, (values ('etsy_app_approved_notified', 'org_etsy_ok', '{"connectorId":"c_k_etsy"}', interval '1 day'), -- not the pending answer: another status, another adapter ('setup_verify_failed', 'org_happy', '{"adapterSlug":"etsy","kind":"invalid_input","status":400}', interval '1 day'), - ('setup_verify_failed', 'org_happy', '{"adapterSlug":"shopify","kind":"auth_failed","status":403}', interval '1 day') + ('setup_verify_failed', 'org_happy', '{"adapterSlug":"shopify","kind":"auth_failed","status":403}', interval '1 day'), + -- setup checks (section 6): an adapter-side 500 not through, a 400 fixed by a + -- verified setup after it, a 404 on a connector that has called since + ('setup_verify_failed', 'org_stuck_e', '{"adapterSlug":"odoo","kind":"upstream_error","status":500,"error":"The target API returned a server error (500). (Invalid field customer_rank on res.partner, see https://odoo.example/web?db=SEEKRIT-DB)"}', interval '2 days'), + ('setup_verify_failed', 'org_stuck_e', '{"adapterSlug":"odoo","kind":"upstream_error","status":500,"error":"The target API returned a server error (500). (Invalid field customer_rank on res.partner, see https://odoo.example/web?db=SEEKRIT-DB)"}', interval '1 day'), + ('setup_verify_failed', 'org_stuck_a', '{"adapterSlug":"reddit","kind":"bad_request","status":400,"error":"Bad body"}', interval '4 days'), + ('setup_verify_failed', 'org_happy', '{"adapterSlug":"github","kind":"not_found","status":404,"error":"Not Found"}', interval '6 days') ) v(event, org, meta, age); SQL @@ -191,7 +197,18 @@ echo "1. --print on a first run" printf 'DOMAIN=cloud.example.test\n' > "$ENVF" run --print > "$WORK/r1.txt" 2>/dev/null R="$WORK/r1.txt" -has "$R" "SUMMARY: 6 new stuck orgs (0 still stuck), 1 connector failing for everyone (+1 under 20 %), 1 of 3 signups without a connector." "summary line" +has "$R" "SUMMARY: 6 new stuck orgs (0 still stuck), 1 connector failing for everyone (+1 under 20 %), 1 of 3 signups without a connector, 2 workspaces stuck at a setup check (1 possibly the adapter)." "summary line" +# Section 6: setup checks. odoo (500, judy) not through and listed; reddit +# fixed by the verified setup after it; github fixed by a successful call; shopify 403 +# counted, not listed; etsy left to section 5. +has "$R" "6. SETUP CHECKS THAT FAIL" "setup checks section" +has "$R" "2 of 4 workspace(s) with a failed check are not through; 1 of them with an error that may be ours." "setup checks totals" +has "$R" "odoo: 1 workspace(s) failed the check, 1 not through since; 2 failed checks: upstream_error 500 ×2" "adapter-side failure per adapter" +has "$R" "2× The target API returned a server error (500). (Invalid field customer_rank on res.partner, see https://odoo.example/web?…" "provider words kept, query string scrubbed" +has "$R" "judy@gmail.com — 2 failed, last" "workspace listed even on free mail when the error may be ours" +has "$R" "shopify: 1 workspace(s) failed the check, 1 not through since; 1 failed checks: auth_failed 403 ×1" "credential failure counted" +if sed -n '/^6\. /,/^--$/p' "$R" | grep -q "carol@example.net"; then bad "credential failure not listed by workspace"; else ok "credential failure not listed by workspace"; fi +if sed -n '/^6\. /,/^--$/p' "$R" | grep -qE "^ (reddit|github|etsy):"; then bad "fixed setups and Etsy not in section 6"; else ok "fixed setups and Etsy not in section 6"; fi has "$R" "[NEW] alice@example.com [opt-out]" "stuck org A, NEW, opt-out flagged" has "$R" "[NEW] bob@example.org" "stuck org B, NEW" has "$R" "Active this week: 6 (6 new, 0 still stuck). Active in the last 30 days: 7. All time: 7." "30-day and all-time context" diff --git a/packages/backend/src/audit/product-event.service.ts b/packages/backend/src/audit/product-event.service.ts index fb0baf976..a5a33904e 100644 --- a/packages/backend/src/audit/product-event.service.ts +++ b/packages/backend/src/audit/product-event.service.ts @@ -595,6 +595,12 @@ export function sanitizeSearchQuery(value: unknown): string | null { * `http://` of `http://http://…` (CodeQL js/polynomial-redos). */ const SCRUB_INPUT_LIMIT = 2000; +/** + * Long enough to keep the provider's own words after our hint: a 500 from + * Odoo starts with the classifier's sentence, and its "Invalid field 'x'" + * came after the old 160-character cut. + */ +const PROVIDER_MESSAGE_MAX = 400; export function scrubProviderMessage(message: unknown, secrets: unknown[] = []): string | null { if (typeof message !== 'string' || !message.trim()) return null; @@ -611,7 +617,7 @@ export function scrubProviderMessage(message: unknown, secrets: unknown[] = []): }); out = out.replace(/[A-Za-z0-9_\-.+/=]{32,}/g, '***'); out = out.replace(/\s+/g, ' ').trim(); - return out.length > 160 ? `${out.slice(0, 159)}…` : out; + return out.length > PROVIDER_MESSAGE_MAX ? `${out.slice(0, PROVIDER_MESSAGE_MAX - 1)}…` : out; } /** The attribution pair, re-sanitized here so no caller can store anything else. */ diff --git a/packages/backend/src/audit/product-event.spec.ts b/packages/backend/src/audit/product-event.spec.ts index 78aacc0e8..a8c516473 100644 --- a/packages/backend/src/audit/product-event.spec.ts +++ b/packages/backend/src/audit/product-event.spec.ts @@ -155,10 +155,16 @@ describe('scrubProviderMessage', () => { expect(scrubProviderMessage('GET https://x.example/api?appid=abc failed', [])).toBe('GET https://x.example/api failed'); expect(scrubProviderMessage('bad key s3cr3t-value', ['s3cr3t-value'])).toBe('bad key ***'); expect(scrubProviderMessage(`token ${'a1'.repeat(20)} expired`)).toBe('token *** expired'); - expect(scrubProviderMessage('x'.repeat(30) + ' ' + 'word '.repeat(60))!.length).toBe(160); + expect(scrubProviderMessage('x'.repeat(30) + ' ' + 'word '.repeat(100))!.length).toBe(400); expect(scrubProviderMessage(' ')).toBeNull(); }); + it("keeps the provider's words that follow our hint", () => { + const hint = 'The target API returned a server error (500). This is on their side, not your configuration.'; + const odoo = `${hint} (Request failed with status code 500: {"name":"builtins.ValueError","message":"Invalid field 'customer_rank' on 'res.partner'"})`; + expect(scrubProviderMessage(odoo)).toContain("Invalid field 'customer_rank' on 'res.partner'"); + }); + it('cuts a URL at its first ? or #, wherever it sits in the text', () => { expect(scrubProviderMessage('see "https://a.example/x#frag" and http://b.example/y?k=v&z=1 now')).toBe( 'see "https://a.example/x" and http://b.example/y now', @@ -177,7 +183,7 @@ describe('scrubProviderMessage', () => { const started = Date.now(); const out = scrubProviderMessage(input); expect(Date.now() - started).toBeLessThan(200); - expect(out!.length).toBeLessThanOrEqual(160); + expect(out!.length).toBeLessThanOrEqual(400); } }); });