From 282a33ea3dd67b7a55c52ce7f1df3ac08f75228e Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:42:42 -0700 Subject: [PATCH 01/34] Support Copilot Harness launches and Autopilot defaults --- docs/copilot-create-project.md | 19 +++- evals/agent-assets.lock.json | 16 +-- package-lock.json | 18 +-- package.json | 4 +- .../agents/azure-debug-generate.agent.md | 2 +- resources/agents/azure-debug-plan.agent.md | 2 +- resources/agents/azure-deploy.agent.md | 2 +- .../agents/azure-project-integrate.agent.md | 2 +- resources/agents/azure-project-plan.agent.md | 2 +- .../agents/azure-project-scaffold.agent.md | 2 +- .../copilotOnRails/openChatWithAgent.ts | 106 ++++++++++++------ src/extension.ts | 4 +- src/utils/copilotOnRails/modelSelection.ts | 36 ++++++ .../copilotOnRails/extension/autopilot.ts | 61 +++++++++- .../extension/harnessSettings.ts | 13 +-- test/copilotOnRails/autopilot.test.ts | 24 ++++ test/copilotOnRails/modelSelection.test.ts | 52 +++++++++ test/copilotOnRails/openChatWithAgent.test.ts | 32 ++++++ 18 files changed, 327 insertions(+), 70 deletions(-) create mode 100644 test/copilotOnRails/autopilot.test.ts create mode 100644 test/copilotOnRails/modelSelection.test.ts create mode 100644 test/copilotOnRails/openChatWithAgent.test.ts diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index b71cc1df8..772c6f909 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -134,9 +134,18 @@ session** running the next agent. Between hand‑offs, agents open **webviews** ## Prerequisites - **VS Code** with **GitHub Copilot** enabled and signed in. +- The **Copilot** chat session target. Copilot on Rails enables the Copilot Harness for the workspace + and waits for custom-agent registration to settle. After creating a fresh phase chat, it primes the + custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids + stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. - A Copilot plan with access to at least one supported model. The model picker lists the Opus, Sonnet, GPT Sol, and GPT Terra models currently available through GitHub Copilot, so newly available versions appear without an extension update. The lowest-version available Opus model is selected by default. + When the Copilot Harness is active, the extension maps that selection only to the matching + `agent-host-copilotcli` model. It does not pass Local or legacy `copilotcli` identifiers, which + VS Code rejects as foreign to the Agent Host session. The extension constructs the Agent Host + selector from the selected Local model ID because the language model extension API does not list + the Agent Host copies that the chat command can resolve. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without @@ -401,6 +410,9 @@ includes an `**Execution Mode**: auto` metadata row. In autopilot, agents hand o `start_project_integrate` → `start_local_development`) and skip the Frontend preview and Next Steps views. Project-folder selection still happens before autopilot starts; a non-empty workspace requires choosing a new subfolder or an empty folder so unattended generation cannot write over existing content. +While the run is active, CoR sets both local chat and Agent Host defaults to Autopilot, enables global +tool auto-approval, and raises the request limit. It restores the user's prior global chat defaults when +Autopilot finishes, is turned off from the status bar, or reaches its safety deadline. --- @@ -456,7 +468,11 @@ silently so a stale copy can't make an agent follow outdated steps. ## The MCP tools The extension exposes these tools to Copilot through the `vscode-azureresourcegroups.mcp` server -("Copilot Azure Resources Extension Tools"). Agents call them to open views and trigger the next stage. +("Copilot Azure Resources Extension Tools"). The server starts eagerly on a loopback TCP transport so +the Copilot Harness can connect before its first tool call. Agents call the tools to open views and +trigger the next stage. Agent manifests include both the Local harness reference names and the +Copilot Harness server label and tool-search alias because the two harnesses resolve those allowlist +entries differently. | Tool | Effect | | --- | --- | @@ -494,6 +510,7 @@ Everything the flow produces lives in the workspace, so it's inspectable and rev | `.copilot-azure/sessions/{id}/context.json` | deploy agent | Current phase and completed phases. Drives the Deployment progress view. | | `.azure/deploy-result.json` *or* `.copilot-azure/sessions/{id}/deploy-result.json` | deploy agent | In-progress and final deployment status, target, endpoints, resources, and recovery attempts. Drives Deployment progress and backs Deployment results. A workspace can hold several; the active session's result is used. | | `.github/agents/**` (+ `.version`) | extension | Copied agent instruction files and the version stamp. | +| `.vscode/settings.json` | extension | Workspace overrides that enable the Copilot Harness for CoR sessions. | Session/diagnostics state is kept in VS Code **workspaceState** (not files): `copilotOnRails.prompt`, `copilotOnRails.createdAt`, and `copilotOnRails.diagnosticEvents` (see below). diff --git a/evals/agent-assets.lock.json b/evals/agent-assets.lock.json index e1a306e07..87c47019f 100644 --- a/evals/agent-assets.lock.json +++ b/evals/agent-assets.lock.json @@ -1,9 +1,9 @@ { - "agentAssetsHash": "d89408d687ef9d5863e1ff1ac173f6c0440a273f449484afb90e096a2df7f710", + "agentAssetsHash": "94e21f1567b59ac091c9f50e49c8d5a1a126571400488c151340e519d98d3987", "scope": "azure-debug-generate.agent.md, azure-debug-generate/**, azure-debug-plan.agent.md, azure-debug-plan/**, azure-deploy.agent.md, azure-deploy/**, azure-project-integrate.agent.md, azure-project-integrate/**, azure-project-plan.agent.md, azure-project-plan/**, azure-project-scaffold.agent.md, azure-project-scaffold/**, shared-references/**", - "updatedAt": "2026-09-22T17:17:03.781Z", + "updatedAt": "2026-10-03T20:58:46.130Z", "files": { - "azure-debug-generate.agent.md": "c7f90ce78e00a2900bcac459393db8a2e76029dd0de77da5d8c0a44cd744b799", + "azure-debug-generate.agent.md": "1db8efeaa09757d446a3d74068a460ce98a46b0387226593f19e24e4c94cb244", "azure-debug-generate/.metadata.json": "c0ccce18678415f23db0073d48cc646f2de446e7821c99c3130553548d958e0c", "azure-debug-generate/instructions.md": "2dbf252f05dc7f523d726500941d4862a64a5579ff530b21400fc8a5294fed29", "azure-debug-generate/references/api-test-collections.md": "9ef4777f4b682b471b5386c97bb02cb08037545eb520d4534f69b784f5cd798b", @@ -25,7 +25,7 @@ "azure-debug-generate/references/runtimes/dotnet.md": "191ce6cb3eee84d4277d746c4ad094750a5dde0003b40ee7a64cbc621af0522f", "azure-debug-generate/references/runtimes/node.md": "7e037c0fdcf48ac440c046df35aa3112a9c356ea404655c3ecbbf862237b30a9", "azure-debug-generate/references/validation.md": "0daa62d9da8fc7d2a7e6a29ced7378720a2fb962cd67e032ce7e74074a6381cd", - "azure-debug-plan.agent.md": "e04cd53adee43b573ccc42eb423c5b4a29b7c28870599ffdf786ea51dbbef219", + "azure-debug-plan.agent.md": "150dd1c1a1ca4e9d0bae38b802050c9fbcf5624f0340ed70da361f7196113239", "azure-debug-plan/.metadata.json": "108c105ec0eaf5d7f0c2268ab2e0b04eb037649dc6668272f5ce42aa6da51de9", "azure-debug-plan/instructions.md": "0e15215047a1703dbbb868a6cd2a58ee1e2dc2bc231fc3f2e81e8056cd3dd9ea", "azure-debug-plan/references/classify.md": "1673b52fb25ff89f0437263ce7082f690c208a0783f140f432bb94a463376efa", @@ -35,7 +35,7 @@ "azure-debug-plan/references/plan-template.md": "fa08df53649226bd749371f08a9f096cd22ac83d1497e9ae90638993c464af0f", "azure-debug-plan/references/project-types.md": "4eca6fbec5cd2220b18ac5bad0ee7034870aa3739c78fb48d4338e2701bde4c6", "azure-debug-plan/references/runtimes.md": "97c9903a22e0580c4c0ff4d27a1e31c8a16e7ce07056f573701bf816bdfe45c3", - "azure-deploy.agent.md": "082f70b8e524d68ddef0faec69e514a4353d9cbc5540fbbc4d2cd7255a26c7d5", + "azure-deploy.agent.md": "c20f791f9f366731079ed5c2d24263c149cff2fbed8362dd6313b6d8a89b96b4", "azure-deploy/.metadata.json": "be0709e660b5da6f558d73e464bed2cd39a3382619bc9076796078843d43cb56", "azure-deploy/cor-references/migration-access.md": "2dce754d450740ade47ab1079e1d3046850c54da7d5acc19209825af8f9fdfb0", "azure-deploy/deploy/instructions.md": "30f877417f9d1da592abf903a12dc961e36c4b5063c48bab561ac1c152055abe", @@ -124,20 +124,20 @@ "azure-deploy/scaffold/references/waf-checklist.md": "d0b50ac5df46569034dd16431aa9ad9faa447f4330a4f4e41447042b160a3f0a", "azure-deploy/scaffold/scripts/scaffold-conformance.ps1": "85398b7fcd06e8207e536f558cca68d24ecf6186e54bf5680d7872aa38913ea0", "azure-deploy/scaffold/scripts/scaffold-conformance.sh": "c9961efccdece4c6d2a1c3a832389b359fb21ca0706fd7c33215d4e8bf267664", - "azure-project-integrate.agent.md": "8b955cc97e2d8e6ef406ef958980b1f0f7d360ab4ab6a602e86f3379e285d72b", + "azure-project-integrate.agent.md": "2d6dc6443170c44d9f628288a4643db901186aca05d7ec2a3938bfe7df4014e2", "azure-project-integrate/.metadata.json": "ee73bb3c96f66ed4c6f9d37db0716600e5deb3a23f55adcefcf34d6ed8d556c0", "azure-project-integrate/instructions.md": "0937e021c41429e71e8977b9a23bc5586e11de80d22ca0f85312b35643ea2ec4", "azure-project-integrate/references/end-to-end.md": "11bc63351517d530c7abc34a33842a89e25f9bad8951a7df2035b8a3c3a872aa", "azure-project-integrate/references/migrations.md": "4aa872a898ad09d4b22841e609c56350529a894a1e4f06030b0cd89ead0f2b13", "azure-project-integrate/references/smoke-test.md": "7c76b1734d5e82efaa23b6b61865b5d6c51d8cce8295b82046e2ac623ac45fa4", "azure-project-integrate/references/wire-live-data.md": "723e0b8ff0be74577d1619af774e9693ba2d97e0e7507eb95540cbacafa14643", - "azure-project-plan.agent.md": "5cc322c956a9f1061060674b884cca45d5ee869eaa7f3e5f977533e6a73188d5", + "azure-project-plan.agent.md": "2b8989ee3bbc23ead45c39f61bd404993c2c83145aed8b9f08cff69229af603c", "azure-project-plan/.metadata.json": "cec03694d63e45c8cb96983bd586790878b5813289885ae94d6cefc42243f37e", "azure-project-plan/instructions.md": "0d68b18de65bbe4fe375329d008373a8c915c6740f1aba5ecfce51980022e0c3", "azure-project-plan/plan.md": "a56f162fca888e1c81ea3f0c419c5f03712df1ebfb039bbcd665d5391572b64d", "azure-project-plan/references/html-preview.md": "2db272c10c4535743366483610dc46fb885d24795298220d0585880671665369", "azure-project-plan/requirements.md": "d53f610177edb81869165abc500cfd19960a887415e12cb77aa4cc0ace4a9b3d", - "azure-project-scaffold.agent.md": "fae02fae75b856b3a57d2bc5133ba9fe42d40257c9e862b1864606228b3d58ab", + "azure-project-scaffold.agent.md": "ab6ac1e0f9dbc2dab7bf87204b5db1e9ecd33e97884c00895b32deaffe5a0a71", "azure-project-scaffold/.metadata.json": "c6e76bb8c81c61ad58ce659e57d9ac7526bb680ad3923913db3a14e1333cd1e0", "azure-project-scaffold/instructions.md": "99ffd0dfc6afd4f9a21a2610ab1b6eb92b63f5b4e8252b962192495bdeef7143", "azure-project-scaffold/references/frontend-preview-steps.md": "dc60b0dd5a997bd1b38410b8c1e68d6453261a3e23b93de00654c864b49539cf", diff --git a/package-lock.json b/package-lock.json index 733264ff7..6f38639da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ "@microsoft/vscode-azext-azureutils": "^4.3.0", "@microsoft/vscode-azext-utils": "^4.1.0", "@microsoft/vscode-azext-webview": "^1.0.2", - "@microsoft/vscode-inproc-mcp": "^1.0.0", + "@microsoft/vscode-inproc-mcp": "^1.1.0", "form-data": "^4.0.6", "fs-extra": "^11.3.0", "jsonc-parser": "^2.2.1", @@ -40,7 +40,7 @@ "@types/vscode-webview": "^1.57.5", "@types/ws": "^8.5.10", "@vscode/test-cli": "*", - "@vscode/test-electron": "*", + "@vscode/test-electron": "^3.1.0", "@vscode/vsce": "*", "esbuild": "*", "esbuild-plugin-copy": "*", @@ -3417,16 +3417,16 @@ } }, "node_modules/@microsoft/vscode-inproc-mcp": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@microsoft/vscode-inproc-mcp/-/vscode-inproc-mcp-1.0.0.tgz", - "integrity": "sha512-KYj0A2dsKTrllAqxA+5CafhR8gY5mfdLTC5vKKNBb3aJ4Dcgau5su/3+S1HMwrRy9j68EgaPhtmCvKGjuOB+9g==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@microsoft/vscode-inproc-mcp/-/vscode-inproc-mcp-1.1.0.tgz", + "integrity": "sha512-WLL/eylaQ+iVOePc3U38cedU1l6BeU7Lmo8dVKuIvLEXLbZ4rFke8tVqu11oG2mFiunBc/9m2dqq0ckjpNlCZg==", "license": "MIT", "dependencies": { "@hono/node-server": "^2.0.10", "@microsoft/vscode-azext-utils": "^4.1.0", "@microsoft/vscode-processutils": "1.0.0", "@modelcontextprotocol/server": "2.0.0-beta.4", - "hono": "^4.12.30", + "hono": "^4.13.7", "zod": "^4.1.13" }, "engines": { @@ -5076,9 +5076,9 @@ } }, "node_modules/@vscode/test-electron": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@vscode/test-electron/-/test-electron-3.0.0.tgz", - "integrity": "sha512-TY5mC7aAjxSLDXsyjhrG8cJHgc/HLdiE5lvtW7hABYQrY24Qwozzr5UoO3HiuAM4Hzz4b7K/eZlwrCILj94CcA==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@vscode/test-electron/-/test-electron-3.1.0.tgz", + "integrity": "sha512-CRqv5u+YYoseuNVJ6Tyo4k0sF0mx4qnKMihRB0PjsUF8Dc0WKtCXo6CNL6nWWm5esfFQsQA/pejMj4ZbpJVLTw==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 379413143..61908380f 100644 --- a/package.json +++ b/package.json @@ -1045,7 +1045,7 @@ "@types/vscode-webview": "^1.57.5", "@types/ws": "^8.5.10", "@vscode/test-cli": "*", - "@vscode/test-electron": "*", + "@vscode/test-electron": "^3.1.0", "@vscode/vsce": "*", "esbuild": "*", "esbuild-plugin-copy": "*", @@ -1061,7 +1061,7 @@ "@microsoft/vscode-azext-azureutils": "^4.3.0", "@microsoft/vscode-azext-utils": "^4.1.0", "@microsoft/vscode-azext-webview": "^1.0.2", - "@microsoft/vscode-inproc-mcp": "^1.0.0", + "@microsoft/vscode-inproc-mcp": "^1.1.0", "form-data": "^4.0.6", "fs-extra": "^11.3.0", "jsonc-parser": "^2.2.1", diff --git a/resources/agents/azure-debug-generate.agent.md b/resources/agents/azure-debug-generate.agent.md index 7cfedeec5..20f163b36 100644 --- a/resources/agents/azure-debug-generate.agent.md +++ b/resources/agents/azure-debug-generate.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-generate description: Generate artifacts for an approved VS Code debugging plan. Responsible for generating docker-compose, VS Code launch/tasks, and emulator orchestration for streamlined project debugging. -tools: [vscode, copilot-azure-resources-extension-tools/*, tool_search, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] target: vscode --- diff --git a/resources/agents/azure-debug-plan.agent.md b/resources/agents/azure-debug-plan.agent.md index 22e1ea968..78eea2845 100644 --- a/resources/agents/azure-debug-plan.agent.md +++ b/resources/agents/azure-debug-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-plan description: Scan an Azure-centric workspace project. Classify its services and dependencies, and produce a local debugging plan covering automated emulator startup, VS Code launch/task configs, and API tests. -tools: [vscode, copilot-azure-resources-extension-tools/*, tool_search, execute, read, browser, edit, search, web, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, browser, edit, search, web, todo] target: vscode --- diff --git a/resources/agents/azure-deploy.agent.md b/resources/agents/azure-deploy.agent.md index bc816ec10..e00ff90d0 100644 --- a/resources/agents/azure-deploy.agent.md +++ b/resources/agents/azure-deploy.agent.md @@ -1,7 +1,7 @@ --- name: azure-deploy description: "Onboard and deploy an Azure-centric project end-to-end using a guided, self-contained onboarding pipeline. Analyzes deployment readiness, selects Azure services and SKUs, estimates cost, validates quota, generates secure Bicep/Terraform, provisions resources, deploys application code, and verifies health. Run after local development is set up. WHEN: deploy to Azure, ship to Azure, host on Azure, create infrastructure, generate IaC, provision resources, go live." -tools: [vscode, copilot-azure-resources-extension-tools/*, tool_search, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- diff --git a/resources/agents/azure-project-integrate.agent.md b/resources/agents/azure-project-integrate.agent.md index a9b011b2d..c7946e6c3 100644 --- a/resources/agents/azure-project-integrate.agent.md +++ b/resources/agents/azure-project-integrate.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-integrate description: Integrate a freshly scaffolded Azure-centric project — create the SQL/PostgreSQL schema migrations (NO seed data), smoke-test the backend so every endpoint responds, wire the frontend to LIVE backend data (replace all mock data), and run the frontend and backend wired together end-to-end. Runs after `azure-project-scaffold`. WHEN "integrate project", "wire to live data", "remove mock data", "smoke test backend", "verify endpoints", "create migrations", "wire frontend and backend", "integrate scaffold", "make the app run". -tools: [vscode, copilot-azure-resources-extension-tools/*, tool_search, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- diff --git a/resources/agents/azure-project-plan.agent.md b/resources/agents/azure-project-plan.agent.md index e43028606..a95b7e818 100644 --- a/resources/agents/azure-project-plan.agent.md +++ b/resources/agents/azure-project-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-plan description: Plan and design a NEW Azure-centric project the user asks to build — gather requirements interactively, produce an approved `.azure/project-plan.md`, then hand off to the `azure-project-scaffold` agent for execution. Use this for EVERY "build/create/make me an app" request, including frontend-only, static, single-page, and offline tools with no backend, no database, and no Azure services. WHEN "plan project", "design app", "new project", "project requirements", "create project plan", "plan my app", "what should I build", "new Azure app", "create testable app", "new API project", "full-stack Azure app", "bootstrap project", "new fullstack project", "create functions project", "build me an app", "make me a web app", "create a website", "simple web app", "little frontend tool", "static site", "frontend only app", "no backend app", "React app", "Vue app", "dashboard app", "Express API", "background worker", "multi-service app". -tools: [vscode, copilot-azure-resources-extension-tools/*, tool_search, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- diff --git a/resources/agents/azure-project-scaffold.agent.md b/resources/agents/azure-project-scaffold.agent.md index 6c2d70658..d23dd4264 100644 --- a/resources/agents/azure-project-scaffold.agent.md +++ b/resources/agents/azure-project-scaffold.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-scaffold description: Scaffold a NEW Azure-centric project from an ALREADY-APPROVED `.azure/project-plan.md` — generate the frontend, backend services, database, and API routes. Does NOT gather requirements or write the plan; that is the `azure-project-plan` agent's job. -tools: [vscode, copilot-azure-resources-extension-tools/*, tool_search, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index ff259853d..414a9b550 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -8,8 +8,9 @@ import * as vscode from 'vscode'; import { ext } from '../../extensionVariables'; import { projectSubmissionState } from '../../tree/project/projectSubmissionState'; import { CopilotOnRailsContext } from '../../utils/copilotOnRails/CopilotOnRailsContext'; +import { resolveCopilotHarnessModel } from '../../utils/copilotOnRails/modelSelection'; import { setCorErrorProp, setCorProp } from '../../utils/copilotOnRails/telemetryUtils'; -import { ensureLocalHarnessOn } from '../../webviews/copilotOnRails/extension/harnessSettings'; +import { ensureCopilotHarnessOn } from '../../webviews/copilotOnRails/extension/harnessSettings'; import { openLoadingView } from '../../webviews/copilotOnRails/extension/openLoadingView'; import { getSessionModel, recordAgentLaunch } from '../../webviews/copilotOnRails/extension/projectSession'; import { saveReloadResumePrompt } from '../../webviews/copilotOnRails/extension/reloadResumePrompt'; @@ -19,6 +20,11 @@ import { ensureAgentInstructions } from './agentInstructions'; const COPILOT_CHAT_EXTENSION_ID = 'GitHub.copilot-chat'; const MANAGE_WORKSPACE_TRUST_COMMAND_ID = 'workbench.trust.manage'; const RELOAD_WINDOW_COMMAND_ID = 'workbench.action.reloadWindow'; +const OPEN_CHAT_COMMAND_ID = 'workbench.action.chat.open'; +const OPEN_CHAT_COMMAND_PREFIX = 'workbench.action.chat.open'; +const AGENT_COMMAND_READY_TIMEOUT_MS = 10_000; +const AGENT_COMMAND_READY_POLL_MS = 100; +const EXISTING_AGENT_COMMAND_SETTLE_MS = 1_000; let agentLaunchInProgress = false; let requireWorkspaceTrustReload = false; @@ -37,26 +43,9 @@ export function registerWorkspaceTrustTracking(): void { async function resolveModelSelector(displayName: string): Promise<{ id?: string; vendor?: string } | undefined> { try { const models = await vscode.lm.selectChatModels(); - // Try matching by "Name (vendor)" format: e.g. "Claude Opus 4.7 (copilot)" - const vendorMatch = displayName.match(/^(.+?)\s*\((\w+)\)\s*$/); - if (vendorMatch) { - const [, name, vendor] = vendorMatch; - const match = models.find( - (m) => m.name === name.trim() && m.vendor === vendor, - ); - if (match) { - return { id: match.id, vendor: match.vendor }; - } - } - // Try matching by name alone - const byName = models.find((m) => m.name === displayName); - if (byName) { - return { id: byName.id, vendor: byName.vendor }; - } - // Try matching by id (in case the caller already has the id) - const byId = models.find((m) => m.id === displayName); - if (byId) { - return { id: byId.id, vendor: byId.vendor }; + const match = resolveCopilotHarnessModel(displayName, models); + if (match) { + return { id: match.id, vendor: match.vendor }; } } catch { // If the lm API isn't available, fall through @@ -101,7 +90,7 @@ export async function ensureCopilotChatReady(context: CopilotOnRailsContext): Pr } if (!copilotChatExtension.isActive) { - await ensureLocalHarnessOn(); + await ensureCopilotHarnessOn(); try { await vscode.window.withProgress( @@ -134,27 +123,35 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: agentLaunchInProgress = true; try { - await ensureLocalHarnessOn(); + await ensureCopilotHarnessOn(); // Fresh chat session per phase hand-off: agents coordinate through the `.azure/*` plan // files on disk, not chat history, so a clean session keeps each agent focused on its phase. await vscode.commands.executeCommand('workbench.action.chat.newChat'); + await waitForAgentModeRegistration(agentName); + await vscode.commands.executeCommand(getAgentChatOpenCommandId(agentName)); + await vscode.commands.executeCommand( + OPEN_CHAT_COMMAND_ID, + buildAgentChatModeOptions(agentName), + ); + const resolvedModel = model ?? getSessionModel(); setCorProp(context, 'chatModelSelectionSource', model ? 'newlySelected' : (getSessionModel() ? 'previouslySelected' : 'default')); const selector = resolvedModel ? await resolveModelSelector(resolvedModel) : undefined; setCorProp(context, 'chatModelResolved', !!selector); - // Custom modes get no per-mode open command, so passing `mode` to the generic chat-open - // command is the supported way to launch one. If the mode hasn't been discovered yet this - // silently opens the default Agent - the reload guard in prepareAndLaunchAgent prevents - // that (see requireWorkspaceTrustReload). - await vscode.commands.executeCommand('workbench.action.chat.open', { - mode: agentName, - query, - ...(selector ? { modelSelector: selector } : {}), - }); + const result = await vscode.commands.executeCommand( + OPEN_CHAT_COMMAND_ID, + buildAgentChatOpenOptions(agentName, query, selector), + ); + if (result === false) { + throw new Error(vscode.l10n.t( + 'The "{0}" Copilot agent request was rejected.', + agentName, + )); + } } catch (err) { const message = err instanceof Error ? err.message : String(err); setCorProp(context, chatLaunchOutcomeKey, 'error'); @@ -181,6 +178,51 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: return true; } +export function getAgentChatOpenCommandId(agentName: string): string { + return `${OPEN_CHAT_COMMAND_PREFIX}${agentName}`; +} + +export function buildAgentChatModeOptions(agentName: string): { mode: string } { + return { mode: agentName }; +} + +export function buildAgentChatOpenOptions( + agentName: string, + query: string, + modelSelector?: { id?: string; vendor?: string }, +): { mode: string; query: string; modelSelector?: { id?: string; vendor?: string }; waitForRequestAcceptance: true } { + return { + mode: agentName, + query, + ...(modelSelector ? { modelSelector } : {}), + waitForRequestAcceptance: true, + }; +} + +async function waitForAgentModeRegistration(agentName: string): Promise { + const commandId = getAgentChatOpenCommandId(agentName); + const deadline = Date.now() + AGENT_COMMAND_READY_TIMEOUT_MS; + let commandWasAlreadyRegistered = false; + + do { + if ((await vscode.commands.getCommands(true)).includes(commandId)) { + if (!commandWasAlreadyRegistered) { + commandWasAlreadyRegistered = true; + await new Promise(resolve => setTimeout(resolve, EXISTING_AGENT_COMMAND_SETTLE_MS)); + continue; + } + return; + } + commandWasAlreadyRegistered = false; + await new Promise(resolve => setTimeout(resolve, AGENT_COMMAND_READY_POLL_MS)); + } while (Date.now() < deadline); + + throw new Error(vscode.l10n.t( + 'The "{0}" Copilot agent was not registered for the selected chat harness.', + agentName, + )); +} + /** Result of {@link prepareAndLaunchAgent}; consumed by {@link launchAgentAndRecordOutcome}. */ type AgentLaunchOutcome = 'chatNotReady' | 'deferred' | 'launchFailed' | 'launched'; diff --git a/src/extension.ts b/src/extension.ts index 5896647a7..2e1dbd2a9 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -67,8 +67,8 @@ import { resumeCreateProjectViewAfterReload } from './webviews/copilotOnRails/ex import { registerDebugPlanImplementedWatcher } from './webviews/copilotOnRails/extension/debugPlanImplementedWatcher'; import { registerDeployInventoryWatcher } from './webviews/copilotOnRails/extension/deployInventoryWatcher'; import { registerDeployProgressWatcher } from './webviews/copilotOnRails/extension/deployProgressWatcher'; -import { registerDeploymentPlanAutoOpen } from './webviews/copilotOnRails/extension/openDeploymentPlanView'; import { registerDeployResultAutoOpen } from './webviews/copilotOnRails/extension/openDeployResultView'; +import { registerDeploymentPlanAutoOpen } from './webviews/copilotOnRails/extension/openDeploymentPlanView'; import { registerRequirementsAutoOpen } from './webviews/copilotOnRails/extension/openRequirementsView'; import { registerResumeAffordances } from './webviews/copilotOnRails/extension/resumeAffordances'; import { resumePendingCreateWithCopilot } from './webviews/copilotOnRails/extension/resumePendingCreateWithCopilot'; @@ -155,7 +155,9 @@ export async function activate(context: vscode.ExtensionContext, perfStats: { lo id: mcpServerId, serverLabel: mcpServerLabel, serverVersion: ext.version, + transport: 'tcp', registerTools: (server) => registerMcpTools(server), + eagerlyStart: true, }); // Reap any temporary database firewall rule an interrupted migration left behind. This is diff --git a/src/utils/copilotOnRails/modelSelection.ts b/src/utils/copilotOnRails/modelSelection.ts index 8e7e1d797..ce0830a3e 100644 --- a/src/utils/copilotOnRails/modelSelection.ts +++ b/src/utils/copilotOnRails/modelSelection.ts @@ -12,6 +12,7 @@ export interface AvailableChatModel { } export const supportedModelNames = ['Opus', 'Sonnet', 'GPT Sol', 'GPT Terra'] as const; +const copilotHarnessVendor = 'agent-host-copilotcli'; export function getSupportedModelName(...identifiers: string[]): typeof supportedModelNames[number] | undefined { const modelIdentifiers = identifiers.join(' '); @@ -31,6 +32,41 @@ export function getDefaultOpusModelOption(models: readonly AvailableChatModel[]) return model ? `${model.name} (${model.vendor})` : undefined; } +export function resolveAvailableChatModel(displayName: string, models: readonly AvailableChatModel[]): AvailableChatModel | undefined { + const vendorMatch = displayName.match(/^(.+?)\s*\(([^)]+)\)\s*$/); + if (vendorMatch) { + const [, name, vendor] = vendorMatch; + const match = models.find(model => model.name === name.trim() && model.vendor === vendor.trim()); + if (match) { + return match; + } + } + + return models.find(model => model.name === displayName) + ?? models.find(model => model.id === displayName); +} + +/** + * Maps a model selected from VS Code's local Copilot model pool to the corresponding + * Agent Host model used by the Copilot Harness. + */ +export function resolveCopilotHarnessModel(displayName: string, models: readonly AvailableChatModel[]): AvailableChatModel | undefined { + const selectedModel = resolveAvailableChatModel(displayName, models); + if (!selectedModel) { + return undefined; + } + + if (selectedModel.vendor === copilotHarnessVendor) { + return selectedModel; + } + + const harnessModels = models.filter(model => model.vendor === copilotHarnessVendor); + return harnessModels.find(model => model.id === selectedModel.id) + ?? harnessModels.find(model => model.family === selectedModel.family) + ?? harnessModels.find(model => model.name === selectedModel.name) + ?? { ...selectedModel, vendor: copilotHarnessVendor }; +} + /** * Returns the currently available supported Copilot models as qualified names * that VS Code can resolve when opening chat. diff --git a/src/webviews/copilotOnRails/extension/autopilot.ts b/src/webviews/copilotOnRails/extension/autopilot.ts index 5dd5f2483..99ba40d3e 100644 --- a/src/webviews/copilotOnRails/extension/autopilot.ts +++ b/src/webviews/copilotOnRails/extension/autopilot.ts @@ -13,13 +13,13 @@ import { settingUtils } from "../../../utils/settingUtils"; import { getDebugPlanStatus, LocalDebugPlanStatus, parseLocalDebugPlanMarkdown } from "../views/utils/parseLocalDebugPlanMarkdown"; import { isApprovedOrLater } from "../views/utils/projectPlanStatus"; import { armDebugPlanImplementedWatcher } from "./debugPlanImplementedWatcher"; -import { ensureLocalHarnessOn } from "./harnessSettings"; +import { ensureCopilotHarnessOn } from "./harnessSettings"; /** * Autopilot mode for the create-project workflow. * It temporarily enables global chat tool auto-approve and restores it later. * It also raises workspace chat request budget for long unattended runs, and forces the - * experimental "Copilot Harness" chat settings off at Workspace scope for the run. + * experimental "Copilot Harness" chat settings on at Workspace scope for the run. */ const AUTO_APPROVE_SECTION = 'chat.tools.global'; @@ -32,6 +32,8 @@ const MAX_REQUESTS_KEY = 'maxRequests'; export const WORKSPACE_MAX_REQUESTS = 9999; const PERMISSIONS_SECTION = 'chat.permissions'; const PERMISSIONS_KEY = 'default'; +const AGENT_HOST_CONFIGURATION_SECTION = 'chat'; +const AGENT_HOST_CONFIGURATION_KEY = 'defaultConfiguration'; /** * Maximum wall-clock duration an autopilot run may keep global auto-approve on. @@ -47,6 +49,7 @@ export const AUTOPILOT_QUERY_MARKER = '[AUTOPILOT MODE]'; const STATE_ACTIVE = 'copilotOnRails.autopilot.active'; const STATE_PRIOR_VALUE = 'copilotOnRails.autopilot.priorAutoApprove'; const STATE_PRIOR_PERMISSION_LEVEL = 'copilotOnRails.autopilot.priorPermissionLevel'; +const STATE_PRIOR_AGENT_HOST_CONFIGURATION = 'copilotOnRails.autopilot.priorAgentHostConfiguration'; /** Epoch ms after which an active run is considered stale and auto-restored. */ const STATE_DEADLINE = 'copilotOnRails.autopilot.deadline'; /** Set once per run to ensure the debug-plan approval telemetry has been recorded */ @@ -115,6 +118,40 @@ async function setPermissionLevelValue(value: unknown): Promise { await config.update(PERMISSIONS_KEY, value, vscode.ConfigurationTarget.Global); } +function getAgentHostConfiguration(): vscode.WorkspaceConfiguration | undefined { + const config = vscode.workspace.getConfiguration(AGENT_HOST_CONFIGURATION_SECTION); + return config.inspect(AGENT_HOST_CONFIGURATION_KEY) ? config : undefined; +} + +export function getAutopilotAgentHostConfiguration(value: unknown): Record { + const configuration = typeof value === 'object' && value !== null && !Array.isArray(value) + ? value as Record + : {}; + return { + ...configuration, + mode: 'autopilot', + approvals: 'allowAll', + }; +} + +async function enableAgentHostAutopilot(context: vscode.ExtensionContext): Promise { + const config = getAgentHostConfiguration(); + if (!config) { + return; + } + + if (context.globalState.get(STATE_PRIOR_AGENT_HOST_CONFIGURATION) === undefined) { + const prior = config.inspect(AGENT_HOST_CONFIGURATION_KEY)?.globalValue; + await context.globalState.update(STATE_PRIOR_AGENT_HOST_CONFIGURATION, prior ?? null); + } + + await config.update( + AGENT_HOST_CONFIGURATION_KEY, + getAutopilotAgentHostConfiguration(config.get(AGENT_HOST_CONFIGURATION_KEY)), + vscode.ConfigurationTarget.Global, + ); +} + function showStatusBarItem(): void { if (!statusBarItem) { statusBarItem = vscode.window.createStatusBarItem(vscode.StatusBarAlignment.Left, 100); @@ -257,7 +294,8 @@ export async function enableAutopilot(context: vscode.ExtensionContext): Promise await setAutoApproveValue(true); await raiseWorkspaceMaxRequests(); await setPermissionLevelValue('autopilot'); - await ensureLocalHarnessOn(); + await enableAgentHostAutopilot(context); + await ensureCopilotHarnessOn(); armAutopilot(deadline); } @@ -281,9 +319,20 @@ export async function disableAutopilot(): Promise { const priorPermission = context.globalState.get(STATE_PRIOR_PERMISSION_LEVEL); await setPermissionLevelValue(priorPermission === null ? undefined : priorPermission); + const priorAgentHostConfiguration = context.globalState.get(STATE_PRIOR_AGENT_HOST_CONFIGURATION); + if (priorAgentHostConfiguration !== undefined) { + const config = getAgentHostConfiguration(); + await config?.update( + AGENT_HOST_CONFIGURATION_KEY, + priorAgentHostConfiguration === null ? undefined : priorAgentHostConfiguration, + vscode.ConfigurationTarget.Global, + ); + } + await context.globalState.update(STATE_ACTIVE, undefined); await context.globalState.update(STATE_PRIOR_VALUE, undefined); await context.globalState.update(STATE_PRIOR_PERMISSION_LEVEL, undefined); + await context.globalState.update(STATE_PRIOR_AGENT_HOST_CONFIGURATION, undefined); await context.globalState.update(STATE_DEADLINE, undefined); await context.workspaceState.update(STATE_DEBUG_APPROVAL_RECORDED, undefined); } @@ -315,6 +364,12 @@ export function registerAutopilot(context: vscode.ExtensionContext): void { if (context.globalState.get(STATE_ACTIVE) === true) { const deadline = context.globalState.get(STATE_DEADLINE) ?? 0; if (Date.now() < deadline) { + void enableAgentHostAutopilot(context).catch(error => { + ext.outputChannel.warn(vscode.l10n.t( + 'Could not restore the Agent Host Autopilot configuration: {0}', + error instanceof Error ? error.message : String(error), + )); + }); armAutopilot(deadline); } else { void disableAutopilot(); diff --git a/src/webviews/copilotOnRails/extension/harnessSettings.ts b/src/webviews/copilotOnRails/extension/harnessSettings.ts index 18bd2a2c2..b85d2fc0f 100644 --- a/src/webviews/copilotOnRails/extension/harnessSettings.ts +++ b/src/webviews/copilotOnRails/extension/harnessSettings.ts @@ -7,12 +7,9 @@ import { ConfigurationTarget, workspace } from 'vscode'; import { settingUtils } from '../../../utils/settingUtils'; /** - * The two experimental "Copilot Harness" chat settings that can silently auto-enable - * across VS Code updates and break Copilot on Rails runs: + * The two experimental "Copilot Harness" chat settings used by Copilot on Rails: * - "Chat > Editor: Prefer Copilot Harness (Experimental)" * - "Chat > Default to Copilot Harness (Experimental)" - * - * Both are booleans; turning them off keeps the local chat harness in use for the run. */ const HARNESS_SETTINGS = [ { prefix: 'chat.editor', key: 'preferCopilotHarness' }, @@ -22,18 +19,18 @@ const HARNESS_SETTINGS = [ export const COPILOT_HARNESS_SETTING_IDS = HARNESS_SETTINGS.map(({ prefix, key }) => `${prefix}.${key}`); /** - * Ensures the local chat harness stays in use for a Copilot on Rails run by turning the two - * experimental "Copilot Harness" settings off at Workspace scope. Like the raised chat request + * Ensures Copilot on Rails uses the Copilot Harness by turning its two experimental settings on + * at Workspace scope. Like the raised chat request * budget, the workspace override is cheap and disposable, so it is intentionally left in place. */ -export async function ensureLocalHarnessOn(): Promise { +export async function ensureCopilotHarnessOn(): Promise { const folder = workspace.workspaceFolders?.[0]; if (!folder) { return; } for (const { prefix, key } of HARNESS_SETTINGS) { try { - await settingUtils.updateWorkspaceSetting(key, false, folder.uri.fsPath, prefix, ConfigurationTarget.Workspace); + await settingUtils.updateWorkspaceSetting(key, true, folder.uri.fsPath, prefix, ConfigurationTarget.Workspace); } catch { // Best effort: the setting may be unknown on older VS Code versions. } diff --git a/test/copilotOnRails/autopilot.test.ts b/test/copilotOnRails/autopilot.test.ts new file mode 100644 index 000000000..2e55f947d --- /dev/null +++ b/test/copilotOnRails/autopilot.test.ts @@ -0,0 +1,24 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { getAutopilotAgentHostConfiguration } from '../../src/webviews/copilotOnRails/extension/autopilot'; + +suite('Copilot on Rails autopilot', () => { + test('sets Agent Host mode and approvals without discarding other defaults', () => { + assert.deepStrictEqual( + getAutopilotAgentHostConfiguration({ + mode: 'interactive', + approvals: 'default', + futureSetting: true, + }), + { + mode: 'autopilot', + approvals: 'allowAll', + futureSetting: true, + }, + ); + }); +}); diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts new file mode 100644 index 000000000..d0422dd38 --- /dev/null +++ b/test/copilotOnRails/modelSelection.test.ts @@ -0,0 +1,52 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { + AvailableChatModel, + resolveAvailableChatModel, + resolveCopilotHarnessModel, +} from '../../src/utils/copilotOnRails/modelSelection'; + +const localModel: AvailableChatModel = { + id: 'gpt-6.1-sol', + vendor: 'copilot', + family: 'gpt-6.1-sol', + version: '6.1', + name: 'GPT-6.1 Sol', +}; + +const harnessModel: AvailableChatModel = { + ...localModel, + vendor: 'agent-host-copilotcli', +}; + +const legacyHarnessModel: AvailableChatModel = { + ...localModel, + vendor: 'copilotcli', +}; + +suite('Copilot on Rails model selection', () => { + test('resolves a qualified local model name', () => { + assert.strictEqual( + resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, harnessModel]), + localModel, + ); + }); + + test('maps a local Copilot model to the Agent Host model with the same id', () => { + assert.strictEqual( + resolveCopilotHarnessModel('GPT-6.1 Sol (copilot)', [localModel, legacyHarnessModel, harnessModel]), + harnessModel, + ); + }); + + test('constructs an Agent Host selector when the extension API omits Agent Host models', () => { + assert.deepStrictEqual( + resolveCopilotHarnessModel('GPT-6.1 Sol (copilot)', [localModel, legacyHarnessModel]), + harnessModel, + ); + }); +}); diff --git a/test/copilotOnRails/openChatWithAgent.test.ts b/test/copilotOnRails/openChatWithAgent.test.ts new file mode 100644 index 000000000..4ba834166 --- /dev/null +++ b/test/copilotOnRails/openChatWithAgent.test.ts @@ -0,0 +1,32 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { buildAgentChatModeOptions, buildAgentChatOpenOptions } from '../../src/commands/copilotOnRails/openChatWithAgent'; + +suite('Copilot on Rails agent chat launch', () => { + test('builds options that select the custom agent without submitting a prompt', () => { + assert.deepStrictEqual( + buildAgentChatModeOptions('azure-project-integrate'), + { mode: 'azure-project-integrate' }, + ); + }); + + test('resolves the agent by name when submitting the prompt', () => { + assert.deepStrictEqual( + buildAgentChatOpenOptions( + 'azure-project-plan', + 'Build a project', + { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + ), + { + mode: 'azure-project-plan', + query: 'Build a project', + modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + waitForRequestAcceptance: true, + }, + ); + }); +}); From 88519389308b725708a03eace151a7a31a47c98a Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:21:34 -0700 Subject: [PATCH 02/34] Ensure custom agent and initial model --- evals/agent-assets.lock.json | 18 ++++----- evals/check-agent-drift.ts | 39 +++++++++++++++++++ .../azure-project-scaffold/instructions.md | 2 +- .../copilotOnRails/openChatWithAgent.ts | 6 ++- test/copilotOnRails/openChatWithAgent.test.ts | 23 +++++++++++ 5 files changed, 76 insertions(+), 12 deletions(-) diff --git a/evals/agent-assets.lock.json b/evals/agent-assets.lock.json index 87c47019f..af6d55e14 100644 --- a/evals/agent-assets.lock.json +++ b/evals/agent-assets.lock.json @@ -1,9 +1,9 @@ { - "agentAssetsHash": "94e21f1567b59ac091c9f50e49c8d5a1a126571400488c151340e519d98d3987", + "agentAssetsHash": "eabd3855d21e38ce607a0f1ea087b9576ad6ee1b4c5d16d6a35cc1d9db408430", "scope": "azure-debug-generate.agent.md, azure-debug-generate/**, azure-debug-plan.agent.md, azure-debug-plan/**, azure-deploy.agent.md, azure-deploy/**, azure-project-integrate.agent.md, azure-project-integrate/**, azure-project-plan.agent.md, azure-project-plan/**, azure-project-scaffold.agent.md, azure-project-scaffold/**, shared-references/**", - "updatedAt": "2026-10-03T20:58:46.130Z", + "updatedAt": "2026-10-04T05:54:18.497Z", "files": { - "azure-debug-generate.agent.md": "1db8efeaa09757d446a3d74068a460ce98a46b0387226593f19e24e4c94cb244", + "azure-debug-generate.agent.md": "8ea64b01675963ad33ec79854b2ecdab026d7843fe474d99a497a0d8135bb847", "azure-debug-generate/.metadata.json": "c0ccce18678415f23db0073d48cc646f2de446e7821c99c3130553548d958e0c", "azure-debug-generate/instructions.md": "2dbf252f05dc7f523d726500941d4862a64a5579ff530b21400fc8a5294fed29", "azure-debug-generate/references/api-test-collections.md": "9ef4777f4b682b471b5386c97bb02cb08037545eb520d4534f69b784f5cd798b", @@ -25,7 +25,7 @@ "azure-debug-generate/references/runtimes/dotnet.md": "191ce6cb3eee84d4277d746c4ad094750a5dde0003b40ee7a64cbc621af0522f", "azure-debug-generate/references/runtimes/node.md": "7e037c0fdcf48ac440c046df35aa3112a9c356ea404655c3ecbbf862237b30a9", "azure-debug-generate/references/validation.md": "0daa62d9da8fc7d2a7e6a29ced7378720a2fb962cd67e032ce7e74074a6381cd", - "azure-debug-plan.agent.md": "150dd1c1a1ca4e9d0bae38b802050c9fbcf5624f0340ed70da361f7196113239", + "azure-debug-plan.agent.md": "ef70a79e0b0e9199576a1b50e2d138ad3d34b766148190f598afa78cb93fe711", "azure-debug-plan/.metadata.json": "108c105ec0eaf5d7f0c2268ab2e0b04eb037649dc6668272f5ce42aa6da51de9", "azure-debug-plan/instructions.md": "0e15215047a1703dbbb868a6cd2a58ee1e2dc2bc231fc3f2e81e8056cd3dd9ea", "azure-debug-plan/references/classify.md": "1673b52fb25ff89f0437263ce7082f690c208a0783f140f432bb94a463376efa", @@ -35,7 +35,7 @@ "azure-debug-plan/references/plan-template.md": "fa08df53649226bd749371f08a9f096cd22ac83d1497e9ae90638993c464af0f", "azure-debug-plan/references/project-types.md": "4eca6fbec5cd2220b18ac5bad0ee7034870aa3739c78fb48d4338e2701bde4c6", "azure-debug-plan/references/runtimes.md": "97c9903a22e0580c4c0ff4d27a1e31c8a16e7ce07056f573701bf816bdfe45c3", - "azure-deploy.agent.md": "c20f791f9f366731079ed5c2d24263c149cff2fbed8362dd6313b6d8a89b96b4", + "azure-deploy.agent.md": "26d829f5f43707c46852cefd522f6ecb6efb500dc58f99f216007569330d402b", "azure-deploy/.metadata.json": "be0709e660b5da6f558d73e464bed2cd39a3382619bc9076796078843d43cb56", "azure-deploy/cor-references/migration-access.md": "2dce754d450740ade47ab1079e1d3046850c54da7d5acc19209825af8f9fdfb0", "azure-deploy/deploy/instructions.md": "30f877417f9d1da592abf903a12dc961e36c4b5063c48bab561ac1c152055abe", @@ -124,22 +124,22 @@ "azure-deploy/scaffold/references/waf-checklist.md": "d0b50ac5df46569034dd16431aa9ad9faa447f4330a4f4e41447042b160a3f0a", "azure-deploy/scaffold/scripts/scaffold-conformance.ps1": "85398b7fcd06e8207e536f558cca68d24ecf6186e54bf5680d7872aa38913ea0", "azure-deploy/scaffold/scripts/scaffold-conformance.sh": "c9961efccdece4c6d2a1c3a832389b359fb21ca0706fd7c33215d4e8bf267664", - "azure-project-integrate.agent.md": "2d6dc6443170c44d9f628288a4643db901186aca05d7ec2a3938bfe7df4014e2", + "azure-project-integrate.agent.md": "bc767ea55ff727de8e3b8603cd36299b45ada339b63e4fae477643d8ec03e5b7", "azure-project-integrate/.metadata.json": "ee73bb3c96f66ed4c6f9d37db0716600e5deb3a23f55adcefcf34d6ed8d556c0", "azure-project-integrate/instructions.md": "0937e021c41429e71e8977b9a23bc5586e11de80d22ca0f85312b35643ea2ec4", "azure-project-integrate/references/end-to-end.md": "11bc63351517d530c7abc34a33842a89e25f9bad8951a7df2035b8a3c3a872aa", "azure-project-integrate/references/migrations.md": "4aa872a898ad09d4b22841e609c56350529a894a1e4f06030b0cd89ead0f2b13", "azure-project-integrate/references/smoke-test.md": "7c76b1734d5e82efaa23b6b61865b5d6c51d8cce8295b82046e2ac623ac45fa4", "azure-project-integrate/references/wire-live-data.md": "723e0b8ff0be74577d1619af774e9693ba2d97e0e7507eb95540cbacafa14643", - "azure-project-plan.agent.md": "2b8989ee3bbc23ead45c39f61bd404993c2c83145aed8b9f08cff69229af603c", + "azure-project-plan.agent.md": "e77e51a8842f2d8b5f026d30baafff1da5458871aca2a9ff6d664b9ba2985cd4", "azure-project-plan/.metadata.json": "cec03694d63e45c8cb96983bd586790878b5813289885ae94d6cefc42243f37e", "azure-project-plan/instructions.md": "0d68b18de65bbe4fe375329d008373a8c915c6740f1aba5ecfce51980022e0c3", "azure-project-plan/plan.md": "a56f162fca888e1c81ea3f0c419c5f03712df1ebfb039bbcd665d5391572b64d", "azure-project-plan/references/html-preview.md": "2db272c10c4535743366483610dc46fb885d24795298220d0585880671665369", "azure-project-plan/requirements.md": "d53f610177edb81869165abc500cfd19960a887415e12cb77aa4cc0ace4a9b3d", - "azure-project-scaffold.agent.md": "ab6ac1e0f9dbc2dab7bf87204b5db1e9ecd33e97884c00895b32deaffe5a0a71", + "azure-project-scaffold.agent.md": "234f87dbc3990f788f7d0b30457a0329f6192ec7a5d67f87e78705dc965969ed", "azure-project-scaffold/.metadata.json": "c6e76bb8c81c61ad58ce659e57d9ac7526bb680ad3923913db3a14e1333cd1e0", - "azure-project-scaffold/instructions.md": "99ffd0dfc6afd4f9a21a2610ab1b6eb92b63f5b4e8252b962192495bdeef7143", + "azure-project-scaffold/instructions.md": "e59d0266cc8c226f8f9b9d7c0985320daed59e33c3a376fe1046a325125f74f6", "azure-project-scaffold/references/frontend-preview-steps.md": "dc60b0dd5a997bd1b38410b8c1e68d6453261a3e23b93de00654c864b49539cf", "azure-project-scaffold/references/frontend-quality-bar.md": "3893f2cc21b7d3334cd3c464d065e7d50c3fc68f2d1cc680c85f78c781f45f52", "azure-project-scaffold/references/sub-agent-strategy.md": "225ae17f9c89722f31bd40e6470ffd8a1c693a4cbc00c8024127f070fc5d9b8e", diff --git a/evals/check-agent-drift.ts b/evals/check-agent-drift.ts index 6d3ba353b..abdd7bfd0 100644 --- a/evals/check-agent-drift.ts +++ b/evals/check-agent-drift.ts @@ -23,6 +23,7 @@ import { createHash } from "node:crypto"; import * as fs from "node:fs"; import * as path from "node:path"; +import { parse as parseYaml } from "yaml"; import { getSupportedModelName, supportedModelNames } from "../src/utils/copilotOnRails/modelSelection.ts"; import { listEvalAssetFiles, SHARED_FOLDER } from "./src/agent-definition.ts"; import { resolveSweepModels } from "./msbench/models.ts"; @@ -158,6 +159,18 @@ const contracts: Contract[] = [ pattern: /login page MUST contain a visible \*\*Create account\*\* button that opens a dedicated create-account page/, grader: "API Login scaffold contract (create-account control and page)", }, + { + file: "azure-project-scaffold.agent.md", + name: "autopilot-integrate-handoff-no-arguments", + pattern: /Call `start_project_integrate` without arguments\.[^\n]*does not accept a `prompt`:\s*```json\s*\{\}/, + grader: "start_project_integrate accepts no input; the extension owns Autopilot propagation", + }, + { + file: "azure-project-scaffold/instructions.md", + name: "autopilot-integrate-manual-no-arguments", + pattern: /call `start_project_integrate` without arguments[^\n]*do not pass a `prompt`/, + grader: "scaffold manual must match the zero-argument integration handoff tool", + }, { file: "azure-debug-plan/references/inventory.md", name: "derived-auth-api-test-inventory", @@ -320,6 +333,32 @@ function describeAssetChanges( const failures: string[] = []; const checked: string[] = []; +for (const file of fs.readdirSync(agentsRoot).filter(name => name.endsWith(".agent.md"))) { + const body = fs.readFileSync(path.join(agentsRoot, file), "utf8"); + const header = /^---\r?\n([\s\S]*?)\r?\n---/.exec(body); + const metadata: unknown = header ? parseYaml(header[1]) : undefined; + const tools: unknown = metadata && typeof metadata === "object" && !Array.isArray(metadata) && "tools" in metadata + ? metadata.tools + : undefined; + const requiredTools = [ + "copilot-azure-resources-extension-tools/*", "Copilot Azure Resources Extension Tools/*", + "tool_search", "toolSearch", "execute", "read", "browser", "edit", "search", "web", "todo", + ]; + if (file !== "azure-debug-plan.agent.md") { + requiredTools.push("agent", "azure-mcp/search", "Azure MCP/*"); + } + if (file === "azure-deploy.agent.md") { + requiredTools.push("bicep/*", "Bicep/*"); + } + const allowedTools = new Set(requiredTools); + if (!Array.isArray(tools) || tools.some(tool => typeof tool !== "string" || !allowedTools.has(tool)) + || requiredTools.some(tool => !tools.includes(tool))) { + failures.push(`scoped-agent-tools (${file}): tools must preserve the phase's required core, CoR, and Azure tools without broad vscode or unrelated MCP access`); + } else { + checked.push(`scoped-agent-tools (${file})`); + } +} + for (const contract of contracts) { const filePath = path.join(agentsRoot, contract.file); if (!fs.existsSync(filePath)) { diff --git a/resources/agents/azure-project-scaffold/instructions.md b/resources/agents/azure-project-scaffold/instructions.md index ff06cf558..f75935dc2 100644 --- a/resources/agents/azure-project-scaffold/instructions.md +++ b/resources/agents/azure-project-scaffold/instructions.md @@ -28,7 +28,7 @@ Requires approved plan. Before starting, verify: **Active when** invoking chat query starts with `[AUTOPILOT MODE]`, **or** `.azure/project-plan.md` contains `executionMode: auto` (front-matter or `**Execution Mode**: auto` row). Run unattended: - **Skip the plan preview & approval** — already approved upstream; scaffold directly (do NOT open plan view or re-request approval). - **Skip the frontend preview approval gate** — do NOT call the `open_frontend_preview_view` tool; the UI is auto-approved in autopilot. -- **Replace the Step 11 "Next Step" question with integrate hand-off** — do NOT call `vscode_askQuestions`. Write `.azure/integration-plan.md`, then hand off unattended via `start_project_integrate`, prefixing `prompt` with `[AUTOPILOT MODE] `. +- **Replace the Step 11 "Next Step" question with integrate hand-off** - do NOT call `vscode_askQuestions`. Write `.azure/integration-plan.md`, then call `start_project_integrate` without arguments (`{}`). The extension carries the active Autopilot mode and query marker into the new chat; do not pass a `prompt`. - All quality work (frontend preview verification, build gates, `.azure/.preview-temp/` cleanup) still applies — autopilot suppresses **only gates and questions**. ## Rules diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index 414a9b550..6f9b0221b 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -10,6 +10,7 @@ import { projectSubmissionState } from '../../tree/project/projectSubmissionStat import { CopilotOnRailsContext } from '../../utils/copilotOnRails/CopilotOnRailsContext'; import { resolveCopilotHarnessModel } from '../../utils/copilotOnRails/modelSelection'; import { setCorErrorProp, setCorProp } from '../../utils/copilotOnRails/telemetryUtils'; +import { AUTOPILOT_QUERY_MARKER, isAutopilotActive } from '../../webviews/copilotOnRails/extension/autopilot'; import { ensureCopilotHarnessOn } from '../../webviews/copilotOnRails/extension/harnessSettings'; import { openLoadingView } from '../../webviews/copilotOnRails/extension/openLoadingView'; import { getSessionModel, recordAgentLaunch } from '../../webviews/copilotOnRails/extension/projectSession'; @@ -144,7 +145,7 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: const result = await vscode.commands.executeCommand( OPEN_CHAT_COMMAND_ID, - buildAgentChatOpenOptions(agentName, query, selector), + buildAgentChatOpenOptions(agentName, query, selector, isAutopilotActive()), ); if (result === false) { throw new Error(vscode.l10n.t( @@ -190,10 +191,11 @@ export function buildAgentChatOpenOptions( agentName: string, query: string, modelSelector?: { id?: string; vendor?: string }, + autopilot = false, ): { mode: string; query: string; modelSelector?: { id?: string; vendor?: string }; waitForRequestAcceptance: true } { return { mode: agentName, - query, + query: autopilot && !query.startsWith(AUTOPILOT_QUERY_MARKER) ? `${AUTOPILOT_QUERY_MARKER} ${query}` : query, ...(modelSelector ? { modelSelector } : {}), waitForRequestAcceptance: true, }; diff --git a/test/copilotOnRails/openChatWithAgent.test.ts b/test/copilotOnRails/openChatWithAgent.test.ts index 4ba834166..4809b6728 100644 --- a/test/copilotOnRails/openChatWithAgent.test.ts +++ b/test/copilotOnRails/openChatWithAgent.test.ts @@ -29,4 +29,27 @@ suite('Copilot on Rails agent chat launch', () => { }, ); }); + + test('carries Autopilot into fresh phase chats without changing the agent or model', () => { + for (const agentName of ['azure-project-scaffold', 'azure-project-integrate', 'azure-debug-plan', 'azure-debug-generate', 'azure-deploy']) { + assert.deepStrictEqual( + buildAgentChatOpenOptions(agentName, 'Continue the project', { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, true), + { + mode: agentName, + query: '[AUTOPILOT MODE] Continue the project', + modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + waitForRequestAcceptance: true, + }, + ); + } + }); + + test('does not duplicate an existing Autopilot marker', () => { + const query = '[AUTOPILOT MODE] I approve the plan.'; + assert.strictEqual(buildAgentChatOpenOptions('azure-project-scaffold', query, undefined, true).query, query); + }); + + test('preserves interactive handoff prompts', () => { + assert.strictEqual(buildAgentChatOpenOptions('azure-project-integrate', 'Continue the project', undefined, false).query, 'Continue the project'); + }); }); From 174df0e0915a8cb662653b34c66c2f905c3e0b8c Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:45:47 -0700 Subject: [PATCH 03/34] Simplify drift file --- evals/check-agent-drift.ts | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/evals/check-agent-drift.ts b/evals/check-agent-drift.ts index abdd7bfd0..07a05abc2 100644 --- a/evals/check-agent-drift.ts +++ b/evals/check-agent-drift.ts @@ -159,18 +159,6 @@ const contracts: Contract[] = [ pattern: /login page MUST contain a visible \*\*Create account\*\* button that opens a dedicated create-account page/, grader: "API Login scaffold contract (create-account control and page)", }, - { - file: "azure-project-scaffold.agent.md", - name: "autopilot-integrate-handoff-no-arguments", - pattern: /Call `start_project_integrate` without arguments\.[^\n]*does not accept a `prompt`:\s*```json\s*\{\}/, - grader: "start_project_integrate accepts no input; the extension owns Autopilot propagation", - }, - { - file: "azure-project-scaffold/instructions.md", - name: "autopilot-integrate-manual-no-arguments", - pattern: /call `start_project_integrate` without arguments[^\n]*do not pass a `prompt`/, - grader: "scaffold manual must match the zero-argument integration handoff tool", - }, { file: "azure-debug-plan/references/inventory.md", name: "derived-auth-api-test-inventory", @@ -333,6 +321,8 @@ function describeAssetChanges( const failures: string[] = []; const checked: string[] = []; +// Agent frontmatter is the permission boundary for every custom agent. Validate the parsed +// `tools` array rather than matching YAML text so formatting changes do not affect this check. for (const file of fs.readdirSync(agentsRoot).filter(name => name.endsWith(".agent.md"))) { const body = fs.readFileSync(path.join(agentsRoot, file), "utf8"); const header = /^---\r?\n([\s\S]*?)\r?\n---/.exec(body); @@ -340,16 +330,27 @@ for (const file of fs.readdirSync(agentsRoot).filter(name => name.endsWith(".age const tools: unknown = metadata && typeof metadata === "object" && !Array.isArray(metadata) && "tools" in metadata ? metadata.tools : undefined; + + // These tools are shared by every phase. Both names in each alias pair are required + // because supported agent hosts expose the same capability under different identifiers. const requiredTools = [ "copilot-azure-resources-extension-tools/*", "Copilot Azure Resources Extension Tools/*", "tool_search", "toolSearch", "execute", "read", "browser", "edit", "search", "web", "todo", ]; + + // The debug-plan phase only produces a local debugging plan. Every other phase may + // delegate work and query Azure, so those phases need both forms of the Azure MCP tool. if (file !== "azure-debug-plan.agent.md") { requiredTools.push("agent", "azure-mcp/search", "Azure MCP/*"); } + + // Deployment also authors and validates infrastructure, which requires the Bicep tools. if (file === "azure-deploy.agent.md") { requiredTools.push("bicep/*", "Bicep/*"); } + + // Require an exact set: reject malformed entries, missing capabilities, and extra tools. + // In particular, broad `vscode` or unrelated MCP access must not bypass phase scoping. const allowedTools = new Set(requiredTools); if (!Array.isArray(tools) || tools.some(tool => typeof tool !== "string" || !allowedTools.has(tool)) || requiredTools.some(tool => !tools.includes(tool))) { From 95ec72f98e208049b751098f17ad0f107c2a0744 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:54:39 -0700 Subject: [PATCH 04/34] Fix agent contract checks --- evals/agent-assets.lock.json | 6 +++--- evals/check-agent-drift.ts | 14 ++++++-------- resources/agents/azure-debug-generate.agent.md | 2 +- resources/agents/azure-debug-plan.agent.md | 2 +- resources/agents/azure-deploy.agent.md | 2 +- resources/agents/azure-project-integrate.agent.md | 2 +- resources/agents/azure-project-plan.agent.md | 2 +- resources/agents/azure-project-scaffold.agent.md | 6 +++--- 8 files changed, 17 insertions(+), 19 deletions(-) diff --git a/evals/agent-assets.lock.json b/evals/agent-assets.lock.json index af6d55e14..8cd1c5244 100644 --- a/evals/agent-assets.lock.json +++ b/evals/agent-assets.lock.json @@ -1,7 +1,7 @@ { - "agentAssetsHash": "eabd3855d21e38ce607a0f1ea087b9576ad6ee1b4c5d16d6a35cc1d9db408430", + "agentAssetsHash": "8b54190c691ca14832ccbfb1de1abc54f1f4e274255e58b856bd1809c3827629", "scope": "azure-debug-generate.agent.md, azure-debug-generate/**, azure-debug-plan.agent.md, azure-debug-plan/**, azure-deploy.agent.md, azure-deploy/**, azure-project-integrate.agent.md, azure-project-integrate/**, azure-project-plan.agent.md, azure-project-plan/**, azure-project-scaffold.agent.md, azure-project-scaffold/**, shared-references/**", - "updatedAt": "2026-10-04T05:54:18.497Z", + "updatedAt": "2026-10-05T17:53:38.043Z", "files": { "azure-debug-generate.agent.md": "8ea64b01675963ad33ec79854b2ecdab026d7843fe474d99a497a0d8135bb847", "azure-debug-generate/.metadata.json": "c0ccce18678415f23db0073d48cc646f2de446e7821c99c3130553548d958e0c", @@ -137,7 +137,7 @@ "azure-project-plan/plan.md": "a56f162fca888e1c81ea3f0c419c5f03712df1ebfb039bbcd665d5391572b64d", "azure-project-plan/references/html-preview.md": "2db272c10c4535743366483610dc46fb885d24795298220d0585880671665369", "azure-project-plan/requirements.md": "d53f610177edb81869165abc500cfd19960a887415e12cb77aa4cc0ace4a9b3d", - "azure-project-scaffold.agent.md": "234f87dbc3990f788f7d0b30457a0329f6192ec7a5d67f87e78705dc965969ed", + "azure-project-scaffold.agent.md": "a48132ce938d08e74f9f8ca241381597b7d980fa342992a1466e8791b126bad1", "azure-project-scaffold/.metadata.json": "c6e76bb8c81c61ad58ce659e57d9ac7526bb680ad3923913db3a14e1333cd1e0", "azure-project-scaffold/instructions.md": "e59d0266cc8c226f8f9b9d7c0985320daed59e33c3a376fe1046a325125f74f6", "azure-project-scaffold/references/frontend-preview-steps.md": "dc60b0dd5a997bd1b38410b8c1e68d6453261a3e23b93de00654c864b49539cf", diff --git a/evals/check-agent-drift.ts b/evals/check-agent-drift.ts index 07a05abc2..4de8194de 100644 --- a/evals/check-agent-drift.ts +++ b/evals/check-agent-drift.ts @@ -23,7 +23,6 @@ import { createHash } from "node:crypto"; import * as fs from "node:fs"; import * as path from "node:path"; -import { parse as parseYaml } from "yaml"; import { getSupportedModelName, supportedModelNames } from "../src/utils/copilotOnRails/modelSelection.ts"; import { listEvalAssetFiles, SHARED_FOLDER } from "./src/agent-definition.ts"; import { resolveSweepModels } from "./msbench/models.ts"; @@ -321,15 +320,14 @@ function describeAssetChanges( const failures: string[] = []; const checked: string[] = []; -// Agent frontmatter is the permission boundary for every custom agent. Validate the parsed -// `tools` array rather than matching YAML text so formatting changes do not affect this check. +// Agent frontmatter is the permission boundary for every custom agent. Manifests use a +// single-line YAML flow sequence for `tools`, so read its entries without adding a runtime +// dependency to the extension's root package. for (const file of fs.readdirSync(agentsRoot).filter(name => name.endsWith(".agent.md"))) { const body = fs.readFileSync(path.join(agentsRoot, file), "utf8"); const header = /^---\r?\n([\s\S]*?)\r?\n---/.exec(body); - const metadata: unknown = header ? parseYaml(header[1]) : undefined; - const tools: unknown = metadata && typeof metadata === "object" && !Array.isArray(metadata) && "tools" in metadata - ? metadata.tools - : undefined; + const toolsLine = header ? /^tools:\s*\[([^\r\n]*)\]\s*$/m.exec(header[1]) : undefined; + const tools = toolsLine?.[1].split(",").map(tool => tool.trim()).filter(Boolean); // These tools are shared by every phase. Both names in each alias pair are required // because supported agent hosts expose the same capability under different identifiers. @@ -352,7 +350,7 @@ for (const file of fs.readdirSync(agentsRoot).filter(name => name.endsWith(".age // Require an exact set: reject malformed entries, missing capabilities, and extra tools. // In particular, broad `vscode` or unrelated MCP access must not bypass phase scoping. const allowedTools = new Set(requiredTools); - if (!Array.isArray(tools) || tools.some(tool => typeof tool !== "string" || !allowedTools.has(tool)) + if (!tools || tools.some(tool => !allowedTools.has(tool)) || requiredTools.some(tool => !tools.includes(tool))) { failures.push(`scoped-agent-tools (${file}): tools must preserve the phase's required core, CoR, and Azure tools without broad vscode or unrelated MCP access`); } else { diff --git a/resources/agents/azure-debug-generate.agent.md b/resources/agents/azure-debug-generate.agent.md index 20f163b36..01b0d3fdd 100644 --- a/resources/agents/azure-debug-generate.agent.md +++ b/resources/agents/azure-debug-generate.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-generate description: Generate artifacts for an approved VS Code debugging plan. Responsible for generating docker-compose, VS Code launch/tasks, and emulator orchestration for streamlined project debugging. -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] target: vscode --- diff --git a/resources/agents/azure-debug-plan.agent.md b/resources/agents/azure-debug-plan.agent.md index 78eea2845..1c0d28e7d 100644 --- a/resources/agents/azure-debug-plan.agent.md +++ b/resources/agents/azure-debug-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-plan description: Scan an Azure-centric workspace project. Classify its services and dependencies, and produce a local debugging plan covering automated emulator startup, VS Code launch/task configs, and API tests. -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, browser, edit, search, web, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, browser, edit, search, web, todo] target: vscode --- diff --git a/resources/agents/azure-deploy.agent.md b/resources/agents/azure-deploy.agent.md index e00ff90d0..c0f7c7715 100644 --- a/resources/agents/azure-deploy.agent.md +++ b/resources/agents/azure-deploy.agent.md @@ -1,7 +1,7 @@ --- name: azure-deploy description: "Onboard and deploy an Azure-centric project end-to-end using a guided, self-contained onboarding pipeline. Analyzes deployment readiness, selects Azure services and SKUs, estimates cost, validates quota, generates secure Bicep/Terraform, provisions resources, deploys application code, and verifies health. Run after local development is set up. WHEN: deploy to Azure, ship to Azure, host on Azure, create infrastructure, generate IaC, provision resources, go live." -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, bicep/*, Bicep/*, todo] --- diff --git a/resources/agents/azure-project-integrate.agent.md b/resources/agents/azure-project-integrate.agent.md index c7946e6c3..80b3b6554 100644 --- a/resources/agents/azure-project-integrate.agent.md +++ b/resources/agents/azure-project-integrate.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-integrate description: Integrate a freshly scaffolded Azure-centric project — create the SQL/PostgreSQL schema migrations (NO seed data), smoke-test the backend so every endpoint responds, wire the frontend to LIVE backend data (replace all mock data), and run the frontend and backend wired together end-to-end. Runs after `azure-project-scaffold`. WHEN "integrate project", "wire to live data", "remove mock data", "smoke test backend", "verify endpoints", "create migrations", "wire frontend and backend", "integrate scaffold", "make the app run". -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] --- diff --git a/resources/agents/azure-project-plan.agent.md b/resources/agents/azure-project-plan.agent.md index a95b7e818..45f711833 100644 --- a/resources/agents/azure-project-plan.agent.md +++ b/resources/agents/azure-project-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-plan description: Plan and design a NEW Azure-centric project the user asks to build — gather requirements interactively, produce an approved `.azure/project-plan.md`, then hand off to the `azure-project-scaffold` agent for execution. Use this for EVERY "build/create/make me an app" request, including frontend-only, static, single-page, and offline tools with no backend, no database, and no Azure services. WHEN "plan project", "design app", "new project", "project requirements", "create project plan", "plan my app", "what should I build", "new Azure app", "create testable app", "new API project", "full-stack Azure app", "bootstrap project", "new fullstack project", "create functions project", "build me an app", "make me a web app", "create a website", "simple web app", "little frontend tool", "static site", "frontend only app", "no backend app", "React app", "Vue app", "dashboard app", "Express API", "background worker", "multi-service app". -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] --- diff --git a/resources/agents/azure-project-scaffold.agent.md b/resources/agents/azure-project-scaffold.agent.md index d23dd4264..d9e4349ac 100644 --- a/resources/agents/azure-project-scaffold.agent.md +++ b/resources/agents/azure-project-scaffold.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-scaffold description: Scaffold a NEW Azure-centric project from an ALREADY-APPROVED `.azure/project-plan.md` — generate the frontend, backend services, database, and API routes. Does NOT gather requirements or write the plan; that is the `azure-project-plan` agent's job. -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] --- @@ -87,10 +87,10 @@ Keep concise + factual: paths + commands checklist, not prose. 1. **Still run Step A** — first read `.azure/project-plan.md`; autopilot never skips approved-plan read. 2. **No approval gate** — upstream already produced + approved plan; scaffold immediately. This agent has no plan preview or approval step in any mode. -3. **Still run Step C** — after scaffolding, write `.azure/integration-plan.md` + hand off unattended. **Skip frontend preview approval gate** (`open_frontend_preview_view`); autopilot auto-approves UI, so hand off directly. Keep integrate agent in autopilot by calling `start_project_integrate` with marker: +3. **Still run Step C** - after scaffolding, write `.azure/integration-plan.md` + hand off unattended. **Skip frontend preview approval gate** (`open_frontend_preview_view`); autopilot auto-approves UI, so hand off directly. Call `start_project_integrate` without arguments. The extension carries the active Autopilot mode and query marker into the new chat: ```json -{ "prompt": "[AUTOPILOT MODE] The project has been scaffolded. Read `.azure/integration-plan.md`, then create the SQL/PostgreSQL migrations (no seed data), smoke-test the backend, wire the frontend to live data, and verify the app end-to-end." } +{} ``` All scaffold quality work (frontend preview verification, backend services, `.azure/.preview-temp/` cleanup at Step 11) remains. Autopilot suppresses **gates and questions**, never quality. From 97a84096ce7f379689845953c7fc6b35fd05ed21 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:58:24 -0700 Subject: [PATCH 05/34] Narrow agent tool permissions --- evals/agent-assets.lock.json | 16 +++--- evals/check-agent-drift.ts | 50 ++++++++++++------- .../agents/azure-debug-generate.agent.md | 2 +- resources/agents/azure-debug-plan.agent.md | 2 +- resources/agents/azure-deploy.agent.md | 2 +- .../agents/azure-project-integrate.agent.md | 2 +- resources/agents/azure-project-plan.agent.md | 2 +- .../agents/azure-project-scaffold.agent.md | 2 +- 8 files changed, 46 insertions(+), 32 deletions(-) diff --git a/evals/agent-assets.lock.json b/evals/agent-assets.lock.json index 8cd1c5244..d68bc1f3e 100644 --- a/evals/agent-assets.lock.json +++ b/evals/agent-assets.lock.json @@ -1,9 +1,9 @@ { - "agentAssetsHash": "8b54190c691ca14832ccbfb1de1abc54f1f4e274255e58b856bd1809c3827629", + "agentAssetsHash": "931b9948a0324997967084c9cb4e78100bb9de2ef7c767980a5a188f3b2f98cf", "scope": "azure-debug-generate.agent.md, azure-debug-generate/**, azure-debug-plan.agent.md, azure-debug-plan/**, azure-deploy.agent.md, azure-deploy/**, azure-project-integrate.agent.md, azure-project-integrate/**, azure-project-plan.agent.md, azure-project-plan/**, azure-project-scaffold.agent.md, azure-project-scaffold/**, shared-references/**", - "updatedAt": "2026-10-05T17:53:38.043Z", + "updatedAt": "2026-10-05T17:56:47.187Z", "files": { - "azure-debug-generate.agent.md": "8ea64b01675963ad33ec79854b2ecdab026d7843fe474d99a497a0d8135bb847", + "azure-debug-generate.agent.md": "1c40d0d37434a892521bd5075d5b582d4a254a2eca827765ec344c22e414320a", "azure-debug-generate/.metadata.json": "c0ccce18678415f23db0073d48cc646f2de446e7821c99c3130553548d958e0c", "azure-debug-generate/instructions.md": "2dbf252f05dc7f523d726500941d4862a64a5579ff530b21400fc8a5294fed29", "azure-debug-generate/references/api-test-collections.md": "9ef4777f4b682b471b5386c97bb02cb08037545eb520d4534f69b784f5cd798b", @@ -25,7 +25,7 @@ "azure-debug-generate/references/runtimes/dotnet.md": "191ce6cb3eee84d4277d746c4ad094750a5dde0003b40ee7a64cbc621af0522f", "azure-debug-generate/references/runtimes/node.md": "7e037c0fdcf48ac440c046df35aa3112a9c356ea404655c3ecbbf862237b30a9", "azure-debug-generate/references/validation.md": "0daa62d9da8fc7d2a7e6a29ced7378720a2fb962cd67e032ce7e74074a6381cd", - "azure-debug-plan.agent.md": "ef70a79e0b0e9199576a1b50e2d138ad3d34b766148190f598afa78cb93fe711", + "azure-debug-plan.agent.md": "0c6be361decaa6a6a64db3a4e75098f31dffebda705ac190050c997c06b2f60d", "azure-debug-plan/.metadata.json": "108c105ec0eaf5d7f0c2268ab2e0b04eb037649dc6668272f5ce42aa6da51de9", "azure-debug-plan/instructions.md": "0e15215047a1703dbbb868a6cd2a58ee1e2dc2bc231fc3f2e81e8056cd3dd9ea", "azure-debug-plan/references/classify.md": "1673b52fb25ff89f0437263ce7082f690c208a0783f140f432bb94a463376efa", @@ -35,7 +35,7 @@ "azure-debug-plan/references/plan-template.md": "fa08df53649226bd749371f08a9f096cd22ac83d1497e9ae90638993c464af0f", "azure-debug-plan/references/project-types.md": "4eca6fbec5cd2220b18ac5bad0ee7034870aa3739c78fb48d4338e2701bde4c6", "azure-debug-plan/references/runtimes.md": "97c9903a22e0580c4c0ff4d27a1e31c8a16e7ce07056f573701bf816bdfe45c3", - "azure-deploy.agent.md": "26d829f5f43707c46852cefd522f6ecb6efb500dc58f99f216007569330d402b", + "azure-deploy.agent.md": "5a9f11a90754c6cc599b8d6dce168d88ab5417af85a6d2b9621283082f296974", "azure-deploy/.metadata.json": "be0709e660b5da6f558d73e464bed2cd39a3382619bc9076796078843d43cb56", "azure-deploy/cor-references/migration-access.md": "2dce754d450740ade47ab1079e1d3046850c54da7d5acc19209825af8f9fdfb0", "azure-deploy/deploy/instructions.md": "30f877417f9d1da592abf903a12dc961e36c4b5063c48bab561ac1c152055abe", @@ -124,20 +124,20 @@ "azure-deploy/scaffold/references/waf-checklist.md": "d0b50ac5df46569034dd16431aa9ad9faa447f4330a4f4e41447042b160a3f0a", "azure-deploy/scaffold/scripts/scaffold-conformance.ps1": "85398b7fcd06e8207e536f558cca68d24ecf6186e54bf5680d7872aa38913ea0", "azure-deploy/scaffold/scripts/scaffold-conformance.sh": "c9961efccdece4c6d2a1c3a832389b359fb21ca0706fd7c33215d4e8bf267664", - "azure-project-integrate.agent.md": "bc767ea55ff727de8e3b8603cd36299b45ada339b63e4fae477643d8ec03e5b7", + "azure-project-integrate.agent.md": "4426f9c7b47ea7c2e974f485441bb612f81d4f7a287d8b05265e8e02da7e2655", "azure-project-integrate/.metadata.json": "ee73bb3c96f66ed4c6f9d37db0716600e5deb3a23f55adcefcf34d6ed8d556c0", "azure-project-integrate/instructions.md": "0937e021c41429e71e8977b9a23bc5586e11de80d22ca0f85312b35643ea2ec4", "azure-project-integrate/references/end-to-end.md": "11bc63351517d530c7abc34a33842a89e25f9bad8951a7df2035b8a3c3a872aa", "azure-project-integrate/references/migrations.md": "4aa872a898ad09d4b22841e609c56350529a894a1e4f06030b0cd89ead0f2b13", "azure-project-integrate/references/smoke-test.md": "7c76b1734d5e82efaa23b6b61865b5d6c51d8cce8295b82046e2ac623ac45fa4", "azure-project-integrate/references/wire-live-data.md": "723e0b8ff0be74577d1619af774e9693ba2d97e0e7507eb95540cbacafa14643", - "azure-project-plan.agent.md": "e77e51a8842f2d8b5f026d30baafff1da5458871aca2a9ff6d664b9ba2985cd4", + "azure-project-plan.agent.md": "62bff64ee3b6b82246e719850f06a435281a2f939031b3d61372434727fa18ba", "azure-project-plan/.metadata.json": "cec03694d63e45c8cb96983bd586790878b5813289885ae94d6cefc42243f37e", "azure-project-plan/instructions.md": "0d68b18de65bbe4fe375329d008373a8c915c6740f1aba5ecfce51980022e0c3", "azure-project-plan/plan.md": "a56f162fca888e1c81ea3f0c419c5f03712df1ebfb039bbcd665d5391572b64d", "azure-project-plan/references/html-preview.md": "2db272c10c4535743366483610dc46fb885d24795298220d0585880671665369", "azure-project-plan/requirements.md": "d53f610177edb81869165abc500cfd19960a887415e12cb77aa4cc0ace4a9b3d", - "azure-project-scaffold.agent.md": "a48132ce938d08e74f9f8ca241381597b7d980fa342992a1466e8791b126bad1", + "azure-project-scaffold.agent.md": "bbad40f36b0a547739410fe9f220594d9b1fd33b0e498c82861039406691ba1a", "azure-project-scaffold/.metadata.json": "c6e76bb8c81c61ad58ce659e57d9ac7526bb680ad3923913db3a14e1333cd1e0", "azure-project-scaffold/instructions.md": "e59d0266cc8c226f8f9b9d7c0985320daed59e33c3a376fe1046a325125f74f6", "azure-project-scaffold/references/frontend-preview-steps.md": "dc60b0dd5a997bd1b38410b8c1e68d6453261a3e23b93de00654c864b49539cf", diff --git a/evals/check-agent-drift.ts b/evals/check-agent-drift.ts index 4de8194de..943b329a9 100644 --- a/evals/check-agent-drift.ts +++ b/evals/check-agent-drift.ts @@ -320,6 +320,30 @@ function describeAssetChanges( const failures: string[] = []; const checked: string[] = []; +// Every phase reads and updates workspace files, runs validation commands, invokes CoR +// hand-off tools, and may need to discover a hand-off tool before calling it. Each alias +// pair names the same capability in Local and Agent Host, so it adds compatibility rather +// than granting another class of access. +const coreAgentTools = [ + "copilot-azure-resources-extension-tools/*", "Copilot Azure Resources Extension Tools/*", + "tool_search", "toolSearch", "execute", "read", "edit", "search", +]; + +const phaseAgentTools: Record = { + // Plan fans out one preview-rendering sub-agent per page. + "azure-project-plan.agent.md": ["agent"], + // Scaffold delegates frontend and backend generation in parallel. + "azure-project-scaffold.agent.md": ["agent"], + // Integrate verifies the live frontend against the running backend. + "azure-project-integrate.agent.md": ["browser"], + // Debug planning and generation operate only on local workspace files and commands. + "azure-debug-plan.agent.md": [], + "azure-debug-generate.agent.md": [], + // Deploy delegates pipeline phases and is the only agent that queries Azure or + // authors and validates infrastructure through the Azure MCP and Bicep servers. + "azure-deploy.agent.md": ["agent", "azure-mcp/search", "Azure MCP/*", "bicep/*", "Bicep/*"], +}; + // Agent frontmatter is the permission boundary for every custom agent. Manifests use a // single-line YAML flow sequence for `tools`, so read its entries without adding a runtime // dependency to the extension's root package. @@ -329,30 +353,20 @@ for (const file of fs.readdirSync(agentsRoot).filter(name => name.endsWith(".age const toolsLine = header ? /^tools:\s*\[([^\r\n]*)\]\s*$/m.exec(header[1]) : undefined; const tools = toolsLine?.[1].split(",").map(tool => tool.trim()).filter(Boolean); - // These tools are shared by every phase. Both names in each alias pair are required - // because supported agent hosts expose the same capability under different identifiers. - const requiredTools = [ - "copilot-azure-resources-extension-tools/*", "Copilot Azure Resources Extension Tools/*", - "tool_search", "toolSearch", "execute", "read", "browser", "edit", "search", "web", "todo", - ]; - - // The debug-plan phase only produces a local debugging plan. Every other phase may - // delegate work and query Azure, so those phases need both forms of the Azure MCP tool. - if (file !== "azure-debug-plan.agent.md") { - requiredTools.push("agent", "azure-mcp/search", "Azure MCP/*"); - } - - // Deployment also authors and validates infrastructure, which requires the Bicep tools. - if (file === "azure-deploy.agent.md") { - requiredTools.push("bicep/*", "Bicep/*"); + const extras = phaseAgentTools[file]; + if (!extras) { + failures.push(`scoped-agent-tools (${file}): add an explicit least-privilege tool policy for this agent`); + continue; } + const requiredTools = [...coreAgentTools, ...extras]; // Require an exact set: reject malformed entries, missing capabilities, and extra tools. - // In particular, broad `vscode` or unrelated MCP access must not bypass phase scoping. + // In particular, broad `vscode`, unused convenience tools, or unrelated MCP access must + // not bypass the least-privilege set derived from each phase's shipped instructions. const allowedTools = new Set(requiredTools); if (!tools || tools.some(tool => !allowedTools.has(tool)) || requiredTools.some(tool => !tools.includes(tool))) { - failures.push(`scoped-agent-tools (${file}): tools must preserve the phase's required core, CoR, and Azure tools without broad vscode or unrelated MCP access`); + failures.push(`scoped-agent-tools (${file}): tools must exactly match the phase's documented least-privilege set`); } else { checked.push(`scoped-agent-tools (${file})`); } diff --git a/resources/agents/azure-debug-generate.agent.md b/resources/agents/azure-debug-generate.agent.md index 01b0d3fdd..5e0fb6c8d 100644 --- a/resources/agents/azure-debug-generate.agent.md +++ b/resources/agents/azure-debug-generate.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-generate description: Generate artifacts for an approved VS Code debugging plan. Responsible for generating docker-compose, VS Code launch/tasks, and emulator orchestration for streamlined project debugging. -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, edit, search] target: vscode --- diff --git a/resources/agents/azure-debug-plan.agent.md b/resources/agents/azure-debug-plan.agent.md index 1c0d28e7d..91fb6ae30 100644 --- a/resources/agents/azure-debug-plan.agent.md +++ b/resources/agents/azure-debug-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-plan description: Scan an Azure-centric workspace project. Classify its services and dependencies, and produce a local debugging plan covering automated emulator startup, VS Code launch/task configs, and API tests. -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, browser, edit, search, web, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, edit, search] target: vscode --- diff --git a/resources/agents/azure-deploy.agent.md b/resources/agents/azure-deploy.agent.md index c0f7c7715..3ac7c23c6 100644 --- a/resources/agents/azure-deploy.agent.md +++ b/resources/agents/azure-deploy.agent.md @@ -1,7 +1,7 @@ --- name: azure-deploy description: "Onboard and deploy an Azure-centric project end-to-end using a guided, self-contained onboarding pipeline. Analyzes deployment readiness, selects Azure services and SKUs, estimates cost, validates quota, generates secure Bicep/Terraform, provisions resources, deploys application code, and verifies health. Run after local development is set up. WHEN: deploy to Azure, ship to Azure, host on Azure, create infrastructure, generate IaC, provision resources, go live." -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, bicep/*, Bicep/*, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, edit, search, azure-mcp/search, Azure MCP/*, bicep/*, Bicep/*] --- diff --git a/resources/agents/azure-project-integrate.agent.md b/resources/agents/azure-project-integrate.agent.md index 80b3b6554..56eea2c36 100644 --- a/resources/agents/azure-project-integrate.agent.md +++ b/resources/agents/azure-project-integrate.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-integrate description: Integrate a freshly scaffolded Azure-centric project — create the SQL/PostgreSQL schema migrations (NO seed data), smoke-test the backend so every endpoint responds, wire the frontend to LIVE backend data (replace all mock data), and run the frontend and backend wired together end-to-end. Runs after `azure-project-scaffold`. WHEN "integrate project", "wire to live data", "remove mock data", "smoke test backend", "verify endpoints", "create migrations", "wire frontend and backend", "integrate scaffold", "make the app run". -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, browser, edit, search] --- diff --git a/resources/agents/azure-project-plan.agent.md b/resources/agents/azure-project-plan.agent.md index 45f711833..7fd9cf2af 100644 --- a/resources/agents/azure-project-plan.agent.md +++ b/resources/agents/azure-project-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-plan description: Plan and design a NEW Azure-centric project the user asks to build — gather requirements interactively, produce an approved `.azure/project-plan.md`, then hand off to the `azure-project-scaffold` agent for execution. Use this for EVERY "build/create/make me an app" request, including frontend-only, static, single-page, and offline tools with no backend, no database, and no Azure services. WHEN "plan project", "design app", "new project", "project requirements", "create project plan", "plan my app", "what should I build", "new Azure app", "create testable app", "new API project", "full-stack Azure app", "bootstrap project", "new fullstack project", "create functions project", "build me an app", "make me a web app", "create a website", "simple web app", "little frontend tool", "static site", "frontend only app", "no backend app", "React app", "Vue app", "dashboard app", "Express API", "background worker", "multi-service app". -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, edit, search] --- diff --git a/resources/agents/azure-project-scaffold.agent.md b/resources/agents/azure-project-scaffold.agent.md index d9e4349ac..f10ad2d10 100644 --- a/resources/agents/azure-project-scaffold.agent.md +++ b/resources/agents/azure-project-scaffold.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-scaffold description: Scaffold a NEW Azure-centric project from an ALREADY-APPROVED `.azure/project-plan.md` — generate the frontend, backend services, database, and API routes. Does NOT gather requirements or write the plan; that is the `azure-project-plan` agent's job. -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] +tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, edit, search] --- From dbd48e424aebc851fe421fe4bc539acf07cb9f6c Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 10:59:18 -0700 Subject: [PATCH 06/34] Limit agent tools to Harness aliases --- evals/agent-assets.lock.json | 16 +++--- evals/check-agent-drift.ts | 52 ------------------- .../agents/azure-debug-generate.agent.md | 2 +- resources/agents/azure-debug-plan.agent.md | 2 +- resources/agents/azure-deploy.agent.md | 2 +- .../agents/azure-project-integrate.agent.md | 2 +- resources/agents/azure-project-plan.agent.md | 2 +- .../agents/azure-project-scaffold.agent.md | 2 +- 8 files changed, 14 insertions(+), 66 deletions(-) diff --git a/evals/agent-assets.lock.json b/evals/agent-assets.lock.json index d68bc1f3e..2ee14ad8a 100644 --- a/evals/agent-assets.lock.json +++ b/evals/agent-assets.lock.json @@ -1,9 +1,9 @@ { - "agentAssetsHash": "931b9948a0324997967084c9cb4e78100bb9de2ef7c767980a5a188f3b2f98cf", + "agentAssetsHash": "a4c41978514da4780b837099eb43d64def2e804a8f067b0cf52f298642fea0c2", "scope": "azure-debug-generate.agent.md, azure-debug-generate/**, azure-debug-plan.agent.md, azure-debug-plan/**, azure-deploy.agent.md, azure-deploy/**, azure-project-integrate.agent.md, azure-project-integrate/**, azure-project-plan.agent.md, azure-project-plan/**, azure-project-scaffold.agent.md, azure-project-scaffold/**, shared-references/**", - "updatedAt": "2026-10-05T17:56:47.187Z", + "updatedAt": "2026-10-05T17:59:09.046Z", "files": { - "azure-debug-generate.agent.md": "1c40d0d37434a892521bd5075d5b582d4a254a2eca827765ec344c22e414320a", + "azure-debug-generate.agent.md": "1db8efeaa09757d446a3d74068a460ce98a46b0387226593f19e24e4c94cb244", "azure-debug-generate/.metadata.json": "c0ccce18678415f23db0073d48cc646f2de446e7821c99c3130553548d958e0c", "azure-debug-generate/instructions.md": "2dbf252f05dc7f523d726500941d4862a64a5579ff530b21400fc8a5294fed29", "azure-debug-generate/references/api-test-collections.md": "9ef4777f4b682b471b5386c97bb02cb08037545eb520d4534f69b784f5cd798b", @@ -25,7 +25,7 @@ "azure-debug-generate/references/runtimes/dotnet.md": "191ce6cb3eee84d4277d746c4ad094750a5dde0003b40ee7a64cbc621af0522f", "azure-debug-generate/references/runtimes/node.md": "7e037c0fdcf48ac440c046df35aa3112a9c356ea404655c3ecbbf862237b30a9", "azure-debug-generate/references/validation.md": "0daa62d9da8fc7d2a7e6a29ced7378720a2fb962cd67e032ce7e74074a6381cd", - "azure-debug-plan.agent.md": "0c6be361decaa6a6a64db3a4e75098f31dffebda705ac190050c997c06b2f60d", + "azure-debug-plan.agent.md": "150dd1c1a1ca4e9d0bae38b802050c9fbcf5624f0340ed70da361f7196113239", "azure-debug-plan/.metadata.json": "108c105ec0eaf5d7f0c2268ab2e0b04eb037649dc6668272f5ce42aa6da51de9", "azure-debug-plan/instructions.md": "0e15215047a1703dbbb868a6cd2a58ee1e2dc2bc231fc3f2e81e8056cd3dd9ea", "azure-debug-plan/references/classify.md": "1673b52fb25ff89f0437263ce7082f690c208a0783f140f432bb94a463376efa", @@ -35,7 +35,7 @@ "azure-debug-plan/references/plan-template.md": "fa08df53649226bd749371f08a9f096cd22ac83d1497e9ae90638993c464af0f", "azure-debug-plan/references/project-types.md": "4eca6fbec5cd2220b18ac5bad0ee7034870aa3739c78fb48d4338e2701bde4c6", "azure-debug-plan/references/runtimes.md": "97c9903a22e0580c4c0ff4d27a1e31c8a16e7ce07056f573701bf816bdfe45c3", - "azure-deploy.agent.md": "5a9f11a90754c6cc599b8d6dce168d88ab5417af85a6d2b9621283082f296974", + "azure-deploy.agent.md": "c20f791f9f366731079ed5c2d24263c149cff2fbed8362dd6313b6d8a89b96b4", "azure-deploy/.metadata.json": "be0709e660b5da6f558d73e464bed2cd39a3382619bc9076796078843d43cb56", "azure-deploy/cor-references/migration-access.md": "2dce754d450740ade47ab1079e1d3046850c54da7d5acc19209825af8f9fdfb0", "azure-deploy/deploy/instructions.md": "30f877417f9d1da592abf903a12dc961e36c4b5063c48bab561ac1c152055abe", @@ -124,20 +124,20 @@ "azure-deploy/scaffold/references/waf-checklist.md": "d0b50ac5df46569034dd16431aa9ad9faa447f4330a4f4e41447042b160a3f0a", "azure-deploy/scaffold/scripts/scaffold-conformance.ps1": "85398b7fcd06e8207e536f558cca68d24ecf6186e54bf5680d7872aa38913ea0", "azure-deploy/scaffold/scripts/scaffold-conformance.sh": "c9961efccdece4c6d2a1c3a832389b359fb21ca0706fd7c33215d4e8bf267664", - "azure-project-integrate.agent.md": "4426f9c7b47ea7c2e974f485441bb612f81d4f7a287d8b05265e8e02da7e2655", + "azure-project-integrate.agent.md": "2d6dc6443170c44d9f628288a4643db901186aca05d7ec2a3938bfe7df4014e2", "azure-project-integrate/.metadata.json": "ee73bb3c96f66ed4c6f9d37db0716600e5deb3a23f55adcefcf34d6ed8d556c0", "azure-project-integrate/instructions.md": "0937e021c41429e71e8977b9a23bc5586e11de80d22ca0f85312b35643ea2ec4", "azure-project-integrate/references/end-to-end.md": "11bc63351517d530c7abc34a33842a89e25f9bad8951a7df2035b8a3c3a872aa", "azure-project-integrate/references/migrations.md": "4aa872a898ad09d4b22841e609c56350529a894a1e4f06030b0cd89ead0f2b13", "azure-project-integrate/references/smoke-test.md": "7c76b1734d5e82efaa23b6b61865b5d6c51d8cce8295b82046e2ac623ac45fa4", "azure-project-integrate/references/wire-live-data.md": "723e0b8ff0be74577d1619af774e9693ba2d97e0e7507eb95540cbacafa14643", - "azure-project-plan.agent.md": "62bff64ee3b6b82246e719850f06a435281a2f939031b3d61372434727fa18ba", + "azure-project-plan.agent.md": "2b8989ee3bbc23ead45c39f61bd404993c2c83145aed8b9f08cff69229af603c", "azure-project-plan/.metadata.json": "cec03694d63e45c8cb96983bd586790878b5813289885ae94d6cefc42243f37e", "azure-project-plan/instructions.md": "0d68b18de65bbe4fe375329d008373a8c915c6740f1aba5ecfce51980022e0c3", "azure-project-plan/plan.md": "a56f162fca888e1c81ea3f0c419c5f03712df1ebfb039bbcd665d5391572b64d", "azure-project-plan/references/html-preview.md": "2db272c10c4535743366483610dc46fb885d24795298220d0585880671665369", "azure-project-plan/requirements.md": "d53f610177edb81869165abc500cfd19960a887415e12cb77aa4cc0ace4a9b3d", - "azure-project-scaffold.agent.md": "bbad40f36b0a547739410fe9f220594d9b1fd33b0e498c82861039406691ba1a", + "azure-project-scaffold.agent.md": "66fee1bc735664c9b51c2d300708fe5d516bc81b3886b802170a8685eca5abd9", "azure-project-scaffold/.metadata.json": "c6e76bb8c81c61ad58ce659e57d9ac7526bb680ad3923913db3a14e1333cd1e0", "azure-project-scaffold/instructions.md": "e59d0266cc8c226f8f9b9d7c0985320daed59e33c3a376fe1046a325125f74f6", "azure-project-scaffold/references/frontend-preview-steps.md": "dc60b0dd5a997bd1b38410b8c1e68d6453261a3e23b93de00654c864b49539cf", diff --git a/evals/check-agent-drift.ts b/evals/check-agent-drift.ts index 943b329a9..6d3ba353b 100644 --- a/evals/check-agent-drift.ts +++ b/evals/check-agent-drift.ts @@ -320,58 +320,6 @@ function describeAssetChanges( const failures: string[] = []; const checked: string[] = []; -// Every phase reads and updates workspace files, runs validation commands, invokes CoR -// hand-off tools, and may need to discover a hand-off tool before calling it. Each alias -// pair names the same capability in Local and Agent Host, so it adds compatibility rather -// than granting another class of access. -const coreAgentTools = [ - "copilot-azure-resources-extension-tools/*", "Copilot Azure Resources Extension Tools/*", - "tool_search", "toolSearch", "execute", "read", "edit", "search", -]; - -const phaseAgentTools: Record = { - // Plan fans out one preview-rendering sub-agent per page. - "azure-project-plan.agent.md": ["agent"], - // Scaffold delegates frontend and backend generation in parallel. - "azure-project-scaffold.agent.md": ["agent"], - // Integrate verifies the live frontend against the running backend. - "azure-project-integrate.agent.md": ["browser"], - // Debug planning and generation operate only on local workspace files and commands. - "azure-debug-plan.agent.md": [], - "azure-debug-generate.agent.md": [], - // Deploy delegates pipeline phases and is the only agent that queries Azure or - // authors and validates infrastructure through the Azure MCP and Bicep servers. - "azure-deploy.agent.md": ["agent", "azure-mcp/search", "Azure MCP/*", "bicep/*", "Bicep/*"], -}; - -// Agent frontmatter is the permission boundary for every custom agent. Manifests use a -// single-line YAML flow sequence for `tools`, so read its entries without adding a runtime -// dependency to the extension's root package. -for (const file of fs.readdirSync(agentsRoot).filter(name => name.endsWith(".agent.md"))) { - const body = fs.readFileSync(path.join(agentsRoot, file), "utf8"); - const header = /^---\r?\n([\s\S]*?)\r?\n---/.exec(body); - const toolsLine = header ? /^tools:\s*\[([^\r\n]*)\]\s*$/m.exec(header[1]) : undefined; - const tools = toolsLine?.[1].split(",").map(tool => tool.trim()).filter(Boolean); - - const extras = phaseAgentTools[file]; - if (!extras) { - failures.push(`scoped-agent-tools (${file}): add an explicit least-privilege tool policy for this agent`); - continue; - } - const requiredTools = [...coreAgentTools, ...extras]; - - // Require an exact set: reject malformed entries, missing capabilities, and extra tools. - // In particular, broad `vscode`, unused convenience tools, or unrelated MCP access must - // not bypass the least-privilege set derived from each phase's shipped instructions. - const allowedTools = new Set(requiredTools); - if (!tools || tools.some(tool => !allowedTools.has(tool)) - || requiredTools.some(tool => !tools.includes(tool))) { - failures.push(`scoped-agent-tools (${file}): tools must exactly match the phase's documented least-privilege set`); - } else { - checked.push(`scoped-agent-tools (${file})`); - } -} - for (const contract of contracts) { const filePath = path.join(agentsRoot, contract.file); if (!fs.existsSync(filePath)) { diff --git a/resources/agents/azure-debug-generate.agent.md b/resources/agents/azure-debug-generate.agent.md index 5e0fb6c8d..20f163b36 100644 --- a/resources/agents/azure-debug-generate.agent.md +++ b/resources/agents/azure-debug-generate.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-generate description: Generate artifacts for an approved VS Code debugging plan. Responsible for generating docker-compose, VS Code launch/tasks, and emulator orchestration for streamlined project debugging. -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, edit, search] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] target: vscode --- diff --git a/resources/agents/azure-debug-plan.agent.md b/resources/agents/azure-debug-plan.agent.md index 91fb6ae30..78eea2845 100644 --- a/resources/agents/azure-debug-plan.agent.md +++ b/resources/agents/azure-debug-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-plan description: Scan an Azure-centric workspace project. Classify its services and dependencies, and produce a local debugging plan covering automated emulator startup, VS Code launch/task configs, and API tests. -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, edit, search] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, browser, edit, search, web, todo] target: vscode --- diff --git a/resources/agents/azure-deploy.agent.md b/resources/agents/azure-deploy.agent.md index 3ac7c23c6..e00ff90d0 100644 --- a/resources/agents/azure-deploy.agent.md +++ b/resources/agents/azure-deploy.agent.md @@ -1,7 +1,7 @@ --- name: azure-deploy description: "Onboard and deploy an Azure-centric project end-to-end using a guided, self-contained onboarding pipeline. Analyzes deployment readiness, selects Azure services and SKUs, estimates cost, validates quota, generates secure Bicep/Terraform, provisions resources, deploys application code, and verifies health. Run after local development is set up. WHEN: deploy to Azure, ship to Azure, host on Azure, create infrastructure, generate IaC, provision resources, go live." -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, edit, search, azure-mcp/search, Azure MCP/*, bicep/*, Bicep/*] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- diff --git a/resources/agents/azure-project-integrate.agent.md b/resources/agents/azure-project-integrate.agent.md index 56eea2c36..c7946e6c3 100644 --- a/resources/agents/azure-project-integrate.agent.md +++ b/resources/agents/azure-project-integrate.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-integrate description: Integrate a freshly scaffolded Azure-centric project — create the SQL/PostgreSQL schema migrations (NO seed data), smoke-test the backend so every endpoint responds, wire the frontend to LIVE backend data (replace all mock data), and run the frontend and backend wired together end-to-end. Runs after `azure-project-scaffold`. WHEN "integrate project", "wire to live data", "remove mock data", "smoke test backend", "verify endpoints", "create migrations", "wire frontend and backend", "integrate scaffold", "make the app run". -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, browser, edit, search] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- diff --git a/resources/agents/azure-project-plan.agent.md b/resources/agents/azure-project-plan.agent.md index 7fd9cf2af..a95b7e818 100644 --- a/resources/agents/azure-project-plan.agent.md +++ b/resources/agents/azure-project-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-plan description: Plan and design a NEW Azure-centric project the user asks to build — gather requirements interactively, produce an approved `.azure/project-plan.md`, then hand off to the `azure-project-scaffold` agent for execution. Use this for EVERY "build/create/make me an app" request, including frontend-only, static, single-page, and offline tools with no backend, no database, and no Azure services. WHEN "plan project", "design app", "new project", "project requirements", "create project plan", "plan my app", "what should I build", "new Azure app", "create testable app", "new API project", "full-stack Azure app", "bootstrap project", "new fullstack project", "create functions project", "build me an app", "make me a web app", "create a website", "simple web app", "little frontend tool", "static site", "frontend only app", "no backend app", "React app", "Vue app", "dashboard app", "Express API", "background worker", "multi-service app". -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, edit, search] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- diff --git a/resources/agents/azure-project-scaffold.agent.md b/resources/agents/azure-project-scaffold.agent.md index f10ad2d10..0352a07ab 100644 --- a/resources/agents/azure-project-scaffold.agent.md +++ b/resources/agents/azure-project-scaffold.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-scaffold description: Scaffold a NEW Azure-centric project from an ALREADY-APPROVED `.azure/project-plan.md` — generate the frontend, backend services, database, and API routes. Does NOT gather requirements or write the plan; that is the `azure-project-plan` agent's job. -tools: [copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, edit, search] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] --- From b9d099805bdc4688969085ef1484d4da8d9a5282 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:03:38 -0700 Subject: [PATCH 07/34] Add Copilot Harness Azure MCP aliases --- evals/agent-assets.lock.json | 14 +++++++------- resources/agents/azure-debug-generate.agent.md | 2 +- resources/agents/azure-deploy.agent.md | 2 +- resources/agents/azure-project-integrate.agent.md | 2 +- resources/agents/azure-project-plan.agent.md | 2 +- resources/agents/azure-project-scaffold.agent.md | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/evals/agent-assets.lock.json b/evals/agent-assets.lock.json index 2ee14ad8a..078b995d3 100644 --- a/evals/agent-assets.lock.json +++ b/evals/agent-assets.lock.json @@ -1,9 +1,9 @@ { - "agentAssetsHash": "a4c41978514da4780b837099eb43d64def2e804a8f067b0cf52f298642fea0c2", + "agentAssetsHash": "1f75802a0c28b60ebb7b5d7f22a485353991869aea6d58e2e939fbcb0471d3ca", "scope": "azure-debug-generate.agent.md, azure-debug-generate/**, azure-debug-plan.agent.md, azure-debug-plan/**, azure-deploy.agent.md, azure-deploy/**, azure-project-integrate.agent.md, azure-project-integrate/**, azure-project-plan.agent.md, azure-project-plan/**, azure-project-scaffold.agent.md, azure-project-scaffold/**, shared-references/**", - "updatedAt": "2026-10-05T17:59:09.046Z", + "updatedAt": "2026-10-05T18:03:28.672Z", "files": { - "azure-debug-generate.agent.md": "1db8efeaa09757d446a3d74068a460ce98a46b0387226593f19e24e4c94cb244", + "azure-debug-generate.agent.md": "f0d7ab870df4619ded4252380198d197eaf03aca8d003694cf473f7a8b580d76", "azure-debug-generate/.metadata.json": "c0ccce18678415f23db0073d48cc646f2de446e7821c99c3130553548d958e0c", "azure-debug-generate/instructions.md": "2dbf252f05dc7f523d726500941d4862a64a5579ff530b21400fc8a5294fed29", "azure-debug-generate/references/api-test-collections.md": "9ef4777f4b682b471b5386c97bb02cb08037545eb520d4534f69b784f5cd798b", @@ -35,7 +35,7 @@ "azure-debug-plan/references/plan-template.md": "fa08df53649226bd749371f08a9f096cd22ac83d1497e9ae90638993c464af0f", "azure-debug-plan/references/project-types.md": "4eca6fbec5cd2220b18ac5bad0ee7034870aa3739c78fb48d4338e2701bde4c6", "azure-debug-plan/references/runtimes.md": "97c9903a22e0580c4c0ff4d27a1e31c8a16e7ce07056f573701bf816bdfe45c3", - "azure-deploy.agent.md": "c20f791f9f366731079ed5c2d24263c149cff2fbed8362dd6313b6d8a89b96b4", + "azure-deploy.agent.md": "a1da0183a9b4c07c5d7e1182cd435a02e19088d0f75b9dd1d06a9f5b3821849a", "azure-deploy/.metadata.json": "be0709e660b5da6f558d73e464bed2cd39a3382619bc9076796078843d43cb56", "azure-deploy/cor-references/migration-access.md": "2dce754d450740ade47ab1079e1d3046850c54da7d5acc19209825af8f9fdfb0", "azure-deploy/deploy/instructions.md": "30f877417f9d1da592abf903a12dc961e36c4b5063c48bab561ac1c152055abe", @@ -124,20 +124,20 @@ "azure-deploy/scaffold/references/waf-checklist.md": "d0b50ac5df46569034dd16431aa9ad9faa447f4330a4f4e41447042b160a3f0a", "azure-deploy/scaffold/scripts/scaffold-conformance.ps1": "85398b7fcd06e8207e536f558cca68d24ecf6186e54bf5680d7872aa38913ea0", "azure-deploy/scaffold/scripts/scaffold-conformance.sh": "c9961efccdece4c6d2a1c3a832389b359fb21ca0706fd7c33215d4e8bf267664", - "azure-project-integrate.agent.md": "2d6dc6443170c44d9f628288a4643db901186aca05d7ec2a3938bfe7df4014e2", + "azure-project-integrate.agent.md": "4ace6f03550113856273d82d7624b32272247431fc363bb0e12c1376238e8697", "azure-project-integrate/.metadata.json": "ee73bb3c96f66ed4c6f9d37db0716600e5deb3a23f55adcefcf34d6ed8d556c0", "azure-project-integrate/instructions.md": "0937e021c41429e71e8977b9a23bc5586e11de80d22ca0f85312b35643ea2ec4", "azure-project-integrate/references/end-to-end.md": "11bc63351517d530c7abc34a33842a89e25f9bad8951a7df2035b8a3c3a872aa", "azure-project-integrate/references/migrations.md": "4aa872a898ad09d4b22841e609c56350529a894a1e4f06030b0cd89ead0f2b13", "azure-project-integrate/references/smoke-test.md": "7c76b1734d5e82efaa23b6b61865b5d6c51d8cce8295b82046e2ac623ac45fa4", "azure-project-integrate/references/wire-live-data.md": "723e0b8ff0be74577d1619af774e9693ba2d97e0e7507eb95540cbacafa14643", - "azure-project-plan.agent.md": "2b8989ee3bbc23ead45c39f61bd404993c2c83145aed8b9f08cff69229af603c", + "azure-project-plan.agent.md": "aec64bb9d77ad872c5e44ff78400cee537db4e30990d3e52e43c412ab8ff74a4", "azure-project-plan/.metadata.json": "cec03694d63e45c8cb96983bd586790878b5813289885ae94d6cefc42243f37e", "azure-project-plan/instructions.md": "0d68b18de65bbe4fe375329d008373a8c915c6740f1aba5ecfce51980022e0c3", "azure-project-plan/plan.md": "a56f162fca888e1c81ea3f0c419c5f03712df1ebfb039bbcd665d5391572b64d", "azure-project-plan/references/html-preview.md": "2db272c10c4535743366483610dc46fb885d24795298220d0585880671665369", "azure-project-plan/requirements.md": "d53f610177edb81869165abc500cfd19960a887415e12cb77aa4cc0ace4a9b3d", - "azure-project-scaffold.agent.md": "66fee1bc735664c9b51c2d300708fe5d516bc81b3886b802170a8685eca5abd9", + "azure-project-scaffold.agent.md": "1fd514686d4c9c30e85947acfefc77c9e4dc7537cd1a5ebd4535c77e8a0126ab", "azure-project-scaffold/.metadata.json": "c6e76bb8c81c61ad58ce659e57d9ac7526bb680ad3923913db3a14e1333cd1e0", "azure-project-scaffold/instructions.md": "e59d0266cc8c226f8f9b9d7c0985320daed59e33c3a376fe1046a325125f74f6", "azure-project-scaffold/references/frontend-preview-steps.md": "dc60b0dd5a997bd1b38410b8c1e68d6453261a3e23b93de00654c864b49539cf", diff --git a/resources/agents/azure-debug-generate.agent.md b/resources/agents/azure-debug-generate.agent.md index 20f163b36..24c030e08 100644 --- a/resources/agents/azure-debug-generate.agent.md +++ b/resources/agents/azure-debug-generate.agent.md @@ -1,7 +1,7 @@ --- name: azure-debug-generate description: Generate artifacts for an approved VS Code debugging plan. Responsible for generating docker-compose, VS Code launch/tasks, and emulator orchestration for streamlined project debugging. -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] target: vscode --- diff --git a/resources/agents/azure-deploy.agent.md b/resources/agents/azure-deploy.agent.md index e00ff90d0..280298293 100644 --- a/resources/agents/azure-deploy.agent.md +++ b/resources/agents/azure-deploy.agent.md @@ -1,7 +1,7 @@ --- name: azure-deploy description: "Onboard and deploy an Azure-centric project end-to-end using a guided, self-contained onboarding pipeline. Analyzes deployment readiness, selects Azure services and SKUs, estimates cost, validates quota, generates secure Bicep/Terraform, provisions resources, deploys application code, and verifies health. Run after local development is set up. WHEN: deploy to Azure, ship to Azure, host on Azure, create infrastructure, generate IaC, provision resources, go live." -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] --- diff --git a/resources/agents/azure-project-integrate.agent.md b/resources/agents/azure-project-integrate.agent.md index c7946e6c3..0bd1b77ac 100644 --- a/resources/agents/azure-project-integrate.agent.md +++ b/resources/agents/azure-project-integrate.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-integrate description: Integrate a freshly scaffolded Azure-centric project — create the SQL/PostgreSQL schema migrations (NO seed data), smoke-test the backend so every endpoint responds, wire the frontend to LIVE backend data (replace all mock data), and run the frontend and backend wired together end-to-end. Runs after `azure-project-scaffold`. WHEN "integrate project", "wire to live data", "remove mock data", "smoke test backend", "verify endpoints", "create migrations", "wire frontend and backend", "integrate scaffold", "make the app run". -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] --- diff --git a/resources/agents/azure-project-plan.agent.md b/resources/agents/azure-project-plan.agent.md index a95b7e818..4eb0b1a75 100644 --- a/resources/agents/azure-project-plan.agent.md +++ b/resources/agents/azure-project-plan.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-plan description: Plan and design a NEW Azure-centric project the user asks to build — gather requirements interactively, produce an approved `.azure/project-plan.md`, then hand off to the `azure-project-scaffold` agent for execution. Use this for EVERY "build/create/make me an app" request, including frontend-only, static, single-page, and offline tools with no backend, no database, and no Azure services. WHEN "plan project", "design app", "new project", "project requirements", "create project plan", "plan my app", "what should I build", "new Azure app", "create testable app", "new API project", "full-stack Azure app", "bootstrap project", "new fullstack project", "create functions project", "build me an app", "make me a web app", "create a website", "simple web app", "little frontend tool", "static site", "frontend only app", "no backend app", "React app", "Vue app", "dashboard app", "Express API", "background worker", "multi-service app". -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] --- diff --git a/resources/agents/azure-project-scaffold.agent.md b/resources/agents/azure-project-scaffold.agent.md index 0352a07ab..21fe290b7 100644 --- a/resources/agents/azure-project-scaffold.agent.md +++ b/resources/agents/azure-project-scaffold.agent.md @@ -1,7 +1,7 @@ --- name: azure-project-scaffold description: Scaffold a NEW Azure-centric project from an ALREADY-APPROVED `.azure/project-plan.md` — generate the frontend, backend services, database, and API routes. Does NOT gather requirements or write the plan; that is the `azure-project-plan` agent's job. -tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, todo] +tools: [vscode, copilot-azure-resources-extension-tools/*, Copilot Azure Resources Extension Tools/*, tool_search, toolSearch, execute, read, agent, browser, edit, search, web, azure-mcp/search, Azure MCP/*, todo] --- From 52a2d11c8e7d83d6a2b8dcf36892519ba146d53e Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:50:33 -0700 Subject: [PATCH 08/34] Ensure chat ready before prompt shows up --- docs/copilot-create-project.md | 3 +++ .../extension/createProjectWithCopilot.ts | 25 ++++++++++++++++--- test/copilotOnRails/modelSelection.test.ts | 20 +++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 772c6f909..5e3c3d0b4 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -141,6 +141,9 @@ session** running the next agent. Between hand‑offs, agents open **webviews** - A Copilot plan with access to at least one supported model. The model picker lists the Opus, Sonnet, GPT Sol, and GPT Terra models currently available through GitHub Copilot, so newly available versions appear without an extension update. The lowest-version available Opus model is selected by default. + The extension activates GitHub Copilot Chat before loading this list. If VS Code reports no supported + models, the extension asks the user to check their Copilot sign-in, plan, and organization model policies + instead of opening the prompt page with an empty picker. When the Copilot Harness is active, the extension maps that selection only to the matching `agent-host-copilotcli` model. It does not pass Local or legacy `copilotcli` identifiers, which VS Code rejects as foreign to the Agent Host session. The extension constructs the Agent Host diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 71cfd576e..411e0cd49 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -5,6 +5,7 @@ import { AzExtFsExtra, UserCancelledError } from "@microsoft/vscode-azext-utils"; import * as vscode from 'vscode'; +import { ensureCopilotChatReady } from "../../../commands/copilotOnRails/openChatWithAgent"; import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/registerCopilotOnRailsCommands"; import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; @@ -21,7 +22,7 @@ export const OPEN_PROJECT_FOLDER_OPTIONS = { forceNewWindow: true } as const; export const PROJECT_FOLDER_SELECTION_TELEMETRY_KEY = 'projectFolderSelection'; export type ProjectFolderSelection = 'newSubfolder' | 'selectedEmptyFolder'; -export async function createProjectWithCopilot(context: CopilotOnRailsContext): Promise { +export async function createProjectWithCopilot(context: CopilotOnRailsContext, initialPrompt?: string, initialModel?: string): Promise { if (!(await ensureFreshWorkspace(context))) { return; } @@ -58,20 +59,36 @@ export async function createProjectWithCopilot(context: CopilotOnRailsContext): } // Nothing detected => start from scratch. - await openCreateProjectView(); + await openCreateProjectView(context, initialPrompt, initialModel); } /** Re-opens the create view pre-filled after a reload-to-discover-agents; no-ops when nothing was stashed. */ export async function resumeCreateProjectViewAfterReload(): Promise { const resume = await consumeReloadResumePrompt(); if (resume) { - await openCreateProjectView(resume.prompt, resume.model); + await vscode.commands.executeCommand( + copilotOnRailsCommandIds.createProjectWithCopilot, + resume.prompt, + resume.model, + ); } } -async function openCreateProjectView(initialPrompt?: string, initialModel?: string): Promise { +async function openCreateProjectView(context: CopilotOnRailsContext, initialPrompt?: string, initialModel?: string): Promise { + if (!(await ensureCopilotChatReady(context))) { + return; + } + const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilot' }); const modelOptions = getSupportedModelOptions(availableModels); + setCorProp(context, 'availableModelCount', modelOptions.length); + if (modelOptions.length === 0) { + void vscode.window.showErrorMessage(vscode.l10n.t( + 'No supported GitHub Copilot models are available. Check that you are signed in to GitHub Copilot and that your plan and organization policies allow a supported model, then try again.', + )); + return; + } + const selectedModel = initialModel && modelOptions.includes(initialModel) ? initialModel : getDefaultOpusModelOption(availableModels); diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts index d0422dd38..003a5e447 100644 --- a/test/copilotOnRails/modelSelection.test.ts +++ b/test/copilotOnRails/modelSelection.test.ts @@ -6,6 +6,7 @@ import assert from 'assert'; import { AvailableChatModel, + getSupportedModelOptions, resolveAvailableChatModel, resolveCopilotHarnessModel, } from '../../src/utils/copilotOnRails/modelSelection'; @@ -28,7 +29,26 @@ const legacyHarnessModel: AvailableChatModel = { vendor: 'copilotcli', }; +const opusModel: AvailableChatModel = { + id: 'claude-opus-5.5', + vendor: 'copilot', + family: 'claude-opus-5.5', + version: '5.5', + name: 'Claude Opus 5.5', +}; + suite('Copilot on Rails model selection', () => { + test('includes available Opus models in the picker', () => { + assert.deepStrictEqual( + getSupportedModelOptions([localModel, opusModel]), + ['Claude Opus 5.5 (copilot)', 'GPT-6.1 Sol (copilot)'], + ); + }); + + test('returns no picker options when VS Code reports no models', () => { + assert.deepStrictEqual(getSupportedModelOptions([]), []); + }); + test('resolves a qualified local model name', () => { assert.strictEqual( resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, harnessModel]), From eafdc781fe1bef1a03663b102d32a6bc4e62303c Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:23:00 -0700 Subject: [PATCH 09/34] Simplify Copilot model filtering --- src/utils/copilotOnRails/modelSelection.ts | 8 ++++---- test/copilotOnRails/modelSelection.test.ts | 20 -------------------- 2 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/utils/copilotOnRails/modelSelection.ts b/src/utils/copilotOnRails/modelSelection.ts index ce0830a3e..6baf45a67 100644 --- a/src/utils/copilotOnRails/modelSelection.ts +++ b/src/utils/copilotOnRails/modelSelection.ts @@ -24,7 +24,7 @@ export function getSupportedModelName(...identifiers: string[]): typeof supporte export function getDefaultOpusModelOption(models: readonly AvailableChatModel[]): string | undefined { const model = models - .filter(model => model.vendor === 'copilot' && getSupportedModelName(model.id, model.family, model.name) === 'Opus') + .filter(model => getSupportedModelName(model.id, model.family, model.name) === 'Opus') .sort((a, b) => a.version.localeCompare(b.version, undefined, { numeric: true, sensitivity: 'base' }) || a.name.localeCompare(b.name))[0]; @@ -68,14 +68,14 @@ export function resolveCopilotHarnessModel(displayName: string, models: readonly } /** - * Returns the currently available supported Copilot models as qualified names - * that VS Code can resolve when opening chat. + * Filters a vendor-scoped API result to supported model families and returns + * qualified names that VS Code can resolve when opening chat. */ export function getSupportedModelOptions(models: readonly AvailableChatModel[]): string[] { const matchingModels = models .map(model => ({ supportedName: getSupportedModelName(model.id, model.family, model.name), model })) .filter((entry): entry is { supportedName: typeof supportedModelNames[number]; model: AvailableChatModel } => - entry.model.vendor === 'copilot' && entry.supportedName !== undefined) + entry.supportedName !== undefined) .sort((a, b) => supportedModelNames.indexOf(a.supportedName) - supportedModelNames.indexOf(b.supportedName) || b.model.version.localeCompare(a.model.version, undefined, { numeric: true, sensitivity: 'base' }) diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts index 003a5e447..d0422dd38 100644 --- a/test/copilotOnRails/modelSelection.test.ts +++ b/test/copilotOnRails/modelSelection.test.ts @@ -6,7 +6,6 @@ import assert from 'assert'; import { AvailableChatModel, - getSupportedModelOptions, resolveAvailableChatModel, resolveCopilotHarnessModel, } from '../../src/utils/copilotOnRails/modelSelection'; @@ -29,26 +28,7 @@ const legacyHarnessModel: AvailableChatModel = { vendor: 'copilotcli', }; -const opusModel: AvailableChatModel = { - id: 'claude-opus-5.5', - vendor: 'copilot', - family: 'claude-opus-5.5', - version: '5.5', - name: 'Claude Opus 5.5', -}; - suite('Copilot on Rails model selection', () => { - test('includes available Opus models in the picker', () => { - assert.deepStrictEqual( - getSupportedModelOptions([localModel, opusModel]), - ['Claude Opus 5.5 (copilot)', 'GPT-6.1 Sol (copilot)'], - ); - }); - - test('returns no picker options when VS Code reports no models', () => { - assert.deepStrictEqual(getSupportedModelOptions([]), []); - }); - test('resolves a qualified local model name', () => { assert.strictEqual( resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, harnessModel]), From 05659a8f9ad2166d4d7b422222d592916719dad5 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 12:24:00 -0700 Subject: [PATCH 10/34] Add temporary model picker diagnostics --- .../extension/createProjectWithCopilot.ts | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 411e0cd49..9fb90e190 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -9,7 +9,7 @@ import { ensureCopilotChatReady } from "../../../commands/copilotOnRails/openCha import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/registerCopilotOnRailsCommands"; import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { getDefaultOpusModelOption, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; +import { getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; @@ -79,10 +79,30 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilot".'); const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilot' }); + console.log( + `[Copilot on Rails model picker] selectChatModels returned ${availableModels.length} model(s):`, + availableModels.map(model => ({ + id: model.id, + vendor: model.vendor, + family: model.family, + version: model.version, + name: model.name, + maxInputTokens: model.maxInputTokens, + supportedName: getSupportedModelName(model.id, model.family, model.name), + })), + ); + const modelOptions = getSupportedModelOptions(availableModels); + console.log( + `[Copilot on Rails model picker] Supported-family filtering produced ${modelOptions.length} picker option(s):`, + modelOptions, + ); + setCorProp(context, 'availableModelCount', modelOptions.length); if (modelOptions.length === 0) { + console.warn('[Copilot on Rails model picker] No models remained after supported-family filtering.'); void vscode.window.showErrorMessage(vscode.l10n.t( 'No supported GitHub Copilot models are available. Check that you are signed in to GitHub Copilot and that your plan and organization policies allow a supported model, then try again.', )); @@ -92,6 +112,11 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm const selectedModel = initialModel && modelOptions.includes(initialModel) ? initialModel : getDefaultOpusModelOption(availableModels); + console.log('[Copilot on Rails model picker] Initial and selected model:', { + requestedInitialModel: initialModel, + selectedModel, + }); + const controller = new CreateProjectViewController({ title: vscode.l10n.t('Create with Copilot'), heading: vscode.l10n.t('What would you like to build?'), From be21449d767fa657e49b18d67050755d7c454dc8 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:19:12 -0700 Subject: [PATCH 11/34] Open Chat before starting Copilot agents --- docs/copilot-create-project.md | 6 +++--- src/commands/copilotOnRails/openChatWithAgent.ts | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 5e3c3d0b4..45ce5b48a 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -185,9 +185,9 @@ description, optionally pick a **Model**, and press **Plan** (or `Ctrl+Enter`). Create with Copilot prompt view

-Pressing **Plan** starts the **`azure-project-plan`** agent in a new Copilot chat session. Every new-project -prompt goes through this flow, including frontend-only apps with no backend, database, or Azure services — -those simply plan a single `frontend` service with **No datastore required**. +Pressing **Plan** opens Copilot Chat if needed, then starts the **`azure-project-plan`** agent in a new chat +session. Every new-project prompt goes through this flow, including frontend-only apps with no backend, +database, or Azure services. Those simply plan a single `frontend` service with **No datastore required**. ## Stage 2 — Review requirements diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index 6f9b0221b..84e918ed2 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -22,6 +22,7 @@ const COPILOT_CHAT_EXTENSION_ID = 'GitHub.copilot-chat'; const MANAGE_WORKSPACE_TRUST_COMMAND_ID = 'workbench.trust.manage'; const RELOAD_WINDOW_COMMAND_ID = 'workbench.action.reloadWindow'; const OPEN_CHAT_COMMAND_ID = 'workbench.action.chat.open'; +const NEW_CHAT_COMMAND_ID = 'workbench.action.chat.newChat'; const OPEN_CHAT_COMMAND_PREFIX = 'workbench.action.chat.open'; const AGENT_COMMAND_READY_TIMEOUT_MS = 10_000; const AGENT_COMMAND_READY_POLL_MS = 100; @@ -126,9 +127,13 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: try { await ensureCopilotHarnessOn(); + // Custom-agent commands are registered from the focused chat widget. Open Chat before + // creating the fresh session so first-time launches do not wait on a widget that does not exist. + await vscode.commands.executeCommand(OPEN_CHAT_COMMAND_ID); + // Fresh chat session per phase hand-off: agents coordinate through the `.azure/*` plan // files on disk, not chat history, so a clean session keeps each agent focused on its phase. - await vscode.commands.executeCommand('workbench.action.chat.newChat'); + await vscode.commands.executeCommand(NEW_CHAT_COMMAND_ID); await waitForAgentModeRegistration(agentName); await vscode.commands.executeCommand(getAgentChatOpenCommandId(agentName)); From 55cd996043acfa43172ae543b1d02449c8073093 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:32:41 -0700 Subject: [PATCH 12/34] Probe Copilot Agent Host models --- .../copilotOnRails/extension/createProjectWithCopilot.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 9fb90e190..ef96eb2f3 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -79,8 +79,8 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilot".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilot' }); + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); console.log( `[Copilot on Rails model picker] selectChatModels returned ${availableModels.length} model(s):`, availableModels.map(model => ({ From ada9f096afa68d4685c1695ef11df7f15ae4fc72 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:34:52 -0700 Subject: [PATCH 13/34] Restore Copilot picker model source --- .../extension/createProjectWithCopilot.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index ef96eb2f3..6baa58eb2 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -79,8 +79,17 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilot".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilot' }); + const allModels = await vscode.lm.selectChatModels(); + const modelCountsByVendor = allModels.reduce>((counts, model) => { + counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; + return counts; + }, {}); + console.log( + '[Copilot on Rails model picker] Registered model counts by vendor:', + modelCountsByVendor, + ); console.log( `[Copilot on Rails model picker] selectChatModels returned ${availableModels.length} model(s):`, availableModels.map(model => ({ From b82a15072f95899d8899d3635c4fa1a7d3f20e30 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:36:45 -0700 Subject: [PATCH 14/34] Initialize Copilot harness before model picker --- docs/copilot-create-project.md | 3 ++- .../extension/createProjectWithCopilot.ts | 13 ++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 45ce5b48a..1de44a190 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -178,7 +178,8 @@ the create flow automatically. **Choose Empty Folder…** also opens the selecte ## Stage 1 — Describe your project -The **Create with Copilot** view opens with the heading **"What would you like to build?"**. Type a +The extension first initializes a Copilot Agent Host chat session so its models are available. The +**Create with Copilot** view then opens with the heading **"What would you like to build?"**. Type a description, optionally pick a **Model**, and press **Plan** (or `Ctrl+Enter`).

diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 6baa58eb2..b480f3556 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -11,6 +11,7 @@ import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/proj import { CopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; import { getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; +import { ensureCopilotHarnessOn } from "./harnessSettings"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; import { consumeReloadResumePrompt } from "./reloadResumePrompt"; @@ -18,6 +19,7 @@ import { writePendingCreateMarker } from "./resumePendingCreateWithCopilot"; const localDev = vscode.l10n.t('Local Development'); const deploy = vscode.l10n.t('Deploy'); +const OPEN_COPILOT_HARNESS_SESSION_COMMAND_ID = 'workbench.action.chat.openNewSessionSidebar.agent-host-copilotcli'; export const OPEN_PROJECT_FOLDER_OPTIONS = { forceNewWindow: true } as const; export const PROJECT_FOLDER_SELECTION_TELEMETRY_KEY = 'projectFolderSelection'; export type ProjectFolderSelection = 'newSubfolder' | 'selectedEmptyFolder'; @@ -79,8 +81,13 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilot".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilot' }); + await ensureCopilotHarnessOn(); + console.log('[Copilot on Rails model picker] Opening a Copilot Agent Host session to register its model provider.'); + await vscode.commands.executeCommand(OPEN_COPILOT_HARNESS_SESSION_COMMAND_ID); + console.log('[Copilot on Rails model picker] Copilot Agent Host session opened.'); + + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); const allModels = await vscode.lm.selectChatModels(); const modelCountsByVendor = allModels.reduce>((counts, model) => { counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; @@ -113,7 +120,7 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm if (modelOptions.length === 0) { console.warn('[Copilot on Rails model picker] No models remained after supported-family filtering.'); void vscode.window.showErrorMessage(vscode.l10n.t( - 'No supported GitHub Copilot models are available. Check that you are signed in to GitHub Copilot and that your plan and organization policies allow a supported model, then try again.', + 'No supported Copilot Agent Host models are available. Check that you are signed in to GitHub Copilot and that your plan and organization policies allow a supported model, then try again.', )); return; } From 78f57edc70d960db07e65c1cb6554856ed758658 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:41:19 -0700 Subject: [PATCH 15/34] Remove ineffective model picker preflight --- docs/copilot-create-project.md | 3 +-- .../extension/createProjectWithCopilot.ts | 13 +++---------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 1de44a190..45ce5b48a 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -178,8 +178,7 @@ the create flow automatically. **Choose Empty Folder…** also opens the selecte ## Stage 1 — Describe your project -The extension first initializes a Copilot Agent Host chat session so its models are available. The -**Create with Copilot** view then opens with the heading **"What would you like to build?"**. Type a +The **Create with Copilot** view opens with the heading **"What would you like to build?"**. Type a description, optionally pick a **Model**, and press **Plan** (or `Ctrl+Enter`).

diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index b480f3556..6baa58eb2 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -11,7 +11,6 @@ import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/proj import { CopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; import { getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; -import { ensureCopilotHarnessOn } from "./harnessSettings"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; import { consumeReloadResumePrompt } from "./reloadResumePrompt"; @@ -19,7 +18,6 @@ import { writePendingCreateMarker } from "./resumePendingCreateWithCopilot"; const localDev = vscode.l10n.t('Local Development'); const deploy = vscode.l10n.t('Deploy'); -const OPEN_COPILOT_HARNESS_SESSION_COMMAND_ID = 'workbench.action.chat.openNewSessionSidebar.agent-host-copilotcli'; export const OPEN_PROJECT_FOLDER_OPTIONS = { forceNewWindow: true } as const; export const PROJECT_FOLDER_SELECTION_TELEMETRY_KEY = 'projectFolderSelection'; export type ProjectFolderSelection = 'newSubfolder' | 'selectedEmptyFolder'; @@ -81,13 +79,8 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - await ensureCopilotHarnessOn(); - console.log('[Copilot on Rails model picker] Opening a Copilot Agent Host session to register its model provider.'); - await vscode.commands.executeCommand(OPEN_COPILOT_HARNESS_SESSION_COMMAND_ID); - console.log('[Copilot on Rails model picker] Copilot Agent Host session opened.'); - - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilot".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilot' }); const allModels = await vscode.lm.selectChatModels(); const modelCountsByVendor = allModels.reduce>((counts, model) => { counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; @@ -120,7 +113,7 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm if (modelOptions.length === 0) { console.warn('[Copilot on Rails model picker] No models remained after supported-family filtering.'); void vscode.window.showErrorMessage(vscode.l10n.t( - 'No supported Copilot Agent Host models are available. Check that you are signed in to GitHub Copilot and that your plan and organization policies allow a supported model, then try again.', + 'No supported GitHub Copilot models are available. Check that you are signed in to GitHub Copilot and that your plan and organization policies allow a supported model, then try again.', )); return; } From 6e30dc09df99a617e9a35cb65999bfc5080d605d Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:47:20 -0700 Subject: [PATCH 16/34] Probe unfiltered Agent Host models --- docs/copilot-create-project.md | 19 ++++++++----------- .../extension/createProjectWithCopilot.ts | 14 +++++++------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 45ce5b48a..9930ce3ad 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -138,17 +138,14 @@ session** running the next agent. Between hand‑offs, agents open **webviews** and waits for custom-agent registration to settle. After creating a fresh phase chat, it primes the custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. -- A Copilot plan with access to at least one supported model. The model picker lists the Opus, Sonnet, - GPT Sol, and GPT Terra models currently available through GitHub Copilot, so newly available - versions appear without an extension update. The lowest-version available Opus model is selected by default. - The extension activates GitHub Copilot Chat before loading this list. If VS Code reports no supported - models, the extension asks the user to check their Copilot sign-in, plan, and organization model policies - instead of opening the prompt page with an empty picker. - When the Copilot Harness is active, the extension maps that selection only to the matching - `agent-host-copilotcli` model. It does not pass Local or legacy `copilotcli` identifiers, which - VS Code rejects as foreign to the Agent Host session. The extension constructs the Agent Host - selector from the selected Local model ID because the language model extension API does not list - the Agent Host copies that the chat command can resolve. +- A Copilot plan with access to at least one model. The model picker lists every model returned by the + `agent-host-copilotcli` language-model vendor without applying a family allowlist. The lowest-version + available Opus model is selected by default. The extension activates GitHub Copilot Chat before loading + this list and reports an error instead of opening the prompt page with an empty picker when the vendor + returns no models. + The selected `agent-host-copilotcli` model ID and vendor are passed to the Copilot Harness when the + project agent starts. The extension does not pass Local or legacy `copilotcli` identifiers, which + VS Code rejects as foreign to the Agent Host session. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 6baa58eb2..2dfda727b 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -9,7 +9,7 @@ import { ensureCopilotChatReady } from "../../../commands/copilotOnRails/openCha import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/registerCopilotOnRailsCommands"; import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; +import { getDefaultOpusModelOption, getSupportedModelName } from "../../../utils/copilotOnRails/modelSelection"; import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; @@ -79,8 +79,8 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilot".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilot' }); + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); const allModels = await vscode.lm.selectChatModels(); const modelCountsByVendor = allModels.reduce>((counts, model) => { counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; @@ -103,17 +103,17 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm })), ); - const modelOptions = getSupportedModelOptions(availableModels); + const modelOptions = [...new Set(availableModels.map(model => `${model.name} (${model.vendor})`))]; console.log( - `[Copilot on Rails model picker] Supported-family filtering produced ${modelOptions.length} picker option(s):`, + `[Copilot on Rails model picker] Unfiltered API result produced ${modelOptions.length} picker option(s):`, modelOptions, ); setCorProp(context, 'availableModelCount', modelOptions.length); if (modelOptions.length === 0) { - console.warn('[Copilot on Rails model picker] No models remained after supported-family filtering.'); + console.warn('[Copilot on Rails model picker] The Agent Host vendor returned no models.'); void vscode.window.showErrorMessage(vscode.l10n.t( - 'No supported GitHub Copilot models are available. Check that you are signed in to GitHub Copilot and that your plan and organization policies allow a supported model, then try again.', + 'The Copilot Agent Host returned no models. Check the Developer Tools console for model picker diagnostics.', )); return; } From 7992acdb9c08d99ace60dbc71b8bb366d700fc4f Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:49:44 -0700 Subject: [PATCH 17/34] Probe unfiltered Copilot CLI models --- docs/copilot-create-project.md | 7 +++---- .../copilotOnRails/extension/createProjectWithCopilot.ts | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 9930ce3ad..721b01c0c 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -139,13 +139,12 @@ session** running the next agent. Between hand‑offs, agents open **webviews** custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. - A Copilot plan with access to at least one model. The model picker lists every model returned by the - `agent-host-copilotcli` language-model vendor without applying a family allowlist. The lowest-version + `copilotcli` language-model vendor without applying a family allowlist. The lowest-version available Opus model is selected by default. The extension activates GitHub Copilot Chat before loading this list and reports an error instead of opening the prompt page with an empty picker when the vendor returns no models. - The selected `agent-host-copilotcli` model ID and vendor are passed to the Copilot Harness when the - project agent starts. The extension does not pass Local or legacy `copilotcli` identifiers, which - VS Code rejects as foreign to the Agent Host session. + When the Copilot Harness starts, the extension maps that selection to an `agent-host-copilotcli` + selector with the same model ID. It does not pass the `copilotcli` vendor to the Agent Host session. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 2dfda727b..02f80e1b1 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -79,8 +79,8 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilotcli".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); const allModels = await vscode.lm.selectChatModels(); const modelCountsByVendor = allModels.reduce>((counts, model) => { counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; @@ -111,9 +111,9 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm setCorProp(context, 'availableModelCount', modelOptions.length); if (modelOptions.length === 0) { - console.warn('[Copilot on Rails model picker] The Agent Host vendor returned no models.'); + console.warn('[Copilot on Rails model picker] The Copilot CLI vendor returned no models.'); void vscode.window.showErrorMessage(vscode.l10n.t( - 'The Copilot Agent Host returned no models. Check the Developer Tools console for model picker diagnostics.', + 'The Copilot CLI model provider returned no models. Check the Developer Tools console for model picker diagnostics.', )); return; } From 004ac25c4810bc739020c776dceb206e6bfa45a3 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:52:16 -0700 Subject: [PATCH 18/34] Restore model family filtering with clean picker labels --- docs/copilot-create-project.md | 9 +++++---- src/utils/copilotOnRails/modelSelection.ts | 9 +++++---- .../extension/createProjectWithCopilot.ts | 12 ++++++------ test/copilotOnRails/modelSelection.test.ts | 8 ++++++++ 4 files changed, 24 insertions(+), 14 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 721b01c0c..d60c1739c 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -138,11 +138,12 @@ session** running the next agent. Between hand‑offs, agents open **webviews** and waits for custom-agent registration to settle. After creating a fresh phase chat, it primes the custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. -- A Copilot plan with access to at least one model. The model picker lists every model returned by the - `copilotcli` language-model vendor without applying a family allowlist. The lowest-version - available Opus model is selected by default. The extension activates GitHub Copilot Chat before loading +- A Copilot plan with access to at least one supported model. The model picker lists Opus, Sonnet, + GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, using their display + names without a vendor suffix. The lowest-version available Opus model is selected by default, + or the first supported option when no Opus model is available. The extension activates GitHub Copilot Chat before loading this list and reports an error instead of opening the prompt page with an empty picker when the vendor - returns no models. + returns no supported models. When the Copilot Harness starts, the extension maps that selection to an `agent-host-copilotcli` selector with the same model ID. It does not pass the `copilotcli` vendor to the Agent Host session. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already diff --git a/src/utils/copilotOnRails/modelSelection.ts b/src/utils/copilotOnRails/modelSelection.ts index 6baf45a67..3a15ff826 100644 --- a/src/utils/copilotOnRails/modelSelection.ts +++ b/src/utils/copilotOnRails/modelSelection.ts @@ -29,7 +29,7 @@ export function getDefaultOpusModelOption(models: readonly AvailableChatModel[]) a.version.localeCompare(b.version, undefined, { numeric: true, sensitivity: 'base' }) || a.name.localeCompare(b.name))[0]; - return model ? `${model.name} (${model.vendor})` : undefined; + return model?.name; } export function resolveAvailableChatModel(displayName: string, models: readonly AvailableChatModel[]): AvailableChatModel | undefined { @@ -51,7 +51,8 @@ export function resolveAvailableChatModel(displayName: string, models: readonly * Agent Host model used by the Copilot Harness. */ export function resolveCopilotHarnessModel(displayName: string, models: readonly AvailableChatModel[]): AvailableChatModel | undefined { - const selectedModel = resolveAvailableChatModel(displayName, models); + const selectedModel = models.find(model => model.vendor === 'copilotcli' && model.name === displayName) + ?? resolveAvailableChatModel(displayName, models); if (!selectedModel) { return undefined; } @@ -69,7 +70,7 @@ export function resolveCopilotHarnessModel(displayName: string, models: readonly /** * Filters a vendor-scoped API result to supported model families and returns - * qualified names that VS Code can resolve when opening chat. + * display names for the model picker. */ export function getSupportedModelOptions(models: readonly AvailableChatModel[]): string[] { const matchingModels = models @@ -81,5 +82,5 @@ export function getSupportedModelOptions(models: readonly AvailableChatModel[]): || b.model.version.localeCompare(a.model.version, undefined, { numeric: true, sensitivity: 'base' }) || a.model.name.localeCompare(b.model.name)); - return [...new Set(matchingModels.map(({ model }) => `${model.name} (${model.vendor})`))]; + return [...new Set(matchingModels.map(({ model }) => model.name))]; } diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 02f80e1b1..1f14fd9a0 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -9,7 +9,7 @@ import { ensureCopilotChatReady } from "../../../commands/copilotOnRails/openCha import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/registerCopilotOnRailsCommands"; import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { getDefaultOpusModelOption, getSupportedModelName } from "../../../utils/copilotOnRails/modelSelection"; +import { getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; @@ -103,24 +103,24 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm })), ); - const modelOptions = [...new Set(availableModels.map(model => `${model.name} (${model.vendor})`))]; + const modelOptions = getSupportedModelOptions(availableModels); console.log( - `[Copilot on Rails model picker] Unfiltered API result produced ${modelOptions.length} picker option(s):`, + `[Copilot on Rails model picker] Supported-family filtering produced ${modelOptions.length} picker option(s):`, modelOptions, ); setCorProp(context, 'availableModelCount', modelOptions.length); if (modelOptions.length === 0) { - console.warn('[Copilot on Rails model picker] The Copilot CLI vendor returned no models.'); + console.warn('[Copilot on Rails model picker] No supported Copilot CLI models are available.'); void vscode.window.showErrorMessage(vscode.l10n.t( - 'The Copilot CLI model provider returned no models. Check the Developer Tools console for model picker diagnostics.', + 'No supported Copilot CLI models are available. Check the Developer Tools console for model picker diagnostics.', )); return; } const selectedModel = initialModel && modelOptions.includes(initialModel) ? initialModel - : getDefaultOpusModelOption(availableModels); + : getDefaultOpusModelOption(availableModels) ?? modelOptions[0]; console.log('[Copilot on Rails model picker] Initial and selected model:', { requestedInitialModel: initialModel, selectedModel, diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts index d0422dd38..bfed34edb 100644 --- a/test/copilotOnRails/modelSelection.test.ts +++ b/test/copilotOnRails/modelSelection.test.ts @@ -49,4 +49,12 @@ suite('Copilot on Rails model selection', () => { harnessModel, ); }); + + test('resolves an unqualified picker name using the Copilot CLI model id', () => { + const cliModel = { ...legacyHarnessModel, id: 'cli-sol' }; + assert.deepStrictEqual( + resolveCopilotHarnessModel('GPT-6.1 Sol', [localModel, cliModel]), + { ...cliModel, vendor: 'agent-host-copilotcli' }, + ); + }); }); From de264b4dc18510db1f65d64d46902b87c7fe0498 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:54:22 -0700 Subject: [PATCH 19/34] Clarify model loading retry message --- docs/copilot-create-project.md | 5 +++-- .../copilotOnRails/extension/createProjectWithCopilot.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index d60c1739c..c0b463a88 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -142,8 +142,9 @@ session** running the next agent. Between hand‑offs, agents open **webviews** GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, using their display names without a vendor suffix. The lowest-version available Opus model is selected by default, or the first supported option when no Opus model is available. The extension activates GitHub Copilot Chat before loading - this list and reports an error instead of opening the prompt page with an empty picker when the vendor - returns no supported models. + this list. If no supported models are available, it explains that Copilot models may still be loading + and asks the user to wait a moment, then click **Create New Project With Copilot** again. If the problem + persists, the message advises checking Copilot sign-in and organization model policies. When the Copilot Harness starts, the extension maps that selection to an `agent-host-copilotcli` selector with the same model ID. It does not pass the `copilotcli` vendor to the Agent Host session. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 1f14fd9a0..23bd0aa3e 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -113,7 +113,7 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm if (modelOptions.length === 0) { console.warn('[Copilot on Rails model picker] No supported Copilot CLI models are available.'); void vscode.window.showErrorMessage(vscode.l10n.t( - 'No supported Copilot CLI models are available. Check the Developer Tools console for model picker diagnostics.', + 'Copilot models may still be loading. Wait a moment, then click Create New Project With Copilot again. If the problem persists, check your Copilot sign-in and organization model policies.', )); return; } From d493cf4a2ecc40929972ba2e7db5a11fb26bf2c2 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:03:41 -0700 Subject: [PATCH 20/34] Retry Agent Host model vendor probe --- docs/copilot-create-project.md | 6 +++--- .../copilotOnRails/extension/createProjectWithCopilot.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index c0b463a88..961324fb1 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -139,14 +139,14 @@ session** running the next agent. Between hand‑offs, agents open **webviews** custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. - A Copilot plan with access to at least one supported model. The model picker lists Opus, Sonnet, - GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, using their display + GPT Sol, and GPT Terra models returned by the `agent-host-copilotcli` language-model vendor, using their display names without a vendor suffix. The lowest-version available Opus model is selected by default, or the first supported option when no Opus model is available. The extension activates GitHub Copilot Chat before loading this list. If no supported models are available, it explains that Copilot models may still be loading and asks the user to wait a moment, then click **Create New Project With Copilot** again. If the problem persists, the message advises checking Copilot sign-in and organization model policies. - When the Copilot Harness starts, the extension maps that selection to an `agent-host-copilotcli` - selector with the same model ID. It does not pass the `copilotcli` vendor to the Agent Host session. + When the Copilot Harness starts, the extension passes the selected model's `agent-host-copilotcli` + vendor and model ID to the Agent Host session. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 23bd0aa3e..3ace7a272 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -79,8 +79,8 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilotcli".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); const allModels = await vscode.lm.selectChatModels(); const modelCountsByVendor = allModels.reduce>((counts, model) => { counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; @@ -111,7 +111,7 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm setCorProp(context, 'availableModelCount', modelOptions.length); if (modelOptions.length === 0) { - console.warn('[Copilot on Rails model picker] No supported Copilot CLI models are available.'); + console.warn('[Copilot on Rails model picker] No supported Copilot Agent Host models are available.'); void vscode.window.showErrorMessage(vscode.l10n.t( 'Copilot models may still be loading. Wait a moment, then click Create New Project With Copilot again. If the problem persists, check your Copilot sign-in and organization model policies.', )); From 5afaad9ad4fd1f85e8346d2f22cbc2578b8a6f16 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:07:11 -0700 Subject: [PATCH 21/34] Revert "Retry Agent Host model vendor probe" This reverts commit d493cf4a2ecc40929972ba2e7db5a11fb26bf2c2. --- docs/copilot-create-project.md | 6 +++--- .../copilotOnRails/extension/createProjectWithCopilot.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 961324fb1..c0b463a88 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -139,14 +139,14 @@ session** running the next agent. Between hand‑offs, agents open **webviews** custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. - A Copilot plan with access to at least one supported model. The model picker lists Opus, Sonnet, - GPT Sol, and GPT Terra models returned by the `agent-host-copilotcli` language-model vendor, using their display + GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, using their display names without a vendor suffix. The lowest-version available Opus model is selected by default, or the first supported option when no Opus model is available. The extension activates GitHub Copilot Chat before loading this list. If no supported models are available, it explains that Copilot models may still be loading and asks the user to wait a moment, then click **Create New Project With Copilot** again. If the problem persists, the message advises checking Copilot sign-in and organization model policies. - When the Copilot Harness starts, the extension passes the selected model's `agent-host-copilotcli` - vendor and model ID to the Agent Host session. + When the Copilot Harness starts, the extension maps that selection to an `agent-host-copilotcli` + selector with the same model ID. It does not pass the `copilotcli` vendor to the Agent Host session. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 3ace7a272..23bd0aa3e 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -79,8 +79,8 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "agent-host-copilotcli".'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'agent-host-copilotcli' }); + console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilotcli".'); + const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); const allModels = await vscode.lm.selectChatModels(); const modelCountsByVendor = allModels.reduce>((counts, model) => { counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; @@ -111,7 +111,7 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm setCorProp(context, 'availableModelCount', modelOptions.length); if (modelOptions.length === 0) { - console.warn('[Copilot on Rails model picker] No supported Copilot Agent Host models are available.'); + console.warn('[Copilot on Rails model picker] No supported Copilot CLI models are available.'); void vscode.window.showErrorMessage(vscode.l10n.t( 'Copilot models may still be loading. Wait a moment, then click Create New Project With Copilot again. If the problem persists, check your Copilot sign-in and organization model policies.', )); From 1d94c19b7d52ebba73000dde24abedbf1c74d566 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:15:17 -0700 Subject: [PATCH 22/34] Track model picker discovery and preserve CLI selectors --- docs/copilot-create-project.md | 21 +++++++- .../copilotOnRails/openChatWithAgent.ts | 8 ++-- src/utils/copilotOnRails/modelSelection.ts | 23 --------- .../extension/createProjectWithCopilot.ts | 27 +++++++++-- .../createProjectWithCopilot.test.ts | 48 +++++++++++++++++++ test/copilotOnRails/modelSelection.test.ts | 32 +++++-------- test/copilotOnRails/openChatWithAgent.test.ts | 8 ++-- 7 files changed, 111 insertions(+), 56 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index c0b463a88..b7fa7094f 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -145,8 +145,8 @@ session** running the next agent. Between hand‑offs, agents open **webviews** this list. If no supported models are available, it explains that Copilot models may still be loading and asks the user to wait a moment, then click **Create New Project With Copilot** again. If the problem persists, the message advises checking Copilot sign-in and organization model policies. - When the Copilot Harness starts, the extension maps that selection to an `agent-host-copilotcli` - selector with the same model ID. It does not pass the `copilotcli` vendor to the Agent Host session. + When the Copilot Harness starts, the extension passes the selected model's actual ID and `copilotcli` + vendor to Chat without remapping it or constructing an Agent Host fallback. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without @@ -579,6 +579,22 @@ The diagnostics object has four fields: `report_agent_launch` contributes at most one diagnostic lifecycle for each agent in a chat session. +Model discovery adds these properties to the `createProjectWithCopilot` event and CoR telemetry: + +| Property | Meaning | +| --- | --- | +| `modelPickerVendor` | The queried vendor, `copilotcli`. | +| `modelPickerReturnedModelCount` | Number of models returned by that vendor. | +| `modelPickerSupportedModelCount` | Number matching the supported-family filter, before duplicate display names are removed. | +| `modelPickerFilteredOutModelCount` | Number excluded by the supported-family filter. | +| `availableModelCount` | Number of distinct options shown in the picker, or zero when the prompt view cannot open. | +| `modelPickerOutcome` | `querying` before discovery completes, `emptyCatalog` for no returned models, `noSupportedModels` when none pass the filter, or `ready` when options are available. | + +Diagnostics also include `modelPickerReturnedModels` with each returned model's ID, name, vendor, +family, and version, and `modelPickerShownModels` with the displayed option names. These catalogs +are workspace-cached diagnostic data only and are not sent to telemetry. The counts and discovery +outcome are mirrored to telemetry separately through the standard CoR property handling. + Privacy guarantees, by design: - Diagnostics are **workspace‑cached only** (VS Code `workspaceState`). @@ -597,6 +613,7 @@ before submitting. | --- | --- | --- | | *"Choose where to create your project."* | The open folder isn't empty. | Choose **Create in New Subfolder…** to build under the current folder, or **Choose Empty Folder…** to build elsewhere. Both open the project in a separate window. | | The new project window opens in Restricted Mode and the Azure Project view is unavailable. | Workspace Trust must be granted explicitly; extensions cannot trust a folder on your behalf. | Select **Trust** from the Restricted Mode banner or Workspace Trust editor. The pending create flow resumes automatically after the extension activates. | +| The create flow says Copilot models may still be loading. | The CLI model catalog is empty or contains no supported models yet. | Wait a moment and click **Create New Project With Copilot** again. Inspect the discovery properties described above to distinguish an empty catalog from models excluded by the family filter and compare returned models with displayed options. If the problem persists, check Copilot sign-in and organization model policies. | | An agent says it needs its instruction files, or behaves oddly / follows outdated steps. | `.github/agents/` is missing or stale. | Accept the download prompt, or run **Download Azure Agent Instructions**. The version stamp auto‑refreshes stale copies. | | Chat opens with the wrong agent or generic Agent mode. | VS Code did not honor the requested custom mode, the custom instructions were not loaded, or the MCP tool was unavailable. | Inspect `diagnosticEvents` for a successful `report_agent_launch` event. Its `agentName` property identifies the agent that reported. If the event is missing, the startup report never reached the CoR MCP server. | | Frontend preview stuck on *"Starting…"*; **Approve UI** never enables (but the app loads in a normal browser). | A second dev server is contending for the preview port. | Stop **all** manually‑started dev servers, free the port, ensure the frontend's `vite.config` is the clean minimal version, then reopen the preview and let it own the server. Don't verify by starting your own server. | diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index 84e918ed2..79cd1bb07 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -8,7 +8,7 @@ import * as vscode from 'vscode'; import { ext } from '../../extensionVariables'; import { projectSubmissionState } from '../../tree/project/projectSubmissionState'; import { CopilotOnRailsContext } from '../../utils/copilotOnRails/CopilotOnRailsContext'; -import { resolveCopilotHarnessModel } from '../../utils/copilotOnRails/modelSelection'; +import { resolveAvailableChatModel } from '../../utils/copilotOnRails/modelSelection'; import { setCorErrorProp, setCorProp } from '../../utils/copilotOnRails/telemetryUtils'; import { AUTOPILOT_QUERY_MARKER, isAutopilotActive } from '../../webviews/copilotOnRails/extension/autopilot'; import { ensureCopilotHarnessOn } from '../../webviews/copilotOnRails/extension/harnessSettings'; @@ -39,13 +39,13 @@ export function registerWorkspaceTrustTracking(): void { } /** - * Resolves a user-facing model name (e.g. "Claude Opus 4.7 (copilot)") to a + * Resolves a user-facing model name (e.g. "Claude Opus 4.7") to a * modelSelector object that VS Code's chat commands expect. */ async function resolveModelSelector(displayName: string): Promise<{ id?: string; vendor?: string } | undefined> { try { - const models = await vscode.lm.selectChatModels(); - const match = resolveCopilotHarnessModel(displayName, models); + const models = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); + const match = resolveAvailableChatModel(displayName, models); if (match) { return { id: match.id, vendor: match.vendor }; } diff --git a/src/utils/copilotOnRails/modelSelection.ts b/src/utils/copilotOnRails/modelSelection.ts index 3a15ff826..1c33d50f4 100644 --- a/src/utils/copilotOnRails/modelSelection.ts +++ b/src/utils/copilotOnRails/modelSelection.ts @@ -12,7 +12,6 @@ export interface AvailableChatModel { } export const supportedModelNames = ['Opus', 'Sonnet', 'GPT Sol', 'GPT Terra'] as const; -const copilotHarnessVendor = 'agent-host-copilotcli'; export function getSupportedModelName(...identifiers: string[]): typeof supportedModelNames[number] | undefined { const modelIdentifiers = identifiers.join(' '); @@ -46,28 +45,6 @@ export function resolveAvailableChatModel(displayName: string, models: readonly ?? models.find(model => model.id === displayName); } -/** - * Maps a model selected from VS Code's local Copilot model pool to the corresponding - * Agent Host model used by the Copilot Harness. - */ -export function resolveCopilotHarnessModel(displayName: string, models: readonly AvailableChatModel[]): AvailableChatModel | undefined { - const selectedModel = models.find(model => model.vendor === 'copilotcli' && model.name === displayName) - ?? resolveAvailableChatModel(displayName, models); - if (!selectedModel) { - return undefined; - } - - if (selectedModel.vendor === copilotHarnessVendor) { - return selectedModel; - } - - const harnessModels = models.filter(model => model.vendor === copilotHarnessVendor); - return harnessModels.find(model => model.id === selectedModel.id) - ?? harnessModels.find(model => model.family === selectedModel.family) - ?? harnessModels.find(model => model.name === selectedModel.name) - ?? { ...selectedModel, vendor: copilotHarnessVendor }; -} - /** * Filters a vendor-scoped API result to supported model families and returns * display names for the model picker. diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 23bd0aa3e..779a9e6d1 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -8,8 +8,8 @@ import * as vscode from 'vscode'; import { ensureCopilotChatReady } from "../../../commands/copilotOnRails/openChatWithAgent"; import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/registerCopilotOnRailsCommands"; import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; -import { CopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; +import { CopilotOnRailsContext, ensureRequiredCopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; +import { AvailableChatModel, getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; @@ -80,7 +80,11 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm } console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilotcli".'); + setCorProp(context, 'modelPickerVendor', 'copilotcli'); + setCorProp(context, 'modelPickerOutcome', 'querying'); const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); + const modelOptions = getSupportedModelOptions(availableModels); + recordModelPickerDiscovery(context, availableModels, modelOptions); const allModels = await vscode.lm.selectChatModels(); const modelCountsByVendor = allModels.reduce>((counts, model) => { counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; @@ -103,13 +107,11 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm })), ); - const modelOptions = getSupportedModelOptions(availableModels); console.log( `[Copilot on Rails model picker] Supported-family filtering produced ${modelOptions.length} picker option(s):`, modelOptions, ); - setCorProp(context, 'availableModelCount', modelOptions.length); if (modelOptions.length === 0) { console.warn('[Copilot on Rails model picker] No supported Copilot CLI models are available.'); void vscode.window.showErrorMessage(vscode.l10n.t( @@ -142,6 +144,23 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm controller.revealToForeground(); } +export function recordModelPickerDiscovery( + context: CopilotOnRailsContext, + models: readonly AvailableChatModel[], + modelOptions: readonly string[], +): void { + const supportedModelCount = models.filter(model => getSupportedModelName(model.id, model.family, model.name) !== undefined).length; + setCorProp(context, 'modelPickerReturnedModelCount', models.length); + setCorProp(context, 'modelPickerSupportedModelCount', supportedModelCount); + setCorProp(context, 'modelPickerFilteredOutModelCount', models.length - supportedModelCount); + setCorProp(context, 'availableModelCount', modelOptions.length); + setCorProp(context, 'modelPickerOutcome', models.length === 0 ? 'emptyCatalog' : modelOptions.length === 0 ? 'noSupportedModels' : 'ready'); + + const properties = ensureRequiredCopilotOnRailsContext(context).diagnostics.properties; + properties.modelPickerReturnedModels = models.map(({ id, name, vendor, family, version }) => ({ id, name, vendor, family, version })); + properties.modelPickerShownModels = [...modelOptions]; +} + /** * Ensures the flow starts from a suitable blank slate. * If no folder is open, or the open folder already contains project content, we offer the diff --git a/test/copilotOnRails/createProjectWithCopilot.test.ts b/test/copilotOnRails/createProjectWithCopilot.test.ts index 4057a3bf8..6969de49d 100644 --- a/test/copilotOnRails/createProjectWithCopilot.test.ts +++ b/test/copilotOnRails/createProjectWithCopilot.test.ts @@ -6,11 +6,13 @@ import { createTestActionContext } from '@microsoft/vscode-azext-utils'; import assert from 'assert'; import { ensureRequiredCopilotOnRailsContext } from '../../src/utils/copilotOnRails/CopilotOnRailsContext'; +import { AvailableChatModel, getSupportedModelOptions } from '../../src/utils/copilotOnRails/modelSelection'; import { OPEN_PROJECT_FOLDER_OPTIONS, PROJECT_FOLDER_SELECTION_TELEMETRY_KEY, ProjectFolderSelection, recordProjectFolderSelection, + recordModelPickerDiscovery, validateProjectSubfolderName, } from '../../src/webviews/copilotOnRails/extension/createProjectWithCopilot'; @@ -41,3 +43,49 @@ suite('Create Project with Copilot folder selection', () => { }); } }); + +suite('Create Project with Copilot model picker diagnostics', () => { + const sonnet: AvailableChatModel = { + id: 'claude-sonnet-5', + vendor: 'copilotcli', + name: 'Claude Sonnet 5', + family: 'sonnet', + version: '5', + }; + const haiku: AvailableChatModel = { + ...sonnet, + id: 'claude-haiku-4.5', + name: 'Claude Haiku 4.5', + family: 'haiku', + version: '4.5', + }; + + for (const scenario of [ + { name: 'empty catalog', models: [], supported: 0, excluded: 0, shown: 0, outcome: 'emptyCatalog' }, + { name: 'unsupported catalog', models: [haiku], supported: 0, excluded: 1, shown: 0, outcome: 'noSupportedModels' }, + { name: 'supported catalog with duplicate labels', models: [sonnet, haiku, sonnet], supported: 2, excluded: 1, shown: 1, outcome: 'ready' }, + ]) { + test(`records counts and outcome for ${scenario.name} in telemetry and diagnostics`, async () => { + const context = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); + const options = getSupportedModelOptions(scenario.models); + + recordModelPickerDiscovery(context, scenario.models, options); + + const expected = { + modelPickerReturnedModelCount: scenario.models.length, + modelPickerSupportedModelCount: scenario.supported, + modelPickerFilteredOutModelCount: scenario.excluded, + availableModelCount: scenario.shown, + modelPickerOutcome: scenario.outcome, + }; + for (const [key, value] of Object.entries(expected)) { + assert.strictEqual(context.telemetry.properties[key], String(value)); + assert.strictEqual(context.diagnostics.properties[key], value); + } + assert.deepStrictEqual(context.diagnostics.properties.modelPickerReturnedModels, scenario.models); + assert.deepStrictEqual(context.diagnostics.properties.modelPickerShownModels, options); + assert.strictEqual(context.telemetry.properties.modelPickerReturnedModels, undefined); + assert.strictEqual(context.telemetry.properties.modelPickerShownModels, undefined); + }); + } +}); diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts index bfed34edb..d99ae7f13 100644 --- a/test/copilotOnRails/modelSelection.test.ts +++ b/test/copilotOnRails/modelSelection.test.ts @@ -7,7 +7,6 @@ import assert from 'assert'; import { AvailableChatModel, resolveAvailableChatModel, - resolveCopilotHarnessModel, } from '../../src/utils/copilotOnRails/modelSelection'; const localModel: AvailableChatModel = { @@ -18,12 +17,7 @@ const localModel: AvailableChatModel = { name: 'GPT-6.1 Sol', }; -const harnessModel: AvailableChatModel = { - ...localModel, - vendor: 'agent-host-copilotcli', -}; - -const legacyHarnessModel: AvailableChatModel = { +const cliModel: AvailableChatModel = { ...localModel, vendor: 'copilotcli', }; @@ -31,30 +25,30 @@ const legacyHarnessModel: AvailableChatModel = { suite('Copilot on Rails model selection', () => { test('resolves a qualified local model name', () => { assert.strictEqual( - resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, harnessModel]), + resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, cliModel]), localModel, ); }); - test('maps a local Copilot model to the Agent Host model with the same id', () => { + test('resolves a qualified Copilot CLI model without changing its vendor', () => { assert.strictEqual( - resolveCopilotHarnessModel('GPT-6.1 Sol (copilot)', [localModel, legacyHarnessModel, harnessModel]), - harnessModel, + resolveAvailableChatModel('GPT-6.1 Sol (copilotcli)', [localModel, cliModel]), + cliModel, ); }); - test('constructs an Agent Host selector when the extension API omits Agent Host models', () => { - assert.deepStrictEqual( - resolveCopilotHarnessModel('GPT-6.1 Sol (copilot)', [localModel, legacyHarnessModel]), - harnessModel, + test('does not fabricate a selector for an unavailable model', () => { + assert.strictEqual( + resolveAvailableChatModel('Unavailable model', [cliModel]), + undefined, ); }); test('resolves an unqualified picker name using the Copilot CLI model id', () => { - const cliModel = { ...legacyHarnessModel, id: 'cli-sol' }; - assert.deepStrictEqual( - resolveCopilotHarnessModel('GPT-6.1 Sol', [localModel, cliModel]), - { ...cliModel, vendor: 'agent-host-copilotcli' }, + const model = { ...cliModel, id: 'cli-sol' }; + assert.strictEqual( + resolveAvailableChatModel('GPT-6.1 Sol', [model]), + model, ); }); }); diff --git a/test/copilotOnRails/openChatWithAgent.test.ts b/test/copilotOnRails/openChatWithAgent.test.ts index 4809b6728..cffd31440 100644 --- a/test/copilotOnRails/openChatWithAgent.test.ts +++ b/test/copilotOnRails/openChatWithAgent.test.ts @@ -19,12 +19,12 @@ suite('Copilot on Rails agent chat launch', () => { buildAgentChatOpenOptions( 'azure-project-plan', 'Build a project', - { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, ), { mode: 'azure-project-plan', query: 'Build a project', - modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + modelSelector: { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, waitForRequestAcceptance: true, }, ); @@ -33,11 +33,11 @@ suite('Copilot on Rails agent chat launch', () => { test('carries Autopilot into fresh phase chats without changing the agent or model', () => { for (const agentName of ['azure-project-scaffold', 'azure-project-integrate', 'azure-debug-plan', 'azure-debug-generate', 'azure-deploy']) { assert.deepStrictEqual( - buildAgentChatOpenOptions(agentName, 'Continue the project', { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, true), + buildAgentChatOpenOptions(agentName, 'Continue the project', { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, true), { mode: agentName, query: '[AUTOPILOT MODE] Continue the project', - modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + modelSelector: { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, waitForRequestAcceptance: true, }, ); From 52bb58fa52b1d9c0a02a8ef0512450eda83cecef Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:24:24 -0700 Subject: [PATCH 23/34] Reset project diagnostics before create view discovery --- docs/copilot-create-project.md | 6 +++++- src/utils/copilotOnRails/diagnosticUtils.ts | 2 +- .../extension/controllers/CreateProjectViewController.ts | 8 ++------ .../copilotOnRails/extension/createProjectWithCopilot.ts | 2 ++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index b7fa7094f..d952c5e67 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -573,12 +573,16 @@ The diagnostics object has four fields: | Field | Value | | --- | --- | | `prompt` | The project description the user typed. | -| `createdAt` | ISO‑8601 timestamp of when the project was first prompted. | +| `createdAt` | ISO‑8601 timestamp of when the create-project view flow started. | | `systemInfo` | The operating system, CPU, Node.js, and VS Code versions captured when the project started. | | `diagnosticEvents` | Up to the **75 most recent** events, each: `timestamp`, `name` (command/tool), `type` (`extensionAction` \| `mcpTool` \| `webviewAction`), `status` (`start` \| `success` \| `error`), and a `properties` bag. Error messages are **masked** before being recorded. | `report_agent_launch` contributes at most one diagnostic lifecycle for each agent in a chat session. +Starting the create-project view flow resets cached CoR project state before Copilot readiness checks +and model discovery. Pressing **Plan** records the submitted prompt without resetting that state, +so the model-discovery diagnostics remain available alongside the planning and later-stage events. + Model discovery adds these properties to the `createProjectWithCopilot` event and CoR telemetry: | Property | Meaning | diff --git a/src/utils/copilotOnRails/diagnosticUtils.ts b/src/utils/copilotOnRails/diagnosticUtils.ts index 44f01f5b4..d1bb01542 100644 --- a/src/utils/copilotOnRails/diagnosticUtils.ts +++ b/src/utils/copilotOnRails/diagnosticUtils.ts @@ -24,7 +24,7 @@ export function getPrompt(): string | undefined { const createdAtKey: string = 'copilotOnRails.createdAt'; /** - * Stamps the current time for workspace project diagnostics metadata, as an ISO 8601 string, as the moment the project was first prompted. + * Stamps the time the create-project view flow started as an ISO 8601 string. */ export function recordCreatedAt(): void { void ext.context.workspaceState.update(createdAtKey, new Date().toISOString()); diff --git a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts index d479072ae..9ceea3415 100644 --- a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts +++ b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts @@ -11,7 +11,7 @@ import { azureProjectPlanAgent } from "../../../../constants"; import { ext } from "../../../../extensionVariables"; import { projectSubmissionState } from "../../../../tree/project/projectSubmissionState"; import { CopilotOnRailsContext } from "../../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { prepareNewCorProject } from "../../../../utils/copilotOnRails/prepareNewCorProject"; +import { recordPrompt } from "../../../../utils/copilotOnRails/diagnosticUtils"; import { callWithDiagnosticsAndTelemetryHandling, corId, setCorProp } from "../../../../utils/copilotOnRails/telemetryUtils"; import { type CreateProjectViewControllerType, type LoadingViewConfiguration } from "../../views/utils/viewConfigTypes"; import { getCopilotOnRailsBundleLocation } from "../copilotOnRailsBundleLocation"; @@ -41,12 +41,8 @@ export class CreateProjectViewController extends CopilotOnRailsWebviewController ); } - /** - * Resets any leftover workspace state from a previous project before launching the - * planning agent, so the two attempts don't get conflated. - */ private async planProject(prompt: string, model?: string): Promise { - await prepareNewCorProject(prompt); + recordPrompt(prompt); await this.openChatWithQuery(prompt, model); } diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 779a9e6d1..2b26a102e 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -10,6 +10,7 @@ import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/regis import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext, ensureRequiredCopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; import { AvailableChatModel, getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; +import { prepareNewCorProject } from "../../../utils/copilotOnRails/prepareNewCorProject"; import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; @@ -75,6 +76,7 @@ export async function resumeCreateProjectViewAfterReload(): Promise { } async function openCreateProjectView(context: CopilotOnRailsContext, initialPrompt?: string, initialModel?: string): Promise { + await prepareNewCorProject(initialPrompt ?? ''); if (!(await ensureCopilotChatReady(context))) { return; } From 302fecd56f397d7ce0f5a8ba1632f2e58692475e Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:45:34 -0700 Subject: [PATCH 24/34] Simplify model discovery diagnostics and defer project ID --- docs/copilot-create-project.md | 12 +-- .../registerDebugSessionWatcher.ts | 5 +- src/utils/copilotOnRails/telemetryUtils.ts | 19 +++-- .../CreateProjectViewController.ts | 3 +- .../controllers/LoadingViewController.ts | 5 +- .../extension/createProjectWithCopilot.ts | 11 +-- .../extension/openRequirementsView.ts | 5 +- .../createProjectWithCopilot.test.ts | 76 ++++++++++++++++--- 8 files changed, 100 insertions(+), 36 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index d952c5e67..1fcef2133 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -582,20 +582,20 @@ The diagnostics object has four fields: Starting the create-project view flow resets cached CoR project state before Copilot readiness checks and model discovery. Pressing **Plan** records the submitted prompt without resetting that state, so the model-discovery diagnostics remain available alongside the planning and later-stage events. +The telemetry-only `corProjectId` is created when the prompt is submitted, not when the create view +loads or models are discovered. Subsequent project events reuse it. Model discovery adds these properties to the `createProjectWithCopilot` event and CoR telemetry: | Property | Meaning | | --- | --- | | `modelPickerVendor` | The queried vendor, `copilotcli`. | -| `modelPickerReturnedModelCount` | Number of models returned by that vendor. | -| `modelPickerSupportedModelCount` | Number matching the supported-family filter, before duplicate display names are removed. | -| `modelPickerFilteredOutModelCount` | Number excluded by the supported-family filter. | -| `availableModelCount` | Number of distinct options shown in the picker, or zero when the prompt view cannot open. | +| `modelPickerAvailableModelCount` | Number of entries in `modelPickerAvailableModels`, before filtering. | +| `modelPickerFilteredModelCount` | Number of entries in `modelPickerFilteredModels`, after filtering and deduplication. | | `modelPickerOutcome` | `querying` before discovery completes, `emptyCatalog` for no returned models, `noSupportedModels` when none pass the filter, or `ready` when options are available. | -Diagnostics also include `modelPickerReturnedModels` with each returned model's ID, name, vendor, -family, and version, and `modelPickerShownModels` with the displayed option names. These catalogs +Diagnostics also include `modelPickerAvailableModels` with the returned model names before filtering, +and `modelPickerFilteredModels` with the displayed model names after filtering and deduplication. These catalogs are workspace-cached diagnostic data only and are not sent to telemetry. The counts and discovery outcome are mirrored to telemetry separately through the standard CoR property handling. diff --git a/src/commands/copilotOnRails/registerDebugSessionWatcher.ts b/src/commands/copilotOnRails/registerDebugSessionWatcher.ts index 9bfc63ac4..242ffbe8e 100644 --- a/src/commands/copilotOnRails/registerDebugSessionWatcher.ts +++ b/src/commands/copilotOnRails/registerDebugSessionWatcher.ts @@ -191,7 +191,10 @@ function computeOutcome(tracked: TrackedDebugSession): 'succeeded' | 'failed' | function reportDebugSessionEvent(eventName: string, properties: Record, measurements?: Record): void { void callWithTelemetryAndErrorHandling(eventName, async (context: IActionContext) => { context.telemetry.properties.isCopilotEvent = 'true'; - context.telemetry.properties.corProjectId = getCorProjectId(); + const projectId = getCorProjectId(); + if (projectId) { + context.telemetry.properties.corProjectId = projectId; + } Object.assign(context.telemetry.properties, properties); if (measurements) { Object.assign(context.telemetry.measurements, measurements); diff --git a/src/utils/copilotOnRails/telemetryUtils.ts b/src/utils/copilotOnRails/telemetryUtils.ts index defb98951..2c38fce15 100644 --- a/src/utils/copilotOnRails/telemetryUtils.ts +++ b/src/utils/copilotOnRails/telemetryUtils.ts @@ -26,16 +26,16 @@ const projectIdKey: string = 'copilotOnRails.projectId'; /** * Returns the persistent project guid associated with the workspace project. - * Automatically handles generating and caching one on first use. * * Note: The project guid is important for stitching together the full chain of Copilot on Rails telemetry. */ -export function getCorProjectId(): string { - let projectId: string | undefined = ext.context.workspaceState.get(projectIdKey); - if (!projectId) { - projectId = uuidv4(); - void ext.context.workspaceState.update(projectIdKey, projectId); - } +export function getCorProjectId(): string | undefined { + return ext.context.workspaceState.get(projectIdKey); +} + +export async function initializeCorProjectId(): Promise { + const projectId = getCorProjectId() ?? uuidv4(); + await ext.context.workspaceState.update(projectIdKey, projectId); return projectId; } @@ -51,7 +51,10 @@ export async function callWithDiagnosticsAndTelemetryHandling( command: (context: CopilotOnRailsContext) => Promise, ): Promise { context.telemetry.properties.isCopilotEvent = 'true'; - context.telemetry.properties.corProjectId = getCorProjectId(); + const projectId = getCorProjectId(); + if (projectId && eventDetails.name !== corId('createProjectWithCopilot')) { + context.telemetry.properties.corProjectId = projectId; + } if (eventDetails.extras) { context.telemetry.properties.copilotSessionId = eventDetails.extras.sessionId; diff --git a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts index 9ceea3415..ad4b431ef 100644 --- a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts +++ b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts @@ -12,7 +12,7 @@ import { ext } from "../../../../extensionVariables"; import { projectSubmissionState } from "../../../../tree/project/projectSubmissionState"; import { CopilotOnRailsContext } from "../../../../utils/copilotOnRails/CopilotOnRailsContext"; import { recordPrompt } from "../../../../utils/copilotOnRails/diagnosticUtils"; -import { callWithDiagnosticsAndTelemetryHandling, corId, setCorProp } from "../../../../utils/copilotOnRails/telemetryUtils"; +import { callWithDiagnosticsAndTelemetryHandling, corId, initializeCorProjectId, setCorProp } from "../../../../utils/copilotOnRails/telemetryUtils"; import { type CreateProjectViewControllerType, type LoadingViewConfiguration } from "../../views/utils/viewConfigTypes"; import { getCopilotOnRailsBundleLocation } from "../copilotOnRailsBundleLocation"; import { openLoadingView } from "../openLoadingView"; @@ -43,6 +43,7 @@ export class CreateProjectViewController extends CopilotOnRailsWebviewController private async planProject(prompt: string, model?: string): Promise { recordPrompt(prompt); + await initializeCorProjectId(); await this.openChatWithQuery(prompt, model); } diff --git a/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts b/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts index 4562e8383..7202cf594 100644 --- a/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts +++ b/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts @@ -40,7 +40,10 @@ export class LoadingViewController extends CopilotOnRailsWebviewController { context.errorHandling.suppressDisplay = true; context.telemetry.properties.isCopilotEvent = 'true'; - context.telemetry.properties.corProjectId = getCorProjectId(); + const projectId = getCorProjectId(); + if (projectId) { + context.telemetry.properties.corProjectId = projectId; + } this.panel.dispose(); await vscode.commands.executeCommand(copilotOnRailsCommandIds.resumeProjectWithCopilot); }); diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 2b26a102e..c15e66516 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -151,16 +151,13 @@ export function recordModelPickerDiscovery( models: readonly AvailableChatModel[], modelOptions: readonly string[], ): void { - const supportedModelCount = models.filter(model => getSupportedModelName(model.id, model.family, model.name) !== undefined).length; - setCorProp(context, 'modelPickerReturnedModelCount', models.length); - setCorProp(context, 'modelPickerSupportedModelCount', supportedModelCount); - setCorProp(context, 'modelPickerFilteredOutModelCount', models.length - supportedModelCount); - setCorProp(context, 'availableModelCount', modelOptions.length); + setCorProp(context, 'modelPickerAvailableModelCount', models.length); + setCorProp(context, 'modelPickerFilteredModelCount', modelOptions.length); setCorProp(context, 'modelPickerOutcome', models.length === 0 ? 'emptyCatalog' : modelOptions.length === 0 ? 'noSupportedModels' : 'ready'); const properties = ensureRequiredCopilotOnRailsContext(context).diagnostics.properties; - properties.modelPickerReturnedModels = models.map(({ id, name, vendor, family, version }) => ({ id, name, vendor, family, version })); - properties.modelPickerShownModels = [...modelOptions]; + properties.modelPickerAvailableModels = models.map(model => model.name); + properties.modelPickerFilteredModels = [...modelOptions]; } /** diff --git a/src/webviews/copilotOnRails/extension/openRequirementsView.ts b/src/webviews/copilotOnRails/extension/openRequirementsView.ts index fe7526547..8260221e6 100644 --- a/src/webviews/copilotOnRails/extension/openRequirementsView.ts +++ b/src/webviews/copilotOnRails/extension/openRequirementsView.ts @@ -80,7 +80,10 @@ export async function openRequirementsViewFromWorkspace(context: IActionContext) ); context.telemetry.properties.isCopilotEvent = 'true'; - context.telemetry.properties.corProjectId = getCorProjectId(); + const projectId = getCorProjectId(); + if (projectId) { + context.telemetry.properties.corProjectId = projectId; + } context.telemetry.properties.requirementsSelected = String(!!selected); if (selected) { await openRequirementsViewAsync(selected); diff --git a/test/copilotOnRails/createProjectWithCopilot.test.ts b/test/copilotOnRails/createProjectWithCopilot.test.ts index 6969de49d..4147e8583 100644 --- a/test/copilotOnRails/createProjectWithCopilot.test.ts +++ b/test/copilotOnRails/createProjectWithCopilot.test.ts @@ -5,8 +5,10 @@ import { createTestActionContext } from '@microsoft/vscode-azext-utils'; import assert from 'assert'; +import { ext } from '../../src/extensionVariables'; import { ensureRequiredCopilotOnRailsContext } from '../../src/utils/copilotOnRails/CopilotOnRailsContext'; import { AvailableChatModel, getSupportedModelOptions } from '../../src/utils/copilotOnRails/modelSelection'; +import { callWithDiagnosticsAndTelemetryHandling, getCorProjectId, initializeCorProjectId } from '../../src/utils/copilotOnRails/telemetryUtils'; import { OPEN_PROJECT_FOLDER_OPTIONS, PROJECT_FOLDER_SELECTION_TELEMETRY_KEY, @@ -45,6 +47,56 @@ suite('Create Project with Copilot folder selection', () => { }); suite('Create Project with Copilot model picker diagnostics', () => { + test('leaves discovery unassigned and initializes the project id at prompt submission', async () => { + const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); + const state = new Map(); + Object.defineProperty(ext, 'context', { + configurable: true, + value: { + globalState: { get: () => false }, + workspaceState: { + get: (key: string, defaultValue?: unknown) => state.get(key) ?? defaultValue, + update: async (key: string, value: unknown) => { state.set(key, value); }, + }, + }, + }); + try { + const discoveryContext = await createTestActionContext(); + await callWithDiagnosticsAndTelemetryHandling( + discoveryContext, + { type: 'extensionAction', name: 'copilotOnRails.createProjectWithCopilot' }, + async () => { assert.strictEqual(getCorProjectId(), undefined); }, + ); + assert.strictEqual(getCorProjectId(), undefined); + assert.strictEqual(discoveryContext.telemetry.properties.corProjectId, undefined); + + const projectId = await initializeCorProjectId(); + assert.ok(projectId); + const submitContext = await createTestActionContext(); + await callWithDiagnosticsAndTelemetryHandling( + submitContext, + { type: 'webviewAction', name: 'createProjectSubmitPrompt' }, + async () => { assert.strictEqual(getCorProjectId(), projectId); }, + ); + assert.strictEqual(submitContext.telemetry.properties.corProjectId, projectId); + assert.strictEqual(getCorProjectId(), projectId); + assert.strictEqual(await initializeCorProjectId(), projectId); + const nextCreateContext = await createTestActionContext(); + await callWithDiagnosticsAndTelemetryHandling( + nextCreateContext, + { type: 'extensionAction', name: 'copilotOnRails.createProjectWithCopilot' }, + async () => { assert.strictEqual(getCorProjectId(), projectId); }, + ); + assert.strictEqual(nextCreateContext.telemetry.properties.corProjectId, undefined); + } finally { + if (originalContext) { + Object.defineProperty(ext, 'context', originalContext); + } else { + Reflect.deleteProperty(ext, 'context'); + } + } + }); + const sonnet: AvailableChatModel = { id: 'claude-sonnet-5', vendor: 'copilotcli', @@ -61,9 +113,9 @@ suite('Create Project with Copilot model picker diagnostics', () => { }; for (const scenario of [ - { name: 'empty catalog', models: [], supported: 0, excluded: 0, shown: 0, outcome: 'emptyCatalog' }, - { name: 'unsupported catalog', models: [haiku], supported: 0, excluded: 1, shown: 0, outcome: 'noSupportedModels' }, - { name: 'supported catalog with duplicate labels', models: [sonnet, haiku, sonnet], supported: 2, excluded: 1, shown: 1, outcome: 'ready' }, + { name: 'empty catalog', models: [], filtered: 0, outcome: 'emptyCatalog' }, + { name: 'unsupported catalog', models: [haiku], filtered: 0, outcome: 'noSupportedModels' }, + { name: 'supported catalog with duplicate labels', models: [sonnet, haiku, sonnet], filtered: 1, outcome: 'ready' }, ]) { test(`records counts and outcome for ${scenario.name} in telemetry and diagnostics`, async () => { const context = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); @@ -72,20 +124,22 @@ suite('Create Project with Copilot model picker diagnostics', () => { recordModelPickerDiscovery(context, scenario.models, options); const expected = { - modelPickerReturnedModelCount: scenario.models.length, - modelPickerSupportedModelCount: scenario.supported, - modelPickerFilteredOutModelCount: scenario.excluded, - availableModelCount: scenario.shown, + modelPickerAvailableModelCount: scenario.models.length, + modelPickerFilteredModelCount: scenario.filtered, modelPickerOutcome: scenario.outcome, }; for (const [key, value] of Object.entries(expected)) { assert.strictEqual(context.telemetry.properties[key], String(value)); assert.strictEqual(context.diagnostics.properties[key], value); } - assert.deepStrictEqual(context.diagnostics.properties.modelPickerReturnedModels, scenario.models); - assert.deepStrictEqual(context.diagnostics.properties.modelPickerShownModels, options); - assert.strictEqual(context.telemetry.properties.modelPickerReturnedModels, undefined); - assert.strictEqual(context.telemetry.properties.modelPickerShownModels, undefined); + assert.deepStrictEqual(context.diagnostics.properties.modelPickerAvailableModels, scenario.models.map(model => model.name)); + assert.deepStrictEqual(context.diagnostics.properties.modelPickerFilteredModels, options); + assert.strictEqual(context.telemetry.properties.modelPickerAvailableModels, undefined); + assert.strictEqual(context.telemetry.properties.modelPickerFilteredModels, undefined); + for (const key of ['modelPickerReturnedModelCount', 'modelPickerSupportedModelCount', 'modelPickerFilteredOutModelCount', 'availableModelCount', 'modelPickerReturnedModels', 'modelPickerShownModels']) { + assert.strictEqual(context.telemetry.properties[key], undefined); + assert.strictEqual(context.diagnostics.properties[key], undefined); + } }); } }); From 53ef4dc2a453fd8f72baceb032a6b8670c1c8d31 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:51:32 -0700 Subject: [PATCH 25/34] Allow default-model planning with one-time discovery retry --- docs/copilot-create-project.md | 26 +++++-- .../copilotOnRails/openChatWithAgent.ts | 11 +-- src/utils/copilotOnRails/modelSelection.ts | 6 ++ .../CreateProjectViewController.ts | 4 +- .../extension/createProjectWithCopilot.ts | 69 +++++++++++-------- .../views/CreateProjectView.tsx | 25 ++++--- .../views/utils/viewConfigTypes.ts | 1 + .../createProjectWithCopilot.test.ts | 50 ++++++++++++++ test/copilotOnRails/modelSelection.test.ts | 19 +++++ test/copilotOnRails/openChatWithAgent.test.ts | 46 ++++++++++++- 10 files changed, 207 insertions(+), 50 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 1fcef2133..b4b62981a 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -142,11 +142,16 @@ session** running the next agent. Between hand‑offs, agents open **webviews** GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, using their display names without a vendor suffix. The lowest-version available Opus model is selected by default, or the first supported option when no Opus model is available. The extension activates GitHub Copilot Chat before loading - this list. If no supported models are available, it explains that Copilot models may still be loading - and asks the user to wait a moment, then click **Create New Project With Copilot** again. If the problem - persists, the message advises checking Copilot sign-in and organization model policies. + this list. If the API returns no models, the extension retries once after one second. This retry + budget resets only when the extension host reloads, not each time the create flow runs. + If discovery fails or returns no supported models, the view still opens without a model picker + or warning. The user can submit the plan normally using VS Code's default model. When the Copilot Harness starts, the extension passes the selected model's actual ID and `copilotcli` vendor to Chat without remapping it or constructing an Agent Host fallback. + **VS Code default** is stored as `default` in the session and model telemetry. It omits the model + selector on every subsequent launch or chat invocation, leaving model choice to VS Code. + This selection is preserved if the create view reopens after a reload, even when models have + become available in the meantime. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without @@ -183,6 +188,12 @@ description, optionally pick a **Model**, and press **Plan** (or `Ctrl+Enter`). Create with Copilot prompt view

+> 📷 First capture needed: the prompt view without a model picker when model discovery is unavailable. + +

+ Create with Copilot prompt view without a model picker +

+ Pressing **Plan** opens Copilot Chat if needed, then starts the **`azure-project-plan`** agent in a new chat session. Every new-project prompt goes through this flow, including frontend-only apps with no backend, database, or Azure services. Those simply plan a single `frontend` service with **No datastore required**. @@ -592,12 +603,17 @@ Model discovery adds these properties to the `createProjectWithCopilot` event an | `modelPickerVendor` | The queried vendor, `copilotcli`. | | `modelPickerAvailableModelCount` | Number of entries in `modelPickerAvailableModels`, before filtering. | | `modelPickerFilteredModelCount` | Number of entries in `modelPickerFilteredModels`, after filtering and deduplication. | -| `modelPickerOutcome` | `querying` before discovery completes, `emptyCatalog` for no returned models, `noSupportedModels` when none pass the filter, or `ready` when options are available. | +| `modelPickerRetried` | Whether this invocation used the one-per-extension-host-session empty-catalog retry. | +| `modelPickerOutcome` | `querying` before discovery completes, `emptyCatalog` for no returned models, `noSupportedModels` when none pass the filter, `queryFailed` when the API throws, or `ready` when supported models are available. | Diagnostics also include `modelPickerAvailableModels` with the returned model names before filtering, and `modelPickerFilteredModels` with the displayed model names after filtering and deduplication. These catalogs are workspace-cached diagnostic data only and are not sent to telemetry. The counts and discovery outcome are mirrored to telemetry separately through the standard CoR property handling. +The `default` sentinel is not counted as an available or filtered model. +Discovery failures record a masked `modelPickerError`. Submitting with the fallback records +`copilotModel: default`, `modelSelectedInView: false`, and `chatModelSelectionSource: default`; +chat options omit `modelSelector`. Privacy guarantees, by design: @@ -617,7 +633,7 @@ before submitting. | --- | --- | --- | | *"Choose where to create your project."* | The open folder isn't empty. | Choose **Create in New Subfolder…** to build under the current folder, or **Choose Empty Folder…** to build elsewhere. Both open the project in a separate window. | | The new project window opens in Restricted Mode and the Azure Project view is unavailable. | Workspace Trust must be granted explicitly; extensions cannot trust a folder on your behalf. | Select **Trust** from the Restricted Mode banner or Workspace Trust editor. The pending create flow resumes automatically after the extension activates. | -| The create flow says Copilot models may still be loading. | The CLI model catalog is empty or contains no supported models yet. | Wait a moment and click **Create New Project With Copilot** again. Inspect the discovery properties described above to distinguish an empty catalog from models excluded by the family filter and compare returned models with displayed options. If the problem persists, check Copilot sign-in and organization model policies. | +| The create view has no model picker. | The CLI model catalog is empty, has no supported models, or discovery failed. | Submit the plan normally; VS Code chooses the model without an extension override. Inspect the discovery properties described above to distinguish the cases, including `modelPickerError` when the API failed and `modelPickerRetried` for the one-time retry. | | An agent says it needs its instruction files, or behaves oddly / follows outdated steps. | `.github/agents/` is missing or stale. | Accept the download prompt, or run **Download Azure Agent Instructions**. The version stamp auto‑refreshes stale copies. | | Chat opens with the wrong agent or generic Agent mode. | VS Code did not honor the requested custom mode, the custom instructions were not loaded, or the MCP tool was unavailable. | Inspect `diagnosticEvents` for a successful `report_agent_launch` event. Its `agentName` property identifies the agent that reported. If the event is missing, the startup report never reached the CoR MCP server. | | Frontend preview stuck on *"Starting…"*; **Approve UI** never enables (but the app loads in a normal browser). | A second dev server is contending for the preview port. | Stop **all** manually‑started dev servers, free the port, ensure the frontend's `vite.config` is the clean minimal version, then reopen the preview and let it own the server. Don't verify by starting your own server. | diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index 79cd1bb07..f210dfcfe 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -8,7 +8,7 @@ import * as vscode from 'vscode'; import { ext } from '../../extensionVariables'; import { projectSubmissionState } from '../../tree/project/projectSubmissionState'; import { CopilotOnRailsContext } from '../../utils/copilotOnRails/CopilotOnRailsContext'; -import { resolveAvailableChatModel } from '../../utils/copilotOnRails/modelSelection'; +import { DEFAULT_CHAT_MODEL, resolveAvailableChatModel } from '../../utils/copilotOnRails/modelSelection'; import { setCorErrorProp, setCorProp } from '../../utils/copilotOnRails/telemetryUtils'; import { AUTOPILOT_QUERY_MARKER, isAutopilotActive } from '../../webviews/copilotOnRails/extension/autopilot'; import { ensureCopilotHarnessOn } from '../../webviews/copilotOnRails/extension/harnessSettings'; @@ -42,7 +42,10 @@ export function registerWorkspaceTrustTracking(): void { * Resolves a user-facing model name (e.g. "Claude Opus 4.7") to a * modelSelector object that VS Code's chat commands expect. */ -async function resolveModelSelector(displayName: string): Promise<{ id?: string; vendor?: string } | undefined> { +export async function resolveModelSelector(displayName: string): Promise<{ id?: string; vendor?: string } | undefined> { + if (displayName === DEFAULT_CHAT_MODEL) { + return undefined; + } try { const models = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); const match = resolveAvailableChatModel(displayName, models); @@ -143,7 +146,7 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: ); const resolvedModel = model ?? getSessionModel(); - setCorProp(context, 'chatModelSelectionSource', model ? 'newlySelected' : (getSessionModel() ? 'previouslySelected' : 'default')); + setCorProp(context, 'chatModelSelectionSource', resolvedModel === DEFAULT_CHAT_MODEL ? 'default' : model ? 'newlySelected' : (getSessionModel() ? 'previouslySelected' : 'default')); const selector = resolvedModel ? await resolveModelSelector(resolvedModel) : undefined; setCorProp(context, 'chatModelResolved', !!selector); @@ -343,7 +346,7 @@ export async function buildChatOpenOptions(context: CopilotOnRailsContext, optio } const model = getSessionModel(); - setCorProp(context, 'chatModelSelectionSource', model ? 'previouslySelected' : 'default'); + setCorProp(context, 'chatModelSelectionSource', model && model !== DEFAULT_CHAT_MODEL ? 'previouslySelected' : 'default'); if (model) { const selector = await resolveModelSelector(model); setCorProp(context, 'chatModelResolved', !!selector); diff --git a/src/utils/copilotOnRails/modelSelection.ts b/src/utils/copilotOnRails/modelSelection.ts index 1c33d50f4..7e5ad209c 100644 --- a/src/utils/copilotOnRails/modelSelection.ts +++ b/src/utils/copilotOnRails/modelSelection.ts @@ -12,6 +12,12 @@ export interface AvailableChatModel { } export const supportedModelNames = ['Opus', 'Sonnet', 'GPT Sol', 'GPT Terra'] as const; +export const DEFAULT_CHAT_MODEL = 'default'; + +export function getModelPickerOptions(models: readonly AvailableChatModel[], initialModel?: string): string[] { + const options = getSupportedModelOptions(models); + return options.length > 0 && initialModel === DEFAULT_CHAT_MODEL ? [DEFAULT_CHAT_MODEL, ...options] : options; +} export function getSupportedModelName(...identifiers: string[]): typeof supportedModelNames[number] | undefined { const modelIdentifiers = identifiers.join(' '); diff --git a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts index ad4b431ef..fd16e1b03 100644 --- a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts +++ b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts @@ -12,6 +12,7 @@ import { ext } from "../../../../extensionVariables"; import { projectSubmissionState } from "../../../../tree/project/projectSubmissionState"; import { CopilotOnRailsContext } from "../../../../utils/copilotOnRails/CopilotOnRailsContext"; import { recordPrompt } from "../../../../utils/copilotOnRails/diagnosticUtils"; +import { DEFAULT_CHAT_MODEL } from "../../../../utils/copilotOnRails/modelSelection"; import { callWithDiagnosticsAndTelemetryHandling, corId, initializeCorProjectId, setCorProp } from "../../../../utils/copilotOnRails/telemetryUtils"; import { type CreateProjectViewControllerType, type LoadingViewConfiguration } from "../../views/utils/viewConfigTypes"; import { getCopilotOnRailsBundleLocation } from "../copilotOnRailsBundleLocation"; @@ -50,9 +51,10 @@ export class CreateProjectViewController extends CopilotOnRailsWebviewController private async openChatWithQuery(query: string, model?: string): Promise { await callWithTelemetryAndErrorHandling(corId('createProjectSubmitPrompt'), async (actionContext: IActionContext) => { await callWithDiagnosticsAndTelemetryHandling(actionContext, { type: 'webviewAction', name: 'createProjectSubmitPrompt' }, async (context: CopilotOnRailsContext) => { - setCorProp(context, 'modelSelectedInView', !!model); + setCorProp(context, 'modelSelectedInView', !!model && model !== DEFAULT_CHAT_MODEL); if (model) { await recordModel(model); + setCorProp(context, 'copilotModel', model); } await recordRecentPrompt(query); diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index c15e66516..1af2a9193 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -3,15 +3,15 @@ * Licensed under the MIT License. See License.md in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { AzExtFsExtra, UserCancelledError } from "@microsoft/vscode-azext-utils"; +import { AzExtFsExtra, parseError, UserCancelledError } from "@microsoft/vscode-azext-utils"; import * as vscode from 'vscode'; import { ensureCopilotChatReady } from "../../../commands/copilotOnRails/openChatWithAgent"; import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/registerCopilotOnRailsCommands"; import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext, ensureRequiredCopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { AvailableChatModel, getDefaultOpusModelOption, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; +import { AvailableChatModel, DEFAULT_CHAT_MODEL, getDefaultOpusModelOption, getModelPickerOptions, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; import { prepareNewCorProject } from "../../../utils/copilotOnRails/prepareNewCorProject"; -import { setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; +import { setCorErrorProp, setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; import { consumeReloadResumePrompt } from "./reloadResumePrompt"; @@ -22,6 +22,8 @@ const deploy = vscode.l10n.t('Deploy'); export const OPEN_PROJECT_FOLDER_OPTIONS = { forceNewWindow: true } as const; export const PROJECT_FOLDER_SELECTION_TELEMETRY_KEY = 'projectFolderSelection'; export type ProjectFolderSelection = 'newSubfolder' | 'selectedEmptyFolder'; +const MODEL_DISCOVERY_RETRY_DELAY_MS = 1_000; +let modelDiscoveryRetryUsed = false; export async function createProjectWithCopilot(context: CopilotOnRailsContext, initialPrompt?: string, initialModel?: string): Promise { if (!(await ensureFreshWorkspace(context))) { @@ -82,22 +84,10 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm } console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilotcli".'); - setCorProp(context, 'modelPickerVendor', 'copilotcli'); - setCorProp(context, 'modelPickerOutcome', 'querying'); - const availableModels = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); - const modelOptions = getSupportedModelOptions(availableModels); - recordModelPickerDiscovery(context, availableModels, modelOptions); - const allModels = await vscode.lm.selectChatModels(); - const modelCountsByVendor = allModels.reduce>((counts, model) => { - counts[model.vendor] = (counts[model.vendor] ?? 0) + 1; - return counts; - }, {}); - console.log( - '[Copilot on Rails model picker] Registered model counts by vendor:', - modelCountsByVendor, - ); + const availableModels = await discoverModelPickerModels(context, () => vscode.lm.selectChatModels({ vendor: 'copilotcli' })); + const modelOptions = getModelPickerOptions(availableModels, initialModel); console.log( - `[Copilot on Rails model picker] selectChatModels returned ${availableModels.length} model(s):`, + `[Copilot on Rails model picker] Available catalog contains ${availableModels.length} model(s):`, availableModels.map(model => ({ id: model.id, vendor: model.vendor, @@ -110,21 +100,13 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm ); console.log( - `[Copilot on Rails model picker] Supported-family filtering produced ${modelOptions.length} picker option(s):`, + `[Copilot on Rails model picker] Discovery produced ${modelOptions.length} picker option(s):`, modelOptions, ); - if (modelOptions.length === 0) { - console.warn('[Copilot on Rails model picker] No supported Copilot CLI models are available.'); - void vscode.window.showErrorMessage(vscode.l10n.t( - 'Copilot models may still be loading. Wait a moment, then click Create New Project With Copilot again. If the problem persists, check your Copilot sign-in and organization model policies.', - )); - return; - } - - const selectedModel = initialModel && modelOptions.includes(initialModel) + const selectedModel = initialModel && (initialModel === DEFAULT_CHAT_MODEL || modelOptions.includes(initialModel)) ? initialModel - : getDefaultOpusModelOption(availableModels) ?? modelOptions[0]; + : getDefaultOpusModelOption(availableModels) ?? modelOptions[0] ?? DEFAULT_CHAT_MODEL; console.log('[Copilot on Rails model picker] Initial and selected model:', { requestedInitialModel: initialModel, selectedModel, @@ -139,6 +121,7 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm planButtonLabel: vscode.l10n.t('Plan'), modelLabel: vscode.l10n.t('Model'), modelOptions, + defaultModelLabel: vscode.l10n.t('VS Code default'), recentPrompts: getRecentPrompts(), initialPrompt, initialModel: selectedModel, @@ -146,6 +129,34 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm controller.revealToForeground(); } +export async function discoverModelPickerModels( + context: CopilotOnRailsContext, + selectModels: () => Thenable, +): Promise { + setCorProp(context, 'modelPickerVendor', 'copilotcli'); + setCorProp(context, 'modelPickerOutcome', 'querying'); + setCorProp(context, 'modelPickerRetried', false); + let models: readonly T[]; + try { + models = await selectModels(); + if (models.length === 0 && !modelDiscoveryRetryUsed) { + modelDiscoveryRetryUsed = true; + setCorProp(context, 'modelPickerRetried', true); + console.log('[Copilot on Rails model picker] Empty catalog; retrying once after one second.'); + await new Promise(resolve => setTimeout(resolve, MODEL_DISCOVERY_RETRY_DELAY_MS)); + models = await selectModels(); + } + } catch (error) { + setCorErrorProp(context, 'modelPickerError', parseError(error).message); + recordModelPickerDiscovery(context, [], []); + setCorProp(context, 'modelPickerOutcome', 'queryFailed'); + console.log('[Copilot on Rails model picker] Model discovery failed; using the VS Code default. See modelPickerError in diagnostics.'); + return []; + } + recordModelPickerDiscovery(context, models, getSupportedModelOptions(models)); + return models; +} + export function recordModelPickerDiscovery( context: CopilotOnRailsContext, models: readonly AvailableChatModel[], diff --git a/src/webviews/copilotOnRails/views/CreateProjectView.tsx b/src/webviews/copilotOnRails/views/CreateProjectView.tsx index 947d4a6bb..62b07098a 100644 --- a/src/webviews/copilotOnRails/views/CreateProjectView.tsx +++ b/src/webviews/copilotOnRails/views/CreateProjectView.tsx @@ -8,6 +8,7 @@ import { ClipboardTaskListLtrRegular } from '@fluentui/react-icons'; import { useConfiguration, WebviewContext } from '@microsoft/vscode-azext-webview/webview'; import * as React from 'react'; import { useContext, useLayoutEffect, useRef, useState, type JSX } from 'react'; +import { DEFAULT_CHAT_MODEL } from '../../../utils/copilotOnRails/modelSelection'; import './styles/createProjectView.scss'; import { type CreateProjectViewControllerType } from './utils/viewConfigTypes'; @@ -40,7 +41,9 @@ export const CreateProjectView = (): JSX.Element => { } }, [prompt]); - const displayName = (model: string) => model.replace(/\s*\(copilot\)\s*$/i, ''); + const displayName = (model: string) => model === DEFAULT_CHAT_MODEL + ? config.defaultModelLabel + : model.replace(/\s*\(copilot\)\s*$/i, ''); const planClicked = () => { if (!prompt.trim()) { @@ -140,15 +143,17 @@ export const CreateProjectView = (): JSX.Element => { />
- + {config.modelOptions.length > 0 && ( + + )} {config.hint}
diff --git a/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts b/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts index b9bd0f539..a891e9735 100644 --- a/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts +++ b/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts @@ -12,6 +12,7 @@ export type CreateProjectViewControllerType = { planButtonLabel: string; modelLabel: string; modelOptions: string[]; + defaultModelLabel: string; /** Recently submitted project-creation prompts, newest-first, for input-history navigation. */ recentPrompts: string[]; /** Pre-fills the prompt input, e.g. when re-opening the view after a reload-for-agent-discovery. */ diff --git a/test/copilotOnRails/createProjectWithCopilot.test.ts b/test/copilotOnRails/createProjectWithCopilot.test.ts index 4147e8583..7b36fe3bd 100644 --- a/test/copilotOnRails/createProjectWithCopilot.test.ts +++ b/test/copilotOnRails/createProjectWithCopilot.test.ts @@ -15,6 +15,7 @@ import { ProjectFolderSelection, recordProjectFolderSelection, recordModelPickerDiscovery, + discoverModelPickerModels, validateProjectSubfolderName, } from '../../src/webviews/copilotOnRails/extension/createProjectWithCopilot'; @@ -47,6 +48,55 @@ suite('Create Project with Copilot folder selection', () => { }); suite('Create Project with Copilot model picker diagnostics', () => { + test('retries an empty catalog once per extension-host session, not on later runs', async () => { + const firstContext = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); + let firstQueries = 0; + const models = await discoverModelPickerModels(firstContext, async () => { + firstQueries++; + return firstQueries === 1 ? [] : [sonnet]; + }); + assert.deepStrictEqual(models, [sonnet]); + assert.strictEqual(firstQueries, 2); + assert.strictEqual(firstContext.diagnostics.properties.modelPickerRetried, true); + assert.strictEqual(firstContext.telemetry.properties.modelPickerRetried, 'true'); + + const nextContext = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); + let nextQueries = 0; + const nextModels = await discoverModelPickerModels(nextContext, async () => { + nextQueries++; + return []; + }); + assert.deepStrictEqual(nextModels, []); + assert.strictEqual(nextQueries, 1); + assert.strictEqual(nextContext.diagnostics.properties.modelPickerRetried, false); + assert.strictEqual(nextContext.diagnostics.properties.modelPickerOutcome, 'emptyCatalog'); + }); + + test('records discovery failures without blocking the default-model fallback', async () => { + const context = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); + const models = await discoverModelPickerModels(context, async () => { throw new Error('Model provider unavailable'); }); + + assert.deepStrictEqual(models, []); + assert.strictEqual(context.diagnostics.properties.modelPickerOutcome, 'queryFailed'); + assert.strictEqual(context.telemetry.properties.modelPickerOutcome, 'queryFailed'); + assert.strictEqual(context.diagnostics.properties.modelPickerError, 'Model provider unavailable'); + assert.strictEqual(context.telemetry.properties.modelPickerError, 'Model provider unavailable'); + assert.strictEqual(context.diagnostics.properties.modelPickerAvailableModelCount, 0); + assert.strictEqual(context.diagnostics.properties.modelPickerFilteredModelCount, 0); + assert.deepStrictEqual(context.diagnostics.properties.modelPickerAvailableModels, []); + assert.deepStrictEqual(context.diagnostics.properties.modelPickerFilteredModels, []); + }); + + test('records available models when discovery succeeds', async () => { + const context = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); + const models = await discoverModelPickerModels(context, async () => [sonnet]); + + assert.deepStrictEqual(models, [sonnet]); + assert.strictEqual(context.diagnostics.properties.modelPickerOutcome, 'ready'); + assert.strictEqual(context.diagnostics.properties.modelPickerVendor, 'copilotcli'); + assert.strictEqual(context.diagnostics.properties.modelPickerError, undefined); + }); + test('leaves discovery unassigned and initializes the project id at prompt submission', async () => { const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); const state = new Map(); diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts index d99ae7f13..7cd66515c 100644 --- a/test/copilotOnRails/modelSelection.test.ts +++ b/test/copilotOnRails/modelSelection.test.ts @@ -6,6 +6,8 @@ import assert from 'assert'; import { AvailableChatModel, + DEFAULT_CHAT_MODEL, + getModelPickerOptions, resolveAvailableChatModel, } from '../../src/utils/copilotOnRails/modelSelection'; @@ -23,6 +25,23 @@ const cliModel: AvailableChatModel = { }; suite('Copilot on Rails model selection', () => { + test('provides no picker options when the catalog is empty', () => { + assert.deepStrictEqual(getModelPickerOptions([]), []); + }); + + test('provides no picker options when no models pass the filter', () => { + const model = { ...cliModel, id: 'haiku', name: 'Claude Haiku 4.5', family: 'haiku' }; + assert.deepStrictEqual(getModelPickerOptions([model]), []); + }); + + test('keeps the normal picker unchanged when supported models are available', () => { + assert.deepStrictEqual(getModelPickerOptions([cliModel]), [cliModel.name]); + }); + + test('preserves the default selection when reopening after models become available', () => { + assert.deepStrictEqual(getModelPickerOptions([cliModel], DEFAULT_CHAT_MODEL), [DEFAULT_CHAT_MODEL, cliModel.name]); + }); + test('resolves a qualified local model name', () => { assert.strictEqual( resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, cliModel]), diff --git a/test/copilotOnRails/openChatWithAgent.test.ts b/test/copilotOnRails/openChatWithAgent.test.ts index cffd31440..b0929ed12 100644 --- a/test/copilotOnRails/openChatWithAgent.test.ts +++ b/test/copilotOnRails/openChatWithAgent.test.ts @@ -4,9 +4,53 @@ *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; -import { buildAgentChatModeOptions, buildAgentChatOpenOptions } from '../../src/commands/copilotOnRails/openChatWithAgent'; +import { createTestActionContext } from '@microsoft/vscode-azext-utils'; +import { buildAgentChatModeOptions, buildAgentChatOpenOptions, buildChatOpenOptions, resolveModelSelector } from '../../src/commands/copilotOnRails/openChatWithAgent'; +import { ext } from '../../src/extensionVariables'; +import { DEFAULT_CHAT_MODEL } from '../../src/utils/copilotOnRails/modelSelection'; +import { getSessionModel, recordModel, recordPhase } from '../../src/webviews/copilotOnRails/extension/projectSession'; suite('Copilot on Rails agent chat launch', () => { + test('omits a model selector when the VS Code default is selected', async () => { + const selector = await resolveModelSelector(DEFAULT_CHAT_MODEL); + assert.deepStrictEqual( + buildAgentChatOpenOptions('azure-project-plan', 'Build a project', selector), + { mode: 'azure-project-plan', query: 'Build a project', waitForRequestAcceptance: true }, + ); + }); + + test('persists the default across phases and omits selectors on subsequent chat calls', async () => { + const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); + const state = new Map(); + Object.defineProperty(ext, 'context', { + configurable: true, + value: { + workspaceState: { + get: (key: string) => state.get(key), + update: async (key: string, value: unknown) => { state.set(key, value); }, + }, + }, + }); + try { + await recordModel(DEFAULT_CHAT_MODEL); + for (const phase of ['scaffold', 'integrate', 'debug', 'deploy'] as const) { + await recordPhase(phase); + assert.strictEqual(getSessionModel(), DEFAULT_CHAT_MODEL); + const context = await createTestActionContext(); + const options = { query: 'Continue the project' }; + assert.deepStrictEqual(await buildChatOpenOptions(context, options), options); + assert.strictEqual(context.telemetry.properties.chatModelSelectionSource, 'default'); + assert.strictEqual(context.telemetry.properties.chatModelResolved, 'false'); + } + } finally { + if (originalContext) { + Object.defineProperty(ext, 'context', originalContext); + } else { + Reflect.deleteProperty(ext, 'context'); + } + } + }); + test('builds options that select the custom agent without submitting a prompt', () => { assert.deepStrictEqual( buildAgentChatModeOptions('azure-project-integrate'), From 1839c74905b28519a09bc6ce0a15b879edf6c1f2 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:58:03 -0700 Subject: [PATCH 26/34] Show inline guidance when model discovery is unavailable --- docs/copilot-create-project.md | 12 +++++++----- .../extension/createProjectWithCopilot.ts | 1 + .../copilotOnRails/views/CreateProjectView.tsx | 6 +++++- .../views/styles/createProjectView.scss | 11 +++++++++++ .../copilotOnRails/views/utils/viewConfigTypes.ts | 1 + 5 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index b4b62981a..98b992ae6 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -144,8 +144,10 @@ session** running the next agent. Between hand‑offs, agents open **webviews** or the first supported option when no Opus model is available. The extension activates GitHub Copilot Chat before loading this list. If the API returns no models, the extension retries once after one second. This retry budget resets only when the extension host reloads, not each time the create flow runs. - If discovery fails or returns no supported models, the view still opens without a model picker - or warning. The user can submit the plan normally using VS Code's default model. + If discovery fails or returns no supported models, the view still opens without a model picker. + An inline message explains that VS Code's default will be used and suggests waiting a moment + before reopening the page to retry discovery. No error notification appears, and the user can + submit the plan normally. When the Copilot Harness starts, the extension passes the selected model's actual ID and `copilotcli` vendor to Chat without remapping it or constructing an Agent Host fallback. **VS Code default** is stored as `default` in the session and model telemetry. It omits the model @@ -188,10 +190,10 @@ description, optionally pick a **Model**, and press **Plan** (or `Ctrl+Enter`). Create with Copilot prompt view

-> 📷 First capture needed: the prompt view without a model picker when model discovery is unavailable. +> 📷 First capture needed: the prompt view with the inline model-unavailable message instead of the picker.

- Create with Copilot prompt view without a model picker + Create with Copilot prompt view with an inline model-unavailable message

Pressing **Plan** opens Copilot Chat if needed, then starts the **`azure-project-plan`** agent in a new chat @@ -633,7 +635,7 @@ before submitting. | --- | --- | --- | | *"Choose where to create your project."* | The open folder isn't empty. | Choose **Create in New Subfolder…** to build under the current folder, or **Choose Empty Folder…** to build elsewhere. Both open the project in a separate window. | | The new project window opens in Restricted Mode and the Azure Project view is unavailable. | Workspace Trust must be granted explicitly; extensions cannot trust a folder on your behalf. | Select **Trust** from the Restricted Mode banner or Workspace Trust editor. The pending create flow resumes automatically after the extension activates. | -| The create view has no model picker. | The CLI model catalog is empty, has no supported models, or discovery failed. | Submit the plan normally; VS Code chooses the model without an extension override. Inspect the discovery properties described above to distinguish the cases, including `modelPickerError` when the API failed and `modelPickerRetried` for the one-time retry. | +| The create view shows a model-unavailable message instead of the picker. | The CLI model catalog is empty, has no supported models, or discovery failed. | Submit the plan normally using VS Code's default, or wait a moment and reopen the page to rediscover models. Inspect the discovery properties described above to distinguish the cases, including `modelPickerError` when the API failed and `modelPickerRetried` for the one-time retry. | | An agent says it needs its instruction files, or behaves oddly / follows outdated steps. | `.github/agents/` is missing or stale. | Accept the download prompt, or run **Download Azure Agent Instructions**. The version stamp auto‑refreshes stale copies. | | Chat opens with the wrong agent or generic Agent mode. | VS Code did not honor the requested custom mode, the custom instructions were not loaded, or the MCP tool was unavailable. | Inspect `diagnosticEvents` for a successful `report_agent_launch` event. Its `agentName` property identifies the agent that reported. If the event is missing, the startup report never reached the CoR MCP server. | | Frontend preview stuck on *"Starting…"*; **Approve UI** never enables (but the app loads in a normal browser). | A second dev server is contending for the preview port. | Stop **all** manually‑started dev servers, free the port, ensure the frontend's `vite.config` is the clean minimal version, then reopen the preview and let it own the server. Don't verify by starting your own server. | diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 1af2a9193..4c0848f29 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -122,6 +122,7 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm modelLabel: vscode.l10n.t('Model'), modelOptions, defaultModelLabel: vscode.l10n.t('VS Code default'), + modelUnavailableMessage: vscode.l10n.t('Models could not be loaded. Using VS Code default. Wait a moment, then reopen this page to try again.'), recentPrompts: getRecentPrompts(), initialPrompt, initialModel: selectedModel, diff --git a/src/webviews/copilotOnRails/views/CreateProjectView.tsx b/src/webviews/copilotOnRails/views/CreateProjectView.tsx index 62b07098a..ab55634c0 100644 --- a/src/webviews/copilotOnRails/views/CreateProjectView.tsx +++ b/src/webviews/copilotOnRails/views/CreateProjectView.tsx @@ -143,7 +143,7 @@ export const CreateProjectView = (): JSX.Element => { />
- {config.modelOptions.length > 0 && ( + {config.modelOptions.length > 0 ? ( + ) : ( + + {config.modelUnavailableMessage} + )} {config.hint}
diff --git a/src/webviews/copilotOnRails/views/styles/createProjectView.scss b/src/webviews/copilotOnRails/views/styles/createProjectView.scss index bd468965c..c009bbf57 100644 --- a/src/webviews/copilotOnRails/views/styles/createProjectView.scss +++ b/src/webviews/copilotOnRails/views/styles/createProjectView.scss @@ -117,13 +117,16 @@ display: flex; align-items: center; justify-content: space-between; + gap: 12px; padding: 8px 12px; border-top: 1px solid var(--vscode-widget-border); background: var(--vscode-editor-background); .actionsLeft { display: flex; + flex-wrap: wrap; align-items: center; + min-width: 0; gap: 10px; .modelDropdown { @@ -146,6 +149,13 @@ } } + .modelFallbackMessage { + max-width: 360px; + font-size: 0.8em; + line-height: 1.4; + color: var(--vscode-descriptionForeground); + } + .hint { font-size: 0.75em; color: var(--vscode-descriptionForeground); @@ -155,6 +165,7 @@ .buttonGroup { display: flex; + flex-shrink: 0; gap: 8px; } } diff --git a/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts b/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts index a891e9735..a0f56e1c0 100644 --- a/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts +++ b/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts @@ -13,6 +13,7 @@ export type CreateProjectViewControllerType = { modelLabel: string; modelOptions: string[]; defaultModelLabel: string; + modelUnavailableMessage: string; /** Recently submitted project-creation prompts, newest-first, for input-history navigation. */ recentPrompts: string[]; /** Pre-fills the prompt input, e.g. when re-opening the view after a reload-for-agent-discovery. */ From 664fccd25af7480aba30461a9419249829b5ee16 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:00:56 -0700 Subject: [PATCH 27/34] Compact model loading guidance into a warning tooltip --- docs/copilot-create-project.md | 9 ++++---- .../extension/createProjectWithCopilot.ts | 3 ++- .../views/CreateProjectView.tsx | 13 +++++++----- .../views/styles/createProjectView.scss | 21 ++++++++++++++++--- .../views/utils/viewConfigTypes.ts | 1 + 5 files changed, 34 insertions(+), 13 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 98b992ae6..907d20701 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -145,9 +145,10 @@ session** running the next agent. Between hand‑offs, agents open **webviews** this list. If the API returns no models, the extension retries once after one second. This retry budget resets only when the extension host reloads, not each time the create flow runs. If discovery fails or returns no supported models, the view still opens without a model picker. - An inline message explains that VS Code's default will be used and suggests waiting a moment - before reopening the page to retry discovery. No error notification appears, and the user can - submit the plan normally. + A compact warning icon and **Models did not finish loading** message replace the picker. + On hover or keyboard focus, a tooltip suggests waiting a moment, then closing and reopening + the page. It also explains that the user can still press **Plan** and change the model manually + in VS Code Chat if the problem persists. No error notification appears, and submission is not blocked. When the Copilot Harness starts, the extension passes the selected model's actual ID and `copilotcli` vendor to Chat without remapping it or constructing an Agent Host fallback. **VS Code default** is stored as `default` in the session and model telemetry. It omits the model @@ -190,7 +191,7 @@ description, optionally pick a **Model**, and press **Plan** (or `Ctrl+Enter`). Create with Copilot prompt view

-> 📷 First capture needed: the prompt view with the inline model-unavailable message instead of the picker. +> 📷 First capture needed: the prompt view with the compact model-unavailable warning and its tooltip instead of the picker.

Create with Copilot prompt view with an inline model-unavailable message diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index 4c0848f29..e0731426d 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -122,7 +122,8 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm modelLabel: vscode.l10n.t('Model'), modelOptions, defaultModelLabel: vscode.l10n.t('VS Code default'), - modelUnavailableMessage: vscode.l10n.t('Models could not be loaded. Using VS Code default. Wait a moment, then reopen this page to try again.'), + modelUnavailableMessage: vscode.l10n.t('Models did not finish loading'), + modelUnavailableTooltip: vscode.l10n.t('Wait a moment, then close and reopen this page to try loading models again. If the problem persists, you can still press Plan and change the model manually in the VS Code Chat model picker.'), recentPrompts: getRecentPrompts(), initialPrompt, initialModel: selectedModel, diff --git a/src/webviews/copilotOnRails/views/CreateProjectView.tsx b/src/webviews/copilotOnRails/views/CreateProjectView.tsx index ab55634c0..e9fa4cc9c 100644 --- a/src/webviews/copilotOnRails/views/CreateProjectView.tsx +++ b/src/webviews/copilotOnRails/views/CreateProjectView.tsx @@ -3,8 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Button, Textarea } from '@fluentui/react-components'; -import { ClipboardTaskListLtrRegular } from '@fluentui/react-icons'; +import { Button, Textarea, Tooltip } from '@fluentui/react-components'; +import { ClipboardTaskListLtrRegular, WarningRegular } from '@fluentui/react-icons'; import { useConfiguration, WebviewContext } from '@microsoft/vscode-azext-webview/webview'; import * as React from 'react'; import { useContext, useLayoutEffect, useRef, useState, type JSX } from 'react'; @@ -154,9 +154,12 @@ export const CreateProjectView = (): JSX.Element => { ))} ) : ( - - {config.modelUnavailableMessage} - + + + + )} {config.hint}

diff --git a/src/webviews/copilotOnRails/views/styles/createProjectView.scss b/src/webviews/copilotOnRails/views/styles/createProjectView.scss index c009bbf57..7217754c3 100644 --- a/src/webviews/copilotOnRails/views/styles/createProjectView.scss +++ b/src/webviews/copilotOnRails/views/styles/createProjectView.scss @@ -150,10 +150,25 @@ } .modelFallbackMessage { - max-width: 360px; - font-size: 0.8em; - line-height: 1.4; + display: inline-flex; + align-items: center; + gap: 5px; + font-size: 0.75em; + line-height: 1.3; color: var(--vscode-descriptionForeground); + cursor: help; + + svg { + flex-shrink: 0; + width: 14px; + height: 14px; + color: var(--vscode-editorWarning-foreground); + } + + &:focus-visible { + outline: 1px solid var(--vscode-focusBorder); + outline-offset: 3px; + } } .hint { diff --git a/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts b/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts index a0f56e1c0..e27b1281b 100644 --- a/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts +++ b/src/webviews/copilotOnRails/views/utils/viewConfigTypes.ts @@ -14,6 +14,7 @@ export type CreateProjectViewControllerType = { modelOptions: string[]; defaultModelLabel: string; modelUnavailableMessage: string; + modelUnavailableTooltip: string; /** Recently submitted project-creation prompts, newest-first, for input-history navigation. */ recentPrompts: string[]; /** Pre-fills the prompt input, e.g. when re-opening the view after a reload-for-agent-discovery. */ From 8f16f5ae88884e74e1257eeba0ff4bb8cb7fc085 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:01:51 -0700 Subject: [PATCH 28/34] Improve model warning and shortcut spacing --- .../copilotOnRails/views/styles/createProjectView.scss | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/webviews/copilotOnRails/views/styles/createProjectView.scss b/src/webviews/copilotOnRails/views/styles/createProjectView.scss index 7217754c3..858bc0b78 100644 --- a/src/webviews/copilotOnRails/views/styles/createProjectView.scss +++ b/src/webviews/copilotOnRails/views/styles/createProjectView.scss @@ -127,7 +127,8 @@ flex-wrap: wrap; align-items: center; min-width: 0; - gap: 10px; + column-gap: 24px; + row-gap: 8px; .modelDropdown { font-size: 0.9em; @@ -152,7 +153,7 @@ .modelFallbackMessage { display: inline-flex; align-items: center; - gap: 5px; + gap: 8px; font-size: 0.75em; line-height: 1.3; color: var(--vscode-descriptionForeground); From fce3dffa1833b1d8e61fb09b988f4e40c7cd46b9 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Oct 2026 15:13:04 -0700 Subject: [PATCH 29/34] Honor explicit Copilot on Rails model selections --- docs/copilot-create-project.md | 34 ++++-- .../copilotOnRails/openChatWithAgent.ts | 30 +++-- src/utils/copilotOnRails/modelSelection.ts | 26 ++++- .../extension/harnessSettings.ts | 20 +++- test/copilotOnRails/modelSelection.test.ts | 49 +++++++- test/copilotOnRails/openChatWithAgent.test.ts | 106 ++++++++++++------ 6 files changed, 204 insertions(+), 61 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 907d20701..1977582ff 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -135,13 +135,23 @@ session** running the next agent. Between hand‑offs, agents open **webviews** - **VS Code** with **GitHub Copilot** enabled and signed in. - The **Copilot** chat session target. Copilot on Rails enables the Copilot Harness for the workspace - and waits for custom-agent registration to settle. After creating a fresh phase chat, it primes the + and sets `github.copilot.chat.cli.autoModel.enabled` to `false` at workspace scope before model + discovery, even if Copilot Chat is already active. User/global settings are left unchanged. + This setting requires a VS Code reload to take effect if the CLI provider is already running. + Versions that do not register this setting skip that override. + Selecting **Auto** in the create view flips that workspace flag to `true` when invoking Chat. + Named-model and `default` selections set it to `false`. The selected mode persists across phases. + This flag controls the public CLI Auto option. Agent Host has separate Auto support, so launches + still use explicit named-model or Auto selectors rather than relying on this flag to choose a model. + The extension waits for custom-agent registration to settle. After creating a fresh phase chat, it primes the custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. - A Copilot plan with access to at least one supported model. The model picker lists Opus, Sonnet, - GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, using their display + GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, plus an explicit + **Auto** option when supported models are available, using their display names without a vendor suffix. The lowest-version available Opus model is selected by default, - or the first supported option when no Opus model is available. The extension activates GitHub Copilot Chat before loading + or the first supported named model when no Opus model is available. Auto is never selected + implicitly as a substitute for a named model. The extension activates GitHub Copilot Chat before loading this list. If the API returns no models, the extension retries once after one second. This retry budget resets only when the extension host reloads, not each time the create flow runs. If discovery fails or returns no supported models, the view still opens without a model picker. @@ -149,8 +159,12 @@ session** running the next agent. Between hand‑offs, agents open **webviews** On hover or keyboard focus, a tooltip suggests waiting a moment, then closing and reopening the page. It also explains that the user can still press **Plan** and change the model manually in VS Code Chat if the problem persists. No error notification appears, and submission is not blocked. - When the Copilot Harness starts, the extension passes the selected model's actual ID and `copilotcli` - vendor to Chat without remapping it or constructing an Agent Host fallback. + Discovery uses the `copilotcli` vendor, but Chat launches use the selected model's actual ID + with vendor `agent-host-copilotcli`. This targets the Agent Host copy of that model; it does not + query a fallback catalog or substitute a different model. Passing the `copilotcli` vendor directly + produces an extension-host identifier that the Agent Host drops, reverting to its default. + Selecting **Auto** passes model ID `auto` to the Agent Host explicitly. Auto is available in our + picker independently of whether the public CLI catalog currently advertises it. **VS Code default** is stored as `default` in the session and model telemetry. It omits the model selector on every subsequent launch or chat invocation, leaving model choice to VS Code. This selection is preserved if the create view reopens after a reload, even when models have @@ -610,13 +624,18 @@ Model discovery adds these properties to the `createProjectWithCopilot` event an | `modelPickerOutcome` | `querying` before discovery completes, `emptyCatalog` for no returned models, `noSupportedModels` when none pass the filter, `queryFailed` when the API throws, or `ready` when supported models are available. | Diagnostics also include `modelPickerAvailableModels` with the returned model names before filtering, -and `modelPickerFilteredModels` with the displayed model names after filtering and deduplication. These catalogs +and `modelPickerFilteredModels` with the supported named models after filtering and deduplication. These catalogs are workspace-cached diagnostic data only and are not sent to telemetry. The counts and discovery outcome are mirrored to telemetry separately through the standard CoR property handling. -The `default` sentinel is not counted as an available or filtered model. +The `default` sentinel is not counted as an available or filtered model. The explicit **Auto** picker +option is not a named model and is excluded from the filtered list/count; if the API advertises Auto, +it still appears in the available list/count. Discovery failures record a masked `modelPickerError`. Submitting with the fallback records `copilotModel: default`, `modelSelectedInView: false`, and `chatModelSelectionSource: default`; chat options omit `modelSelector`. +Named-model launches record `chatModelSelectorId` and `chatModelSelectorVendor` in telemetry and +diagnostics. `chatModelResolved` means the extension constructed a selector, not that VS Code confirmed +the final selection. VS Code does not expose the active Chat model through a consumer extension API. Privacy guarantees, by design: @@ -637,6 +656,7 @@ before submitting. | *"Choose where to create your project."* | The open folder isn't empty. | Choose **Create in New Subfolder…** to build under the current folder, or **Choose Empty Folder…** to build elsewhere. Both open the project in a separate window. | | The new project window opens in Restricted Mode and the Azure Project view is unavailable. | Workspace Trust must be granted explicitly; extensions cannot trust a folder on your behalf. | Select **Trust** from the Restricted Mode banner or Workspace Trust editor. The pending create flow resumes automatically after the extension activates. | | The create view shows a model-unavailable message instead of the picker. | The CLI model catalog is empty, has no supported models, or discovery failed. | Submit the plan normally using VS Code's default, or wait a moment and reopen the page to rediscover models. Inspect the discovery properties described above to distinguish the cases, including `modelPickerError` when the API failed and `modelPickerRetried` for the one-time retry. | +| Chat shows Auto after a named model was selected in the create view. | The launch selector may refer to the wrong model pool, or VS Code may have changed the selection. | Inspect the submission event's `copilotModel`, `chatModelSelectorId`, and `chatModelSelectorVendor`. A named Copilot Harness selection should use vendor `agent-host-copilotcli`, even though discovery uses `copilotcli`. A resolved selector alone does not prove the Chat selection was applied. | | An agent says it needs its instruction files, or behaves oddly / follows outdated steps. | `.github/agents/` is missing or stale. | Accept the download prompt, or run **Download Azure Agent Instructions**. The version stamp auto‑refreshes stale copies. | | Chat opens with the wrong agent or generic Agent mode. | VS Code did not honor the requested custom mode, the custom instructions were not loaded, or the MCP tool was unavailable. | Inspect `diagnosticEvents` for a successful `report_agent_launch` event. Its `agentName` property identifies the agent that reported. If the event is missing, the startup report never reached the CoR MCP server. | | Frontend preview stuck on *"Starting…"*; **Approve UI** never enables (but the app loads in a normal browser). | A second dev server is contending for the preview port. | Stop **all** manually‑started dev servers, free the port, ensure the frontend's `vite.config` is the clean minimal version, then reopen the preview and let it own the server. Don't verify by starting your own server. | diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index f210dfcfe..5a7b6a0fe 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -8,10 +8,10 @@ import * as vscode from 'vscode'; import { ext } from '../../extensionVariables'; import { projectSubmissionState } from '../../tree/project/projectSubmissionState'; import { CopilotOnRailsContext } from '../../utils/copilotOnRails/CopilotOnRailsContext'; -import { DEFAULT_CHAT_MODEL, resolveAvailableChatModel } from '../../utils/copilotOnRails/modelSelection'; +import { AUTO_CHAT_MODEL, DEFAULT_CHAT_MODEL, resolveCopilotHarnessModelSelector } from '../../utils/copilotOnRails/modelSelection'; import { setCorErrorProp, setCorProp } from '../../utils/copilotOnRails/telemetryUtils'; import { AUTOPILOT_QUERY_MARKER, isAutopilotActive } from '../../webviews/copilotOnRails/extension/autopilot'; -import { ensureCopilotHarnessOn } from '../../webviews/copilotOnRails/extension/harnessSettings'; +import { ensureCopilotHarnessOn, setCopilotCliAutoModelEnabled } from '../../webviews/copilotOnRails/extension/harnessSettings'; import { openLoadingView } from '../../webviews/copilotOnRails/extension/openLoadingView'; import { getSessionModel, recordAgentLaunch } from '../../webviews/copilotOnRails/extension/projectSession'; import { saveReloadResumePrompt } from '../../webviews/copilotOnRails/extension/reloadResumePrompt'; @@ -43,15 +43,12 @@ export function registerWorkspaceTrustTracking(): void { * modelSelector object that VS Code's chat commands expect. */ export async function resolveModelSelector(displayName: string): Promise<{ id?: string; vendor?: string } | undefined> { - if (displayName === DEFAULT_CHAT_MODEL) { - return undefined; + if (displayName === DEFAULT_CHAT_MODEL || displayName === AUTO_CHAT_MODEL) { + return resolveCopilotHarnessModelSelector(displayName, []); } try { const models = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); - const match = resolveAvailableChatModel(displayName, models); - if (match) { - return { id: match.id, vendor: match.vendor }; - } + return resolveCopilotHarnessModelSelector(displayName, models); } catch { // If the lm API isn't available, fall through } @@ -94,9 +91,9 @@ export async function ensureCopilotChatReady(context: CopilotOnRailsContext): Pr return false; } - if (!copilotChatExtension.isActive) { - await ensureCopilotHarnessOn(); + await ensureCopilotHarnessOn(); + if (!copilotChatExtension.isActive) { try { await vscode.window.withProgress( { location: vscode.ProgressLocation.Notification, title: vscode.l10n.t('Starting GitHub Copilot Chat...') }, @@ -128,7 +125,8 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: agentLaunchInProgress = true; try { - await ensureCopilotHarnessOn(); + const resolvedModel = model ?? getSessionModel(); + await ensureCopilotHarnessOn(resolvedModel === AUTO_CHAT_MODEL); // Custom-agent commands are registered from the focused chat widget. Open Chat before // creating the fresh session so first-time launches do not wait on a widget that does not exist. @@ -145,11 +143,14 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: buildAgentChatModeOptions(agentName), ); - const resolvedModel = model ?? getSessionModel(); setCorProp(context, 'chatModelSelectionSource', resolvedModel === DEFAULT_CHAT_MODEL ? 'default' : model ? 'newlySelected' : (getSessionModel() ? 'previouslySelected' : 'default')); const selector = resolvedModel ? await resolveModelSelector(resolvedModel) : undefined; setCorProp(context, 'chatModelResolved', !!selector); + if (selector) { + setCorProp(context, 'chatModelSelectorId', selector.id); + setCorProp(context, 'chatModelSelectorVendor', selector.vendor); + } const result = await vscode.commands.executeCommand( OPEN_CHAT_COMMAND_ID, @@ -346,10 +347,15 @@ export async function buildChatOpenOptions(context: CopilotOnRailsContext, optio } const model = getSessionModel(); + await setCopilotCliAutoModelEnabled(model === AUTO_CHAT_MODEL); setCorProp(context, 'chatModelSelectionSource', model && model !== DEFAULT_CHAT_MODEL ? 'previouslySelected' : 'default'); if (model) { const selector = await resolveModelSelector(model); setCorProp(context, 'chatModelResolved', !!selector); + if (selector) { + setCorProp(context, 'chatModelSelectorId', selector.id); + setCorProp(context, 'chatModelSelectorVendor', selector.vendor); + } return selector ? { ...options, modelSelector: selector } : options; } return options; diff --git a/src/utils/copilotOnRails/modelSelection.ts b/src/utils/copilotOnRails/modelSelection.ts index 7e5ad209c..6f9deb287 100644 --- a/src/utils/copilotOnRails/modelSelection.ts +++ b/src/utils/copilotOnRails/modelSelection.ts @@ -13,10 +13,15 @@ export interface AvailableChatModel { export const supportedModelNames = ['Opus', 'Sonnet', 'GPT Sol', 'GPT Terra'] as const; export const DEFAULT_CHAT_MODEL = 'default'; +export const AUTO_CHAT_MODEL = 'Auto'; export function getModelPickerOptions(models: readonly AvailableChatModel[], initialModel?: string): string[] { const options = getSupportedModelOptions(models); - return options.length > 0 && initialModel === DEFAULT_CHAT_MODEL ? [DEFAULT_CHAT_MODEL, ...options] : options; + if (options.length === 0) { + return []; + } + const pickerOptions = [...options, AUTO_CHAT_MODEL]; + return initialModel === DEFAULT_CHAT_MODEL ? [DEFAULT_CHAT_MODEL, ...pickerOptions] : pickerOptions; } export function getSupportedModelName(...identifiers: string[]): typeof supportedModelNames[number] | undefined { @@ -29,7 +34,8 @@ export function getSupportedModelName(...identifiers: string[]): typeof supporte export function getDefaultOpusModelOption(models: readonly AvailableChatModel[]): string | undefined { const model = models - .filter(model => getSupportedModelName(model.id, model.family, model.name) === 'Opus') + .filter(model => model.id !== 'auto' && model.name !== AUTO_CHAT_MODEL + && getSupportedModelName(model.id, model.family, model.name) === 'Opus') .sort((a, b) => a.version.localeCompare(b.version, undefined, { numeric: true, sensitivity: 'base' }) || a.name.localeCompare(b.name))[0]; @@ -51,12 +57,28 @@ export function resolveAvailableChatModel(displayName: string, models: readonly ?? models.find(model => model.id === displayName); } +export function resolveCopilotHarnessModelSelector( + displayName: string, + models: readonly AvailableChatModel[], +): { id: string; vendor: string } | undefined { + if (displayName === DEFAULT_CHAT_MODEL) { + return undefined; + } + if (displayName === AUTO_CHAT_MODEL) { + return { id: 'auto', vendor: 'agent-host-copilotcli' }; + } + const model = resolveAvailableChatModel(displayName, models); + // Agent Host drops extension-host model identifiers instead of applying them. + return model ? { id: model.id, vendor: 'agent-host-copilotcli' } : undefined; +} + /** * Filters a vendor-scoped API result to supported model families and returns * display names for the model picker. */ export function getSupportedModelOptions(models: readonly AvailableChatModel[]): string[] { const matchingModels = models + .filter(model => model.id !== 'auto' && model.name !== AUTO_CHAT_MODEL) .map(model => ({ supportedName: getSupportedModelName(model.id, model.family, model.name), model })) .filter((entry): entry is { supportedName: typeof supportedModelNames[number]; model: AvailableChatModel } => entry.supportedName !== undefined) diff --git a/src/webviews/copilotOnRails/extension/harnessSettings.ts b/src/webviews/copilotOnRails/extension/harnessSettings.ts index b85d2fc0f..7c2f3262d 100644 --- a/src/webviews/copilotOnRails/extension/harnessSettings.ts +++ b/src/webviews/copilotOnRails/extension/harnessSettings.ts @@ -15,15 +15,15 @@ const HARNESS_SETTINGS = [ { prefix: 'chat.editor', key: 'preferCopilotHarness' }, { prefix: 'chat', key: 'defaultToCopilotHarness' }, ] as const; +const CLI_AUTO_MODEL_SETTING = { prefix: 'github.copilot.chat.cli', key: 'autoModel.enabled' } as const; export const COPILOT_HARNESS_SETTING_IDS = HARNESS_SETTINGS.map(({ prefix, key }) => `${prefix}.${key}`); /** - * Ensures Copilot on Rails uses the Copilot Harness by turning its two experimental settings on - * at Workspace scope. Like the raised chat request + * Enables the Copilot Harness and configures explicit Auto selection at Workspace scope. Like the raised chat request * budget, the workspace override is cheap and disposable, so it is intentionally left in place. */ -export async function ensureCopilotHarnessOn(): Promise { +export async function ensureCopilotHarnessOn(useAutoModel = false): Promise { const folder = workspace.workspaceFolders?.[0]; if (!folder) { return; @@ -35,4 +35,18 @@ export async function ensureCopilotHarnessOn(): Promise { // Best effort: the setting may be unknown on older VS Code versions. } } + await setCopilotCliAutoModelEnabled(useAutoModel); +} + +export async function setCopilotCliAutoModelEnabled(enabled: boolean): Promise { + const folder = workspace.workspaceFolders?.[0]; + if (!folder) { + return; + } + const { prefix, key } = CLI_AUTO_MODEL_SETTING; + if (workspace.getConfiguration(prefix, folder.uri).inspect(key)) { + await settingUtils.updateWorkspaceSetting(key, enabled, folder.uri.fsPath, prefix, ConfigurationTarget.Workspace); + } else { + console.log('[Copilot on Rails] CLI Auto model setting is unavailable in this version; skipping its workspace override.'); + } } diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts index 7cd66515c..c6fd05227 100644 --- a/test/copilotOnRails/modelSelection.test.ts +++ b/test/copilotOnRails/modelSelection.test.ts @@ -6,9 +6,13 @@ import assert from 'assert'; import { AvailableChatModel, + AUTO_CHAT_MODEL, DEFAULT_CHAT_MODEL, + getDefaultOpusModelOption, getModelPickerOptions, + getSupportedModelOptions, resolveAvailableChatModel, + resolveCopilotHarnessModelSelector, } from '../../src/utils/copilotOnRails/modelSelection'; const localModel: AvailableChatModel = { @@ -25,6 +29,45 @@ const cliModel: AvailableChatModel = { }; suite('Copilot on Rails model selection', () => { + test('resolves explicit Auto without needing the CLI catalog to advertise it', () => { + assert.deepStrictEqual( + resolveCopilotHarnessModelSelector(AUTO_CHAT_MODEL, []), + { id: 'auto', vendor: 'agent-host-copilotcli' }, + ); + }); + + test('does not treat Auto family metadata as a supported named model', () => { + const auto = { ...cliModel, id: 'auto', name: AUTO_CHAT_MODEL, family: 'claude-opus-4.6' }; + assert.deepStrictEqual(getSupportedModelOptions([auto, cliModel]), [cliModel.name]); + assert.deepStrictEqual(getModelPickerOptions([auto]), []); + assert.strictEqual(getDefaultOpusModelOption([auto, cliModel]), undefined); + }); + + test('offers Auto once and keeps a named model first regardless of the CLI Auto flag', () => { + const auto = { ...cliModel, id: 'auto', name: AUTO_CHAT_MODEL, family: 'claude-sonnet-5' }; + assert.deepStrictEqual(getModelPickerOptions([cliModel]), [cliModel.name, AUTO_CHAT_MODEL]); + assert.deepStrictEqual(getModelPickerOptions([auto, cliModel]), [cliModel.name, AUTO_CHAT_MODEL]); + assert.deepStrictEqual(getModelPickerOptions([cliModel], AUTO_CHAT_MODEL), [cliModel.name, AUTO_CHAT_MODEL]); + }); + + test('maps a discovered CLI model id to the Agent Host launch vendor', () => { + assert.deepStrictEqual( + resolveCopilotHarnessModelSelector('GPT-6.1 Sol', [cliModel]), + { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + ); + }); + + test('does not resolve an unavailable model to a fallback selector', () => { + assert.strictEqual(resolveCopilotHarnessModelSelector('Unavailable model', [cliModel]), undefined); + }); + + test('omits a selector for the default sentinel even if a provider lists that id', () => { + assert.strictEqual( + resolveCopilotHarnessModelSelector(DEFAULT_CHAT_MODEL, [{ ...cliModel, id: DEFAULT_CHAT_MODEL }]), + undefined, + ); + }); + test('provides no picker options when the catalog is empty', () => { assert.deepStrictEqual(getModelPickerOptions([]), []); }); @@ -34,12 +77,12 @@ suite('Copilot on Rails model selection', () => { assert.deepStrictEqual(getModelPickerOptions([model]), []); }); - test('keeps the normal picker unchanged when supported models are available', () => { - assert.deepStrictEqual(getModelPickerOptions([cliModel]), [cliModel.name]); + test('keeps supported models available alongside explicit Auto', () => { + assert.deepStrictEqual(getModelPickerOptions([cliModel]), [cliModel.name, AUTO_CHAT_MODEL]); }); test('preserves the default selection when reopening after models become available', () => { - assert.deepStrictEqual(getModelPickerOptions([cliModel], DEFAULT_CHAT_MODEL), [DEFAULT_CHAT_MODEL, cliModel.name]); + assert.deepStrictEqual(getModelPickerOptions([cliModel], DEFAULT_CHAT_MODEL), [DEFAULT_CHAT_MODEL, cliModel.name, AUTO_CHAT_MODEL]); }); test('resolves a qualified local model name', () => { diff --git a/test/copilotOnRails/openChatWithAgent.test.ts b/test/copilotOnRails/openChatWithAgent.test.ts index b0929ed12..5f283466a 100644 --- a/test/copilotOnRails/openChatWithAgent.test.ts +++ b/test/copilotOnRails/openChatWithAgent.test.ts @@ -5,12 +5,25 @@ import assert from 'assert'; import { createTestActionContext } from '@microsoft/vscode-azext-utils'; +import { ConfigurationTarget, workspace } from 'vscode'; import { buildAgentChatModeOptions, buildAgentChatOpenOptions, buildChatOpenOptions, resolveModelSelector } from '../../src/commands/copilotOnRails/openChatWithAgent'; import { ext } from '../../src/extensionVariables'; -import { DEFAULT_CHAT_MODEL } from '../../src/utils/copilotOnRails/modelSelection'; +import { AUTO_CHAT_MODEL, DEFAULT_CHAT_MODEL } from '../../src/utils/copilotOnRails/modelSelection'; +import { settingUtils } from '../../src/utils/settingUtils'; +import { ensureCopilotHarnessOn } from '../../src/webviews/copilotOnRails/extension/harnessSettings'; import { getSessionModel, recordModel, recordPhase } from '../../src/webviews/copilotOnRails/extension/projectSession'; suite('Copilot on Rails agent chat launch', () => { + const originalUpdateWorkspaceSetting = settingUtils.updateWorkspaceSetting; + const workspaceSettingUpdates: Parameters[] = []; + setup(() => { + workspaceSettingUpdates.length = 0; + settingUtils.updateWorkspaceSetting = async (...args) => { workspaceSettingUpdates.push(args); }; + }); + teardown(() => { + settingUtils.updateWorkspaceSetting = originalUpdateWorkspaceSetting; + }); + test('omits a model selector when the VS Code default is selected', async () => { const selector = await resolveModelSelector(DEFAULT_CHAT_MODEL); assert.deepStrictEqual( @@ -19,37 +32,62 @@ suite('Copilot on Rails agent chat launch', () => { ); }); - test('persists the default across phases and omits selectors on subsequent chat calls', async () => { - const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); - const state = new Map(); - Object.defineProperty(ext, 'context', { - configurable: true, - value: { - workspaceState: { - get: (key: string) => state.get(key), - update: async (key: string, value: unknown) => { state.set(key, value); }, + test('selects Auto explicitly when requested', async () => { + assert.deepStrictEqual( + await resolveModelSelector(AUTO_CHAT_MODEL), + { id: 'auto', vendor: 'agent-host-copilotcli' }, + ); + }); + + test('disables CLI Auto during initial workspace setup', async () => { + await ensureCopilotHarnessOn(); + const folder = workspace.workspaceFolders?.[0]; + const settingAvailable = folder && workspace.getConfiguration('github.copilot.chat.cli', folder.uri).inspect('autoModel.enabled'); + const autoUpdates = workspaceSettingUpdates.filter(([key]) => key === 'autoModel.enabled'); + assert.deepStrictEqual(autoUpdates, settingAvailable + ? [['autoModel.enabled', false, folder?.uri.fsPath, 'github.copilot.chat.cli', ConfigurationTarget.Workspace]] + : []); + }); + + for (const model of [DEFAULT_CHAT_MODEL, AUTO_CHAT_MODEL]) { + test(`persists ${model} across phases and configures its workspace Auto flag`, async () => { + const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); + const state = new Map(); + Object.defineProperty(ext, 'context', { + configurable: true, + value: { + workspaceState: { + get: (key: string) => state.get(key), + update: async (key: string, value: unknown) => { state.set(key, value); }, + }, }, - }, - }); - try { - await recordModel(DEFAULT_CHAT_MODEL); - for (const phase of ['scaffold', 'integrate', 'debug', 'deploy'] as const) { - await recordPhase(phase); - assert.strictEqual(getSessionModel(), DEFAULT_CHAT_MODEL); - const context = await createTestActionContext(); - const options = { query: 'Continue the project' }; - assert.deepStrictEqual(await buildChatOpenOptions(context, options), options); - assert.strictEqual(context.telemetry.properties.chatModelSelectionSource, 'default'); - assert.strictEqual(context.telemetry.properties.chatModelResolved, 'false'); - } - } finally { - if (originalContext) { - Object.defineProperty(ext, 'context', originalContext); - } else { - Reflect.deleteProperty(ext, 'context'); + }); + try { + await recordModel(model); + for (const phase of ['scaffold', 'integrate', 'debug', 'deploy'] as const) { + await recordPhase(phase); + assert.strictEqual(getSessionModel(), model); + const context = await createTestActionContext(); + const options = { query: 'Continue the project' }; + assert.deepStrictEqual(await buildChatOpenOptions(context, options), model === AUTO_CHAT_MODEL + ? { ...options, modelSelector: { id: 'auto', vendor: 'agent-host-copilotcli' } } + : options); + assert.strictEqual(context.telemetry.properties.chatModelSelectionSource, model === DEFAULT_CHAT_MODEL ? 'default' : 'previouslySelected'); + assert.strictEqual(context.telemetry.properties.chatModelResolved, String(model === AUTO_CHAT_MODEL)); + } + const folder = workspace.workspaceFolders?.[0]; + const settingAvailable = folder && workspace.getConfiguration('github.copilot.chat.cli', folder.uri).inspect('autoModel.enabled'); + const expectedUpdate = ['autoModel.enabled', model === AUTO_CHAT_MODEL, folder?.uri.fsPath, 'github.copilot.chat.cli', ConfigurationTarget.Workspace]; + assert.deepStrictEqual(workspaceSettingUpdates, settingAvailable ? Array(4).fill(expectedUpdate) : []); + } finally { + if (originalContext) { + Object.defineProperty(ext, 'context', originalContext); + } else { + Reflect.deleteProperty(ext, 'context'); + } } - } - }); + }); + } test('builds options that select the custom agent without submitting a prompt', () => { assert.deepStrictEqual( @@ -63,12 +101,12 @@ suite('Copilot on Rails agent chat launch', () => { buildAgentChatOpenOptions( 'azure-project-plan', 'Build a project', - { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, + { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, ), { mode: 'azure-project-plan', query: 'Build a project', - modelSelector: { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, + modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, waitForRequestAcceptance: true, }, ); @@ -77,11 +115,11 @@ suite('Copilot on Rails agent chat launch', () => { test('carries Autopilot into fresh phase chats without changing the agent or model', () => { for (const agentName of ['azure-project-scaffold', 'azure-project-integrate', 'azure-debug-plan', 'azure-debug-generate', 'azure-deploy']) { assert.deepStrictEqual( - buildAgentChatOpenOptions(agentName, 'Continue the project', { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, true), + buildAgentChatOpenOptions(agentName, 'Continue the project', { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, true), { mode: agentName, query: '[AUTOPILOT MODE] Continue the project', - modelSelector: { id: 'gpt-6.1-sol', vendor: 'copilotcli' }, + modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, waitForRequestAcceptance: true, }, ); From accadfcd1a8208edf3c673dac47ee5c8c59d4a1e Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:14:41 -0700 Subject: [PATCH 30/34] Simplify docs --- docs/copilot-create-project.md | 68 +++++++++------------------------- 1 file changed, 17 insertions(+), 51 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 1977582ff..0e84bdc75 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -134,41 +134,10 @@ session** running the next agent. Between hand‑offs, agents open **webviews** ## Prerequisites - **VS Code** with **GitHub Copilot** enabled and signed in. -- The **Copilot** chat session target. Copilot on Rails enables the Copilot Harness for the workspace - and sets `github.copilot.chat.cli.autoModel.enabled` to `false` at workspace scope before model - discovery, even if Copilot Chat is already active. User/global settings are left unchanged. - This setting requires a VS Code reload to take effect if the CLI provider is already running. - Versions that do not register this setting skip that override. - Selecting **Auto** in the create view flips that workspace flag to `true` when invoking Chat. - Named-model and `default` selections set it to `false`. The selected mode persists across phases. - This flag controls the public CLI Auto option. Agent Host has separate Auto support, so launches - still use explicit named-model or Auto selectors rather than relying on this flag to choose a model. - The extension waits for custom-agent registration to settle. After creating a fresh phase chat, it primes the - custom mode without sending a prompt, then resolves the agent by name when submitting. This avoids - stale Agent Host plugin revisions and prevents a silent fallback to **Agent**. -- A Copilot plan with access to at least one supported model. The model picker lists Opus, Sonnet, - GPT Sol, and GPT Terra models returned by the `copilotcli` language-model vendor, plus an explicit - **Auto** option when supported models are available, using their display - names without a vendor suffix. The lowest-version available Opus model is selected by default, - or the first supported named model when no Opus model is available. Auto is never selected - implicitly as a substitute for a named model. The extension activates GitHub Copilot Chat before loading - this list. If the API returns no models, the extension retries once after one second. This retry - budget resets only when the extension host reloads, not each time the create flow runs. - If discovery fails or returns no supported models, the view still opens without a model picker. - A compact warning icon and **Models did not finish loading** message replace the picker. - On hover or keyboard focus, a tooltip suggests waiting a moment, then closing and reopening - the page. It also explains that the user can still press **Plan** and change the model manually - in VS Code Chat if the problem persists. No error notification appears, and submission is not blocked. - Discovery uses the `copilotcli` vendor, but Chat launches use the selected model's actual ID - with vendor `agent-host-copilotcli`. This targets the Agent Host copy of that model; it does not - query a fallback catalog or substitute a different model. Passing the `copilotcli` vendor directly - produces an extension-host identifier that the Agent Host drops, reverting to its default. - Selecting **Auto** passes model ID `auto` to the Agent Host explicitly. Auto is available in our - picker independently of whether the public CLI catalog currently advertises it. - **VS Code default** is stored as `default` in the session and model telemetry. It omits the model - selector on every subsequent launch or chat invocation, leaving model choice to VS Code. - This selection is preserved if the create view reopens after a reload, even when models have - become available in the meantime. +- The **Copilot CLI** chat target. Copilot on Rails manages Auto through the workspace setting + `github.copilot.chat.cli.autoModel.enabled`, enabling it only when **Auto** is selected in the picker. +- A Copilot plan. If no models load in time, the view shows **Models did not finish loading**. + You can still press **Plan** using **VS Code default** and change the model in Chat. - **A clean project folder.** The flow needs an empty workspace root to build in. If the open folder already contains files, choose **Create in New Subfolder…** to create the project under the current folder, or **Choose Empty Folder…** to build elsewhere. Either choice opens the project in a separate window without @@ -613,29 +582,26 @@ so the model-discovery diagnostics remain available alongside the planning and l The telemetry-only `corProjectId` is created when the prompt is submitted, not when the create view loads or models are discovered. Subsequent project events reuse it. -Model discovery adds these properties to the `createProjectWithCopilot` event and CoR telemetry: +An empty catalog gets one retry after one second; the retry budget resets on extension-host reload. +Discovery adds these properties to the `createProjectWithCopilot` event and CoR telemetry: | Property | Meaning | | --- | --- | | `modelPickerVendor` | The queried vendor, `copilotcli`. | | `modelPickerAvailableModelCount` | Number of entries in `modelPickerAvailableModels`, before filtering. | | `modelPickerFilteredModelCount` | Number of entries in `modelPickerFilteredModels`, after filtering and deduplication. | -| `modelPickerRetried` | Whether this invocation used the one-per-extension-host-session empty-catalog retry. | +| `modelPickerRetried` | Whether this invocation retried an empty catalog. | | `modelPickerOutcome` | `querying` before discovery completes, `emptyCatalog` for no returned models, `noSupportedModels` when none pass the filter, `queryFailed` when the API throws, or `ready` when supported models are available. | -Diagnostics also include `modelPickerAvailableModels` with the returned model names before filtering, -and `modelPickerFilteredModels` with the supported named models after filtering and deduplication. These catalogs -are workspace-cached diagnostic data only and are not sent to telemetry. The counts and discovery -outcome are mirrored to telemetry separately through the standard CoR property handling. -The `default` sentinel is not counted as an available or filtered model. The explicit **Auto** picker -option is not a named model and is excluded from the filtered list/count; if the API advertises Auto, -it still appears in the available list/count. -Discovery failures record a masked `modelPickerError`. Submitting with the fallback records -`copilotModel: default`, `modelSelectedInView: false`, and `chatModelSelectionSource: default`; -chat options omit `modelSelector`. -Named-model launches record `chatModelSelectorId` and `chatModelSelectorVendor` in telemetry and -diagnostics. `chatModelResolved` means the extension constructed a selector, not that VS Code confirmed -the final selection. VS Code does not expose the active Chat model through a consumer extension API. +The diagnostic-only lists `modelPickerAvailableModels` and `modelPickerFilteredModels` contain returned +names and deduplicated supported names, respectively. Auto is excluded from the filtered list/count +but remains in the available list/count if returned by the API. The `default` sentinel is never counted. +Discovery errors record a masked `modelPickerError`. + +The fallback records `copilotModel: default`, `modelSelectedInView: false`, and +`chatModelSelectionSource: default`, and omits `modelSelector`. Explicit selections record +`chatModelSelectorId` and `chatModelSelectorVendor` in telemetry and diagnostics. +`chatModelResolved` confirms selector construction, not VS Code's final Chat selection. Privacy guarantees, by design: @@ -656,7 +622,7 @@ before submitting. | *"Choose where to create your project."* | The open folder isn't empty. | Choose **Create in New Subfolder…** to build under the current folder, or **Choose Empty Folder…** to build elsewhere. Both open the project in a separate window. | | The new project window opens in Restricted Mode and the Azure Project view is unavailable. | Workspace Trust must be granted explicitly; extensions cannot trust a folder on your behalf. | Select **Trust** from the Restricted Mode banner or Workspace Trust editor. The pending create flow resumes automatically after the extension activates. | | The create view shows a model-unavailable message instead of the picker. | The CLI model catalog is empty, has no supported models, or discovery failed. | Submit the plan normally using VS Code's default, or wait a moment and reopen the page to rediscover models. Inspect the discovery properties described above to distinguish the cases, including `modelPickerError` when the API failed and `modelPickerRetried` for the one-time retry. | -| Chat shows Auto after a named model was selected in the create view. | The launch selector may refer to the wrong model pool, or VS Code may have changed the selection. | Inspect the submission event's `copilotModel`, `chatModelSelectorId`, and `chatModelSelectorVendor`. A named Copilot Harness selection should use vendor `agent-host-copilotcli`, even though discovery uses `copilotcli`. A resolved selector alone does not prove the Chat selection was applied. | +| Chat shows Auto after selecting a named model. | Wrong model pool or a selection change in VS Code. | Compare `copilotModel` with `chatModelSelectorId`; `chatModelSelectorVendor` should be `agent-host-copilotcli`. | | An agent says it needs its instruction files, or behaves oddly / follows outdated steps. | `.github/agents/` is missing or stale. | Accept the download prompt, or run **Download Azure Agent Instructions**. The version stamp auto‑refreshes stale copies. | | Chat opens with the wrong agent or generic Agent mode. | VS Code did not honor the requested custom mode, the custom instructions were not loaded, or the MCP tool was unavailable. | Inspect `diagnosticEvents` for a successful `report_agent_launch` event. Its `agentName` property identifies the agent that reported. If the event is missing, the startup report never reached the CoR MCP server. | | Frontend preview stuck on *"Starting…"*; **Approve UI** never enables (but the app loads in a normal browser). | A second dev server is contending for the preview port. | Stop **all** manually‑started dev servers, free the port, ensure the frontend's `vite.config` is the clean minimal version, then reopen the preview and let it own the server. Don't verify by starting your own server. | From d5eb56876b03aff288070dcf92d88c4b1371b83f Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:37:34 -0700 Subject: [PATCH 31/34] Various improvements --- docs/copilot-create-project.md | 28 ++++--------------- .../registerDebugSessionWatcher.ts | 5 +--- src/utils/copilotOnRails/diagnosticUtils.ts | 2 +- src/utils/copilotOnRails/telemetryUtils.ts | 19 ++++++------- .../CreateProjectViewController.ts | 7 ++--- .../controllers/LoadingViewController.ts | 5 +--- .../extension/createProjectWithCopilot.ts | 2 -- .../extension/openRequirementsView.ts | 5 +--- .../createProjectWithCopilot.test.ts | 15 ++++------ 9 files changed, 27 insertions(+), 61 deletions(-) diff --git a/docs/copilot-create-project.md b/docs/copilot-create-project.md index 0e84bdc75..d6892418d 100644 --- a/docs/copilot-create-project.md +++ b/docs/copilot-create-project.md @@ -174,12 +174,6 @@ description, optionally pick a **Model**, and press **Plan** (or `Ctrl+Enter`). Create with Copilot prompt view

-> 📷 First capture needed: the prompt view with the compact model-unavailable warning and its tooltip instead of the picker. - -

- Create with Copilot prompt view with an inline model-unavailable message -

- Pressing **Plan** opens Copilot Chat if needed, then starts the **`azure-project-plan`** agent in a new chat session. Every new-project prompt goes through this flow, including frontend-only apps with no backend, database, or Azure services. Those simply plan a single `frontend` service with **No datastore required**. @@ -408,9 +402,6 @@ includes an `**Execution Mode**: auto` metadata row. In autopilot, agents hand o `start_project_integrate` → `start_local_development`) and skip the Frontend preview and Next Steps views. Project-folder selection still happens before autopilot starts; a non-empty workspace requires choosing a new subfolder or an empty folder so unattended generation cannot write over existing content. -While the run is active, CoR sets both local chat and Agent Host defaults to Autopilot, enables global -tool auto-approval, and raises the request limit. It restores the user's prior global chat defaults when -Autopilot finishes, is turned off from the status bar, or reaches its safety deadline. --- @@ -466,11 +457,7 @@ silently so a stale copy can't make an agent follow outdated steps. ## The MCP tools The extension exposes these tools to Copilot through the `vscode-azureresourcegroups.mcp` server -("Copilot Azure Resources Extension Tools"). The server starts eagerly on a loopback TCP transport so -the Copilot Harness can connect before its first tool call. Agents call the tools to open views and -trigger the next stage. Agent manifests include both the Local harness reference names and the -Copilot Harness server label and tool-search alias because the two harnesses resolve those allowlist -entries differently. +("Copilot Azure Resources Extension Tools"). | Tool | Effect | | --- | --- | @@ -569,18 +556,15 @@ The diagnostics object has four fields: | Field | Value | | --- | --- | -| `prompt` | The project description the user typed. | -| `createdAt` | ISO‑8601 timestamp of when the create-project view flow started. | -| `systemInfo` | The operating system, CPU, Node.js, and VS Code versions captured when the project started. | +| `prompt` | The submitted project description. | +| `createdAt` | ISO-8601 timestamp of when Plan was pressed. | +| `systemInfo` | The operating system, CPU, Node.js, and VS Code versions captured on prompt submission. | | `diagnosticEvents` | Up to the **75 most recent** events, each: `timestamp`, `name` (command/tool), `type` (`extensionAction` \| `mcpTool` \| `webviewAction`), `status` (`start` \| `success` \| `error`), and a `properties` bag. Error messages are **masked** before being recorded. | `report_agent_launch` contributes at most one diagnostic lifecycle for each agent in a chat session. -Starting the create-project view flow resets cached CoR project state before Copilot readiness checks -and model discovery. Pressing **Plan** records the submitted prompt without resetting that state, -so the model-discovery diagnostics remain available alongside the planning and later-stage events. -The telemetry-only `corProjectId` is created when the prompt is submitted, not when the create view -loads or models are discovered. Subsequent project events reuse it. +Pressing **Plan** resets cached CoR project state and records the submitted prompt, timestamp, +system info, and telemetry-only `corProjectId`. An empty catalog gets one retry after one second; the retry budget resets on extension-host reload. Discovery adds these properties to the `createProjectWithCopilot` event and CoR telemetry: diff --git a/src/commands/copilotOnRails/registerDebugSessionWatcher.ts b/src/commands/copilotOnRails/registerDebugSessionWatcher.ts index 242ffbe8e..9bfc63ac4 100644 --- a/src/commands/copilotOnRails/registerDebugSessionWatcher.ts +++ b/src/commands/copilotOnRails/registerDebugSessionWatcher.ts @@ -191,10 +191,7 @@ function computeOutcome(tracked: TrackedDebugSession): 'succeeded' | 'failed' | function reportDebugSessionEvent(eventName: string, properties: Record, measurements?: Record): void { void callWithTelemetryAndErrorHandling(eventName, async (context: IActionContext) => { context.telemetry.properties.isCopilotEvent = 'true'; - const projectId = getCorProjectId(); - if (projectId) { - context.telemetry.properties.corProjectId = projectId; - } + context.telemetry.properties.corProjectId = getCorProjectId(); Object.assign(context.telemetry.properties, properties); if (measurements) { Object.assign(context.telemetry.measurements, measurements); diff --git a/src/utils/copilotOnRails/diagnosticUtils.ts b/src/utils/copilotOnRails/diagnosticUtils.ts index d1bb01542..5a12ae7e0 100644 --- a/src/utils/copilotOnRails/diagnosticUtils.ts +++ b/src/utils/copilotOnRails/diagnosticUtils.ts @@ -24,7 +24,7 @@ export function getPrompt(): string | undefined { const createdAtKey: string = 'copilotOnRails.createdAt'; /** - * Stamps the time the create-project view flow started as an ISO 8601 string. + * Stamps the prompt submission time as an ISO 8601 string. */ export function recordCreatedAt(): void { void ext.context.workspaceState.update(createdAtKey, new Date().toISOString()); diff --git a/src/utils/copilotOnRails/telemetryUtils.ts b/src/utils/copilotOnRails/telemetryUtils.ts index 2c38fce15..defb98951 100644 --- a/src/utils/copilotOnRails/telemetryUtils.ts +++ b/src/utils/copilotOnRails/telemetryUtils.ts @@ -26,16 +26,16 @@ const projectIdKey: string = 'copilotOnRails.projectId'; /** * Returns the persistent project guid associated with the workspace project. + * Automatically handles generating and caching one on first use. * * Note: The project guid is important for stitching together the full chain of Copilot on Rails telemetry. */ -export function getCorProjectId(): string | undefined { - return ext.context.workspaceState.get(projectIdKey); -} - -export async function initializeCorProjectId(): Promise { - const projectId = getCorProjectId() ?? uuidv4(); - await ext.context.workspaceState.update(projectIdKey, projectId); +export function getCorProjectId(): string { + let projectId: string | undefined = ext.context.workspaceState.get(projectIdKey); + if (!projectId) { + projectId = uuidv4(); + void ext.context.workspaceState.update(projectIdKey, projectId); + } return projectId; } @@ -51,10 +51,7 @@ export async function callWithDiagnosticsAndTelemetryHandling( command: (context: CopilotOnRailsContext) => Promise, ): Promise { context.telemetry.properties.isCopilotEvent = 'true'; - const projectId = getCorProjectId(); - if (projectId && eventDetails.name !== corId('createProjectWithCopilot')) { - context.telemetry.properties.corProjectId = projectId; - } + context.telemetry.properties.corProjectId = getCorProjectId(); if (eventDetails.extras) { context.telemetry.properties.copilotSessionId = eventDetails.extras.sessionId; diff --git a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts index fd16e1b03..0a499fadb 100644 --- a/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts +++ b/src/webviews/copilotOnRails/extension/controllers/CreateProjectViewController.ts @@ -11,9 +11,9 @@ import { azureProjectPlanAgent } from "../../../../constants"; import { ext } from "../../../../extensionVariables"; import { projectSubmissionState } from "../../../../tree/project/projectSubmissionState"; import { CopilotOnRailsContext } from "../../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { recordPrompt } from "../../../../utils/copilotOnRails/diagnosticUtils"; import { DEFAULT_CHAT_MODEL } from "../../../../utils/copilotOnRails/modelSelection"; -import { callWithDiagnosticsAndTelemetryHandling, corId, initializeCorProjectId, setCorProp } from "../../../../utils/copilotOnRails/telemetryUtils"; +import { prepareNewCorProject } from "../../../../utils/copilotOnRails/prepareNewCorProject"; +import { callWithDiagnosticsAndTelemetryHandling, corId, setCorProp } from "../../../../utils/copilotOnRails/telemetryUtils"; import { type CreateProjectViewControllerType, type LoadingViewConfiguration } from "../../views/utils/viewConfigTypes"; import { getCopilotOnRailsBundleLocation } from "../copilotOnRailsBundleLocation"; import { openLoadingView } from "../openLoadingView"; @@ -43,8 +43,7 @@ export class CreateProjectViewController extends CopilotOnRailsWebviewController } private async planProject(prompt: string, model?: string): Promise { - recordPrompt(prompt); - await initializeCorProjectId(); + await prepareNewCorProject(prompt); await this.openChatWithQuery(prompt, model); } diff --git a/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts b/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts index 7202cf594..4562e8383 100644 --- a/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts +++ b/src/webviews/copilotOnRails/extension/controllers/LoadingViewController.ts @@ -40,10 +40,7 @@ export class LoadingViewController extends CopilotOnRailsWebviewController { context.errorHandling.suppressDisplay = true; context.telemetry.properties.isCopilotEvent = 'true'; - const projectId = getCorProjectId(); - if (projectId) { - context.telemetry.properties.corProjectId = projectId; - } + context.telemetry.properties.corProjectId = getCorProjectId(); this.panel.dispose(); await vscode.commands.executeCommand(copilotOnRailsCommandIds.resumeProjectWithCopilot); }); diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index e0731426d..b81d5ddbc 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -10,7 +10,6 @@ import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/regis import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext, ensureRequiredCopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; import { AvailableChatModel, DEFAULT_CHAT_MODEL, getDefaultOpusModelOption, getModelPickerOptions, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; -import { prepareNewCorProject } from "../../../utils/copilotOnRails/prepareNewCorProject"; import { setCorErrorProp, setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; @@ -78,7 +77,6 @@ export async function resumeCreateProjectViewAfterReload(): Promise { } async function openCreateProjectView(context: CopilotOnRailsContext, initialPrompt?: string, initialModel?: string): Promise { - await prepareNewCorProject(initialPrompt ?? ''); if (!(await ensureCopilotChatReady(context))) { return; } diff --git a/src/webviews/copilotOnRails/extension/openRequirementsView.ts b/src/webviews/copilotOnRails/extension/openRequirementsView.ts index 8260221e6..fe7526547 100644 --- a/src/webviews/copilotOnRails/extension/openRequirementsView.ts +++ b/src/webviews/copilotOnRails/extension/openRequirementsView.ts @@ -80,10 +80,7 @@ export async function openRequirementsViewFromWorkspace(context: IActionContext) ); context.telemetry.properties.isCopilotEvent = 'true'; - const projectId = getCorProjectId(); - if (projectId) { - context.telemetry.properties.corProjectId = projectId; - } + context.telemetry.properties.corProjectId = getCorProjectId(); context.telemetry.properties.requirementsSelected = String(!!selected); if (selected) { await openRequirementsViewAsync(selected); diff --git a/test/copilotOnRails/createProjectWithCopilot.test.ts b/test/copilotOnRails/createProjectWithCopilot.test.ts index 7b36fe3bd..b0071501c 100644 --- a/test/copilotOnRails/createProjectWithCopilot.test.ts +++ b/test/copilotOnRails/createProjectWithCopilot.test.ts @@ -8,7 +8,7 @@ import assert from 'assert'; import { ext } from '../../src/extensionVariables'; import { ensureRequiredCopilotOnRailsContext } from '../../src/utils/copilotOnRails/CopilotOnRailsContext'; import { AvailableChatModel, getSupportedModelOptions } from '../../src/utils/copilotOnRails/modelSelection'; -import { callWithDiagnosticsAndTelemetryHandling, getCorProjectId, initializeCorProjectId } from '../../src/utils/copilotOnRails/telemetryUtils'; +import { callWithDiagnosticsAndTelemetryHandling, getCorProjectId } from '../../src/utils/copilotOnRails/telemetryUtils'; import { OPEN_PROJECT_FOLDER_OPTIONS, PROJECT_FOLDER_SELECTION_TELEMETRY_KEY, @@ -97,7 +97,7 @@ suite('Create Project with Copilot model picker diagnostics', () => { assert.strictEqual(context.diagnostics.properties.modelPickerError, undefined); }); - test('leaves discovery unassigned and initializes the project id at prompt submission', async () => { + test('creates and reuses the project id through the shared wrapper', async () => { const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); const state = new Map(); Object.defineProperty(ext, 'context', { @@ -115,13 +115,11 @@ suite('Create Project with Copilot model picker diagnostics', () => { await callWithDiagnosticsAndTelemetryHandling( discoveryContext, { type: 'extensionAction', name: 'copilotOnRails.createProjectWithCopilot' }, - async () => { assert.strictEqual(getCorProjectId(), undefined); }, + async () => { assert.ok(getCorProjectId()); }, ); - assert.strictEqual(getCorProjectId(), undefined); - assert.strictEqual(discoveryContext.telemetry.properties.corProjectId, undefined); - - const projectId = await initializeCorProjectId(); + const projectId = getCorProjectId(); assert.ok(projectId); + assert.strictEqual(discoveryContext.telemetry.properties.corProjectId, projectId); const submitContext = await createTestActionContext(); await callWithDiagnosticsAndTelemetryHandling( submitContext, @@ -130,14 +128,13 @@ suite('Create Project with Copilot model picker diagnostics', () => { ); assert.strictEqual(submitContext.telemetry.properties.corProjectId, projectId); assert.strictEqual(getCorProjectId(), projectId); - assert.strictEqual(await initializeCorProjectId(), projectId); const nextCreateContext = await createTestActionContext(); await callWithDiagnosticsAndTelemetryHandling( nextCreateContext, { type: 'extensionAction', name: 'copilotOnRails.createProjectWithCopilot' }, async () => { assert.strictEqual(getCorProjectId(), projectId); }, ); - assert.strictEqual(nextCreateContext.telemetry.properties.corProjectId, undefined); + assert.strictEqual(nextCreateContext.telemetry.properties.corProjectId, projectId); } finally { if (originalContext) { Object.defineProperty(ext, 'context', originalContext); From 4aa76c7d5dcceff2465e6711d6bde834e7a49181 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 6 Oct 2026 00:42:13 -0700 Subject: [PATCH 32/34] Remove debug logs --- .../extension/createProjectWithCopilot.ts | 27 +------------------ .../extension/harnessSettings.ts | 2 -- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts index b81d5ddbc..1daec9df9 100644 --- a/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts +++ b/src/webviews/copilotOnRails/extension/createProjectWithCopilot.ts @@ -9,7 +9,7 @@ import { ensureCopilotChatReady } from "../../../commands/copilotOnRails/openCha import { copilotOnRailsCommandIds } from "../../../commands/copilotOnRails/registerCopilotOnRailsCommands"; import { DEBUG_PLAN_FILE_GLOB, PROJECT_PLAN_FILE_GLOB } from "../../../tree/project/projectPlanFiles"; import { CopilotOnRailsContext, ensureRequiredCopilotOnRailsContext } from "../../../utils/copilotOnRails/CopilotOnRailsContext"; -import { AvailableChatModel, DEFAULT_CHAT_MODEL, getDefaultOpusModelOption, getModelPickerOptions, getSupportedModelName, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; +import { AvailableChatModel, DEFAULT_CHAT_MODEL, getDefaultOpusModelOption, getModelPickerOptions, getSupportedModelOptions } from "../../../utils/copilotOnRails/modelSelection"; import { setCorErrorProp, setCorProp } from "../../../utils/copilotOnRails/telemetryUtils"; import { CreateProjectViewController } from "./controllers/CreateProjectViewController"; import { getRecentPrompts } from "./recentPrompts"; @@ -81,34 +81,11 @@ async function openCreateProjectView(context: CopilotOnRailsContext, initialProm return; } - console.log('[Copilot on Rails model picker] Calling selectChatModels with vendor "copilotcli".'); const availableModels = await discoverModelPickerModels(context, () => vscode.lm.selectChatModels({ vendor: 'copilotcli' })); const modelOptions = getModelPickerOptions(availableModels, initialModel); - console.log( - `[Copilot on Rails model picker] Available catalog contains ${availableModels.length} model(s):`, - availableModels.map(model => ({ - id: model.id, - vendor: model.vendor, - family: model.family, - version: model.version, - name: model.name, - maxInputTokens: model.maxInputTokens, - supportedName: getSupportedModelName(model.id, model.family, model.name), - })), - ); - - console.log( - `[Copilot on Rails model picker] Discovery produced ${modelOptions.length} picker option(s):`, - modelOptions, - ); - const selectedModel = initialModel && (initialModel === DEFAULT_CHAT_MODEL || modelOptions.includes(initialModel)) ? initialModel : getDefaultOpusModelOption(availableModels) ?? modelOptions[0] ?? DEFAULT_CHAT_MODEL; - console.log('[Copilot on Rails model picker] Initial and selected model:', { - requestedInitialModel: initialModel, - selectedModel, - }); const controller = new CreateProjectViewController({ title: vscode.l10n.t('Create with Copilot'), @@ -142,7 +119,6 @@ export async function discoverModelPickerModels( if (models.length === 0 && !modelDiscoveryRetryUsed) { modelDiscoveryRetryUsed = true; setCorProp(context, 'modelPickerRetried', true); - console.log('[Copilot on Rails model picker] Empty catalog; retrying once after one second.'); await new Promise(resolve => setTimeout(resolve, MODEL_DISCOVERY_RETRY_DELAY_MS)); models = await selectModels(); } @@ -150,7 +126,6 @@ export async function discoverModelPickerModels( setCorErrorProp(context, 'modelPickerError', parseError(error).message); recordModelPickerDiscovery(context, [], []); setCorProp(context, 'modelPickerOutcome', 'queryFailed'); - console.log('[Copilot on Rails model picker] Model discovery failed; using the VS Code default. See modelPickerError in diagnostics.'); return []; } recordModelPickerDiscovery(context, models, getSupportedModelOptions(models)); diff --git a/src/webviews/copilotOnRails/extension/harnessSettings.ts b/src/webviews/copilotOnRails/extension/harnessSettings.ts index 7c2f3262d..545ea3726 100644 --- a/src/webviews/copilotOnRails/extension/harnessSettings.ts +++ b/src/webviews/copilotOnRails/extension/harnessSettings.ts @@ -46,7 +46,5 @@ export async function setCopilotCliAutoModelEnabled(enabled: boolean): Promise Date: Tue, 6 Oct 2026 10:09:44 -0700 Subject: [PATCH 33/34] More improvements --- .../copilotOnRails/openChatWithAgent.ts | 2 +- src/utils/copilotOnRails/modelSelection.ts | 2 + .../extension/harnessSettings.ts | 2 +- .../createProjectWithCopilot.test.ts | 86 ++--------- test/copilotOnRails/modelSelection.test.ts | 76 +++------- test/copilotOnRails/openChatWithAgent.test.ts | 141 ++++++++---------- 6 files changed, 93 insertions(+), 216 deletions(-) diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index 5a7b6a0fe..c7c52c502 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -126,7 +126,7 @@ export async function launchAgentChat(context: CopilotOnRailsContext, agentName: agentLaunchInProgress = true; try { const resolvedModel = model ?? getSessionModel(); - await ensureCopilotHarnessOn(resolvedModel === AUTO_CHAT_MODEL); + await ensureCopilotHarnessOn({ useAutoModel: resolvedModel === AUTO_CHAT_MODEL }); // Custom-agent commands are registered from the focused chat widget. Open Chat before // creating the fresh session so first-time launches do not wait on a widget that does not exist. diff --git a/src/utils/copilotOnRails/modelSelection.ts b/src/utils/copilotOnRails/modelSelection.ts index 6f9deb287..241020d69 100644 --- a/src/utils/copilotOnRails/modelSelection.ts +++ b/src/utils/copilotOnRails/modelSelection.ts @@ -12,6 +12,8 @@ export interface AvailableChatModel { } export const supportedModelNames = ['Opus', 'Sonnet', 'GPT Sol', 'GPT Terra'] as const; + +/** Defer to whatever model is already highlighted in VS Code Chat by omitting modelSelector (not the same as auto). */ export const DEFAULT_CHAT_MODEL = 'default'; export const AUTO_CHAT_MODEL = 'Auto'; diff --git a/src/webviews/copilotOnRails/extension/harnessSettings.ts b/src/webviews/copilotOnRails/extension/harnessSettings.ts index 545ea3726..6f6405024 100644 --- a/src/webviews/copilotOnRails/extension/harnessSettings.ts +++ b/src/webviews/copilotOnRails/extension/harnessSettings.ts @@ -23,7 +23,7 @@ export const COPILOT_HARNESS_SETTING_IDS = HARNESS_SETTINGS.map(({ prefix, key } * Enables the Copilot Harness and configures explicit Auto selection at Workspace scope. Like the raised chat request * budget, the workspace override is cheap and disposable, so it is intentionally left in place. */ -export async function ensureCopilotHarnessOn(useAutoModel = false): Promise { +export async function ensureCopilotHarnessOn({ useAutoModel = false }: { useAutoModel?: boolean } = {}): Promise { const folder = workspace.workspaceFolders?.[0]; if (!folder) { return; diff --git a/test/copilotOnRails/createProjectWithCopilot.test.ts b/test/copilotOnRails/createProjectWithCopilot.test.ts index b0071501c..7a64be8f5 100644 --- a/test/copilotOnRails/createProjectWithCopilot.test.ts +++ b/test/copilotOnRails/createProjectWithCopilot.test.ts @@ -5,16 +5,13 @@ import { createTestActionContext } from '@microsoft/vscode-azext-utils'; import assert from 'assert'; -import { ext } from '../../src/extensionVariables'; import { ensureRequiredCopilotOnRailsContext } from '../../src/utils/copilotOnRails/CopilotOnRailsContext'; -import { AvailableChatModel, getSupportedModelOptions } from '../../src/utils/copilotOnRails/modelSelection'; -import { callWithDiagnosticsAndTelemetryHandling, getCorProjectId } from '../../src/utils/copilotOnRails/telemetryUtils'; +import { AvailableChatModel } from '../../src/utils/copilotOnRails/modelSelection'; import { OPEN_PROJECT_FOLDER_OPTIONS, PROJECT_FOLDER_SELECTION_TELEMETRY_KEY, ProjectFolderSelection, recordProjectFolderSelection, - recordModelPickerDiscovery, discoverModelPickerModels, validateProjectSubfolderName, } from '../../src/webviews/copilotOnRails/extension/createProjectWithCopilot'; @@ -87,63 +84,6 @@ suite('Create Project with Copilot model picker diagnostics', () => { assert.deepStrictEqual(context.diagnostics.properties.modelPickerFilteredModels, []); }); - test('records available models when discovery succeeds', async () => { - const context = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); - const models = await discoverModelPickerModels(context, async () => [sonnet]); - - assert.deepStrictEqual(models, [sonnet]); - assert.strictEqual(context.diagnostics.properties.modelPickerOutcome, 'ready'); - assert.strictEqual(context.diagnostics.properties.modelPickerVendor, 'copilotcli'); - assert.strictEqual(context.diagnostics.properties.modelPickerError, undefined); - }); - - test('creates and reuses the project id through the shared wrapper', async () => { - const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); - const state = new Map(); - Object.defineProperty(ext, 'context', { - configurable: true, - value: { - globalState: { get: () => false }, - workspaceState: { - get: (key: string, defaultValue?: unknown) => state.get(key) ?? defaultValue, - update: async (key: string, value: unknown) => { state.set(key, value); }, - }, - }, - }); - try { - const discoveryContext = await createTestActionContext(); - await callWithDiagnosticsAndTelemetryHandling( - discoveryContext, - { type: 'extensionAction', name: 'copilotOnRails.createProjectWithCopilot' }, - async () => { assert.ok(getCorProjectId()); }, - ); - const projectId = getCorProjectId(); - assert.ok(projectId); - assert.strictEqual(discoveryContext.telemetry.properties.corProjectId, projectId); - const submitContext = await createTestActionContext(); - await callWithDiagnosticsAndTelemetryHandling( - submitContext, - { type: 'webviewAction', name: 'createProjectSubmitPrompt' }, - async () => { assert.strictEqual(getCorProjectId(), projectId); }, - ); - assert.strictEqual(submitContext.telemetry.properties.corProjectId, projectId); - assert.strictEqual(getCorProjectId(), projectId); - const nextCreateContext = await createTestActionContext(); - await callWithDiagnosticsAndTelemetryHandling( - nextCreateContext, - { type: 'extensionAction', name: 'copilotOnRails.createProjectWithCopilot' }, - async () => { assert.strictEqual(getCorProjectId(), projectId); }, - ); - assert.strictEqual(nextCreateContext.telemetry.properties.corProjectId, projectId); - } finally { - if (originalContext) { - Object.defineProperty(ext, 'context', originalContext); - } else { - Reflect.deleteProperty(ext, 'context'); - } - } - }); - const sonnet: AvailableChatModel = { id: 'claude-sonnet-5', vendor: 'copilotcli', @@ -160,19 +100,18 @@ suite('Create Project with Copilot model picker diagnostics', () => { }; for (const scenario of [ - { name: 'empty catalog', models: [], filtered: 0, outcome: 'emptyCatalog' }, - { name: 'unsupported catalog', models: [haiku], filtered: 0, outcome: 'noSupportedModels' }, - { name: 'supported catalog with duplicate labels', models: [sonnet, haiku, sonnet], filtered: 1, outcome: 'ready' }, + { name: 'empty catalog', models: [], filtered: [], outcome: 'emptyCatalog' }, + { name: 'unsupported catalog', models: [haiku], filtered: [], outcome: 'noSupportedModels' }, + { name: 'supported catalog with duplicate labels', models: [sonnet, haiku, sonnet], filtered: [sonnet.name], outcome: 'ready' }, ]) { test(`records counts and outcome for ${scenario.name} in telemetry and diagnostics`, async () => { const context = ensureRequiredCopilotOnRailsContext(await createTestActionContext()); - const options = getSupportedModelOptions(scenario.models); - - recordModelPickerDiscovery(context, scenario.models, options); + assert.deepStrictEqual(await discoverModelPickerModels(context, async () => scenario.models), scenario.models); const expected = { + modelPickerVendor: 'copilotcli', modelPickerAvailableModelCount: scenario.models.length, - modelPickerFilteredModelCount: scenario.filtered, + modelPickerFilteredModelCount: scenario.filtered.length, modelPickerOutcome: scenario.outcome, }; for (const [key, value] of Object.entries(expected)) { @@ -180,13 +119,10 @@ suite('Create Project with Copilot model picker diagnostics', () => { assert.strictEqual(context.diagnostics.properties[key], value); } assert.deepStrictEqual(context.diagnostics.properties.modelPickerAvailableModels, scenario.models.map(model => model.name)); - assert.deepStrictEqual(context.diagnostics.properties.modelPickerFilteredModels, options); - assert.strictEqual(context.telemetry.properties.modelPickerAvailableModels, undefined); - assert.strictEqual(context.telemetry.properties.modelPickerFilteredModels, undefined); - for (const key of ['modelPickerReturnedModelCount', 'modelPickerSupportedModelCount', 'modelPickerFilteredOutModelCount', 'availableModelCount', 'modelPickerReturnedModels', 'modelPickerShownModels']) { - assert.strictEqual(context.telemetry.properties[key], undefined); - assert.strictEqual(context.diagnostics.properties[key], undefined); - } + assert.deepStrictEqual(context.diagnostics.properties.modelPickerFilteredModels, scenario.filtered); + assert.deepStrictEqual(Object.keys(context.telemetry.properties).filter(key => key.startsWith('modelPicker')).sort(), [ + 'modelPickerAvailableModelCount', 'modelPickerFilteredModelCount', 'modelPickerOutcome', 'modelPickerRetried', 'modelPickerVendor', + ]); }); } }); diff --git a/test/copilotOnRails/modelSelection.test.ts b/test/copilotOnRails/modelSelection.test.ts index c6fd05227..bd00c5181 100644 --- a/test/copilotOnRails/modelSelection.test.ts +++ b/test/copilotOnRails/modelSelection.test.ts @@ -25,92 +25,52 @@ const localModel: AvailableChatModel = { const cliModel: AvailableChatModel = { ...localModel, + id: 'cli-sol', vendor: 'copilotcli', }; suite('Copilot on Rails model selection', () => { - test('resolves explicit Auto without needing the CLI catalog to advertise it', () => { + test('resolves explicit Auto and default without substituting missing named models', () => { assert.deepStrictEqual( resolveCopilotHarnessModelSelector(AUTO_CHAT_MODEL, []), { id: 'auto', vendor: 'agent-host-copilotcli' }, ); + assert.strictEqual(resolveCopilotHarnessModelSelector('Unavailable model', [cliModel]), undefined); + assert.strictEqual(resolveCopilotHarnessModelSelector(DEFAULT_CHAT_MODEL, [{ ...cliModel, id: DEFAULT_CHAT_MODEL }]), undefined); }); - test('does not treat Auto family metadata as a supported named model', () => { - const auto = { ...cliModel, id: 'auto', name: AUTO_CHAT_MODEL, family: 'claude-opus-4.6' }; - assert.deepStrictEqual(getSupportedModelOptions([auto, cliModel]), [cliModel.name]); + test('excludes Auto from named-model filtering and prefers an actual CLI Opus model', () => { + const opus = { ...cliModel, id: 'claude-opus-4.6', name: 'Claude Opus 4.6', family: 'opus', version: '4.6' }; + const auto = { ...opus, id: 'auto', name: AUTO_CHAT_MODEL }; + assert.deepStrictEqual(getSupportedModelOptions([auto, opus, cliModel]), [opus.name, cliModel.name]); assert.deepStrictEqual(getModelPickerOptions([auto]), []); - assert.strictEqual(getDefaultOpusModelOption([auto, cliModel]), undefined); + assert.strictEqual(getDefaultOpusModelOption([auto, opus]), opus.name); }); - test('offers Auto once and keeps a named model first regardless of the CLI Auto flag', () => { + test('offers Auto once after named models and preserves a reopened default selection', () => { const auto = { ...cliModel, id: 'auto', name: AUTO_CHAT_MODEL, family: 'claude-sonnet-5' }; assert.deepStrictEqual(getModelPickerOptions([cliModel]), [cliModel.name, AUTO_CHAT_MODEL]); assert.deepStrictEqual(getModelPickerOptions([auto, cliModel]), [cliModel.name, AUTO_CHAT_MODEL]); - assert.deepStrictEqual(getModelPickerOptions([cliModel], AUTO_CHAT_MODEL), [cliModel.name, AUTO_CHAT_MODEL]); + assert.deepStrictEqual(getModelPickerOptions([cliModel], DEFAULT_CHAT_MODEL), [DEFAULT_CHAT_MODEL, cliModel.name, AUTO_CHAT_MODEL]); }); test('maps a discovered CLI model id to the Agent Host launch vendor', () => { assert.deepStrictEqual( resolveCopilotHarnessModelSelector('GPT-6.1 Sol', [cliModel]), - { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + { id: cliModel.id, vendor: 'agent-host-copilotcli' }, ); }); - test('does not resolve an unavailable model to a fallback selector', () => { - assert.strictEqual(resolveCopilotHarnessModelSelector('Unavailable model', [cliModel]), undefined); - }); - - test('omits a selector for the default sentinel even if a provider lists that id', () => { - assert.strictEqual( - resolveCopilotHarnessModelSelector(DEFAULT_CHAT_MODEL, [{ ...cliModel, id: DEFAULT_CHAT_MODEL }]), - undefined, - ); - }); - - test('provides no picker options when the catalog is empty', () => { + test('hides the picker when no supported named models are available', () => { assert.deepStrictEqual(getModelPickerOptions([]), []); - }); - - test('provides no picker options when no models pass the filter', () => { const model = { ...cliModel, id: 'haiku', name: 'Claude Haiku 4.5', family: 'haiku' }; assert.deepStrictEqual(getModelPickerOptions([model]), []); }); - test('keeps supported models available alongside explicit Auto', () => { - assert.deepStrictEqual(getModelPickerOptions([cliModel]), [cliModel.name, AUTO_CHAT_MODEL]); - }); - - test('preserves the default selection when reopening after models become available', () => { - assert.deepStrictEqual(getModelPickerOptions([cliModel], DEFAULT_CHAT_MODEL), [DEFAULT_CHAT_MODEL, cliModel.name, AUTO_CHAT_MODEL]); - }); - - test('resolves a qualified local model name', () => { - assert.strictEqual( - resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, cliModel]), - localModel, - ); - }); - - test('resolves a qualified Copilot CLI model without changing its vendor', () => { - assert.strictEqual( - resolveAvailableChatModel('GPT-6.1 Sol (copilotcli)', [localModel, cliModel]), - cliModel, - ); - }); - - test('does not fabricate a selector for an unavailable model', () => { - assert.strictEqual( - resolveAvailableChatModel('Unavailable model', [cliModel]), - undefined, - ); - }); - - test('resolves an unqualified picker name using the Copilot CLI model id', () => { - const model = { ...cliModel, id: 'cli-sol' }; - assert.strictEqual( - resolveAvailableChatModel('GPT-6.1 Sol', [model]), - model, - ); + test('resolves display names, IDs, and legacy vendor-qualified names', () => { + assert.strictEqual(resolveAvailableChatModel(cliModel.name, [cliModel]), cliModel); + assert.strictEqual(resolveAvailableChatModel(cliModel.id, [cliModel]), cliModel); + assert.strictEqual(resolveAvailableChatModel('GPT-6.1 Sol (copilot)', [localModel, cliModel]), localModel); + assert.strictEqual(resolveAvailableChatModel('GPT-6.1 Sol (copilotcli)', [localModel, cliModel]), cliModel); }); }); diff --git a/test/copilotOnRails/openChatWithAgent.test.ts b/test/copilotOnRails/openChatWithAgent.test.ts index 5f283466a..ab189a416 100644 --- a/test/copilotOnRails/openChatWithAgent.test.ts +++ b/test/copilotOnRails/openChatWithAgent.test.ts @@ -24,114 +24,93 @@ suite('Copilot on Rails agent chat launch', () => { settingUtils.updateWorkspaceSetting = originalUpdateWorkspaceSetting; }); - test('omits a model selector when the VS Code default is selected', async () => { - const selector = await resolveModelSelector(DEFAULT_CHAT_MODEL); + test('builds custom-agent requests with an explicit model or the VS Code default', async () => { + const agentName = 'azure-project-plan'; + const query = 'Build a project'; + const expected = { mode: agentName, query, waitForRequestAcceptance: true }; + const selector = { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }; + assert.deepStrictEqual(buildAgentChatModeOptions(agentName), { mode: agentName }); assert.deepStrictEqual( - buildAgentChatOpenOptions('azure-project-plan', 'Build a project', selector), - { mode: 'azure-project-plan', query: 'Build a project', waitForRequestAcceptance: true }, + buildAgentChatOpenOptions(agentName, query, selector), + { ...expected, modelSelector: selector }, ); - }); - - test('selects Auto explicitly when requested', async () => { assert.deepStrictEqual( - await resolveModelSelector(AUTO_CHAT_MODEL), - { id: 'auto', vendor: 'agent-host-copilotcli' }, + buildAgentChatOpenOptions(agentName, query, await resolveModelSelector(DEFAULT_CHAT_MODEL)), + expected, ); }); - test('disables CLI Auto during initial workspace setup', async () => { - await ensureCopilotHarnessOn(); + test('enables the Copilot Harness at workspace scope and honors explicit Auto', async () => { const folder = workspace.workspaceFolders?.[0]; - const settingAvailable = folder && workspace.getConfiguration('github.copilot.chat.cli', folder.uri).inspect('autoModel.enabled'); + assert.ok(folder, 'The extension test runner must provide a workspace'); + await ensureCopilotHarnessOn(); + assert.deepStrictEqual(workspaceSettingUpdates.slice(0, 2), [ + ['preferCopilotHarness', true, folder.uri.fsPath, 'chat.editor', ConfigurationTarget.Workspace], + ['defaultToCopilotHarness', true, folder.uri.fsPath, 'chat', ConfigurationTarget.Workspace], + ]); + await ensureCopilotHarnessOn({ useAutoModel: true }); + const settingAvailable = workspace.getConfiguration('github.copilot.chat.cli', folder.uri).inspect('autoModel.enabled'); const autoUpdates = workspaceSettingUpdates.filter(([key]) => key === 'autoModel.enabled'); assert.deepStrictEqual(autoUpdates, settingAvailable - ? [['autoModel.enabled', false, folder?.uri.fsPath, 'github.copilot.chat.cli', ConfigurationTarget.Workspace]] + ? [false, true].map(enabled => ['autoModel.enabled', enabled, folder.uri.fsPath, 'github.copilot.chat.cli', ConfigurationTarget.Workspace]) : []); }); - for (const model of [DEFAULT_CHAT_MODEL, AUTO_CHAT_MODEL]) { - test(`persists ${model} across phases and configures its workspace Auto flag`, async () => { - const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); - const state = new Map(); - Object.defineProperty(ext, 'context', { - configurable: true, - value: { - workspaceState: { - get: (key: string) => state.get(key), - update: async (key: string, value: unknown) => { state.set(key, value); }, - }, + test('preserves default and Auto selections across a phase handoff', async () => { + const originalContext = Object.getOwnPropertyDescriptor(ext, 'context'); + const state = new Map(); + Object.defineProperty(ext, 'context', { + configurable: true, + value: { + workspaceState: { + get: (key: string) => state.get(key), + update: async (key: string, value: unknown) => { state.set(key, value); }, }, - }); - try { + }, + }); + try { + for (const model of [DEFAULT_CHAT_MODEL, AUTO_CHAT_MODEL]) { await recordModel(model); - for (const phase of ['scaffold', 'integrate', 'debug', 'deploy'] as const) { - await recordPhase(phase); - assert.strictEqual(getSessionModel(), model); - const context = await createTestActionContext(); - const options = { query: 'Continue the project' }; - assert.deepStrictEqual(await buildChatOpenOptions(context, options), model === AUTO_CHAT_MODEL - ? { ...options, modelSelector: { id: 'auto', vendor: 'agent-host-copilotcli' } } - : options); - assert.strictEqual(context.telemetry.properties.chatModelSelectionSource, model === DEFAULT_CHAT_MODEL ? 'default' : 'previouslySelected'); - assert.strictEqual(context.telemetry.properties.chatModelResolved, String(model === AUTO_CHAT_MODEL)); - } - const folder = workspace.workspaceFolders?.[0]; - const settingAvailable = folder && workspace.getConfiguration('github.copilot.chat.cli', folder.uri).inspect('autoModel.enabled'); - const expectedUpdate = ['autoModel.enabled', model === AUTO_CHAT_MODEL, folder?.uri.fsPath, 'github.copilot.chat.cli', ConfigurationTarget.Workspace]; - assert.deepStrictEqual(workspaceSettingUpdates, settingAvailable ? Array(4).fill(expectedUpdate) : []); - } finally { - if (originalContext) { - Object.defineProperty(ext, 'context', originalContext); - } else { - Reflect.deleteProperty(ext, 'context'); - } + await recordPhase('scaffold'); + assert.strictEqual(getSessionModel(), model); + const context = await createTestActionContext(); + const options = { query: 'Continue the project' }; + assert.deepStrictEqual(await buildChatOpenOptions(context, options), model === AUTO_CHAT_MODEL + ? { ...options, modelSelector: { id: 'auto', vendor: 'agent-host-copilotcli' } } + : options); + assert.strictEqual(context.telemetry.properties.chatModelSelectionSource, model === DEFAULT_CHAT_MODEL ? 'default' : 'previouslySelected'); + assert.strictEqual(context.telemetry.properties.chatModelResolved, String(model === AUTO_CHAT_MODEL)); } - }); - } - - test('builds options that select the custom agent without submitting a prompt', () => { - assert.deepStrictEqual( - buildAgentChatModeOptions('azure-project-integrate'), - { mode: 'azure-project-integrate' }, - ); + const folder = workspace.workspaceFolders?.[0]; + const settingAvailable = folder && workspace.getConfiguration('github.copilot.chat.cli', folder.uri).inspect('autoModel.enabled'); + const expectedUpdates = [false, true].map(enabled => + ['autoModel.enabled', enabled, folder?.uri.fsPath, 'github.copilot.chat.cli', ConfigurationTarget.Workspace]); + assert.deepStrictEqual(workspaceSettingUpdates, settingAvailable ? expectedUpdates : []); + } finally { + if (originalContext) { + Object.defineProperty(ext, 'context', originalContext); + } else { + Reflect.deleteProperty(ext, 'context'); + } + } }); - test('resolves the agent by name when submitting the prompt', () => { + test('marks Autopilot requests once without changing their agent or model', () => { assert.deepStrictEqual( buildAgentChatOpenOptions( - 'azure-project-plan', - 'Build a project', + 'azure-project-scaffold', + 'Continue the project', { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, + true, ), { - mode: 'azure-project-plan', - query: 'Build a project', + mode: 'azure-project-scaffold', + query: '[AUTOPILOT MODE] Continue the project', modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, waitForRequestAcceptance: true, }, ); - }); - - test('carries Autopilot into fresh phase chats without changing the agent or model', () => { - for (const agentName of ['azure-project-scaffold', 'azure-project-integrate', 'azure-debug-plan', 'azure-debug-generate', 'azure-deploy']) { - assert.deepStrictEqual( - buildAgentChatOpenOptions(agentName, 'Continue the project', { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, true), - { - mode: agentName, - query: '[AUTOPILOT MODE] Continue the project', - modelSelector: { id: 'gpt-6.1-sol', vendor: 'agent-host-copilotcli' }, - waitForRequestAcceptance: true, - }, - ); - } - }); - - test('does not duplicate an existing Autopilot marker', () => { const query = '[AUTOPILOT MODE] I approve the plan.'; assert.strictEqual(buildAgentChatOpenOptions('azure-project-scaffold', query, undefined, true).query, query); }); - - test('preserves interactive handoff prompts', () => { - assert.strictEqual(buildAgentChatOpenOptions('azure-project-integrate', 'Continue the project', undefined, false).query, 'Continue the project'); - }); }); From 0c63cf1a43a7e7ab6d3da775ab3591eb25eb34c2 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 6 Oct 2026 10:21:11 -0700 Subject: [PATCH 34/34] Add a comment --- src/commands/copilotOnRails/openChatWithAgent.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/commands/copilotOnRails/openChatWithAgent.ts b/src/commands/copilotOnRails/openChatWithAgent.ts index c7c52c502..b9f3d5d5b 100644 --- a/src/commands/copilotOnRails/openChatWithAgent.ts +++ b/src/commands/copilotOnRails/openChatWithAgent.ts @@ -43,9 +43,11 @@ export function registerWorkspaceTrustTracking(): void { * modelSelector object that VS Code's chat commands expect. */ export async function resolveModelSelector(displayName: string): Promise<{ id?: string; vendor?: string } | undefined> { + // If auto or default, defer to VS Code chat, since we don't need to provide any specific model if (displayName === DEFAULT_CHAT_MODEL || displayName === AUTO_CHAT_MODEL) { return resolveCopilotHarnessModelSelector(displayName, []); } + try { const models = await vscode.lm.selectChatModels({ vendor: 'copilotcli' }); return resolveCopilotHarnessModelSelector(displayName, models);