fix(agent): stop identities only on a failed check

.github/workflows/ci.yml

@@ -364,6 +364,35 @@ jobs:
         env:
           AWS_LC_SYS_NO_ASM: true
 
+      # A failed Agent Identity conformance run keeps its work directory, whose agent data directories and logs are the
+      # only record of the state that made it fail.
+      # The filter matches that directory under `target`, the temporary directory or `ProgramData`, and a file the runner
+      # cannot read, such as one only SYSTEM can open, is reported and skipped.
+      # UNTESTED: these steps run only after a failure; when changing them, check by hand with a failing conformance test
+      # that the artifact holds the kept agent data directories.
+      - name: Collect kept Agent Identity conformance directories
+        if: ${{ failure() }}
+        shell: pwsh
+        run: |
+          $Destination = Join-Path $Env:RUNNER_TEMP 'agent-identity-conformance'
+          New-Item -ItemType Directory -Force $Destination | Out-Null
+          $Parents = @('target', [IO.Path]::GetTempPath(), $Env:ProgramData) | Where-Object { $_ }
+          $Index = 0
+          Get-ChildItem -Path $Parents -Directory -Filter 'agent-identity-integration*' -ErrorAction SilentlyContinue |
+            ForEach-Object {
+              $Index++
+              $Copy = Join-Path $Destination "$Index-$($_.Name)"
+              Copy-Item -LiteralPath $_.FullName -Destination $Copy -Recurse -ErrorAction Continue
+            }
+
+      - name: Upload kept Agent Identity conformance directories
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: agent-identity-conformance-${{ matrix.os }}-${{ matrix.arch }}-attempt-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/agent-identity-conformance
+          if-no-files-found: ignore
+
       - name: Timing-sensitive tests
         run: |
           cargo test --locked --target ${{ matrix.target }} -p testsuite --test integration_tests timing:: -- --ignored --test-threads=1

crates/agent-identity/src/private_file.rs

@@ -68,6 +68,22 @@ fn untrusted_stored_identity(path: &Utf8Path, source: anyhow::Error) -> anyhow::
     })
 }
 
+/// Turns a failed open of the authority directory or `identity.json` at `path` into an [`UntrustedStoredIdentity`] when
+/// it fails the trust check, and leaves any other failure, which only kept the check from running, an ordinary error.
+///
+/// [`crates/agent-identity/INTENT.md`] ("Local state"): only a stored identity that fails the check stops all identity
+/// processing.
+/// A failure that names no checked path, which a valid path never causes, counts as untrusted.
+///
+/// [`crates/agent-identity/INTENT.md`]: ../INTENT.md
+fn check_failure(path: &Utf8Path, error: anyhow::Error) -> anyhow::Error {
+    if error.downcast_ref::<private_fs::Error>().is_none_or(fails_trust_check) {
+        untrusted_stored_identity(path, error)
+    } else {
+        error.context(format!("check stored identity {path}"))
+    }
+}
+
 /// Gives SYSTEM full control, and the agent's account too when `grant_current_user` is set.
 ///
 /// [`crates/agent-identity/INTENT.md`] ("Local state"): on Linux and macOS the agent's account owns what it creates and
@@ -250,7 +266,7 @@ pub(crate) fn open_authority_directory(path: &Utf8Path, grant_current_user: bool
     let (authorities, name) = match open_parent(path) {
         Ok(opened) => opened,
         Err(error) if is_not_found(&error) => return Ok(None),
-        Err(error) => return Err(untrusted_stored_identity(path, error)),
+        Err(error) => return Err(check_failure(path, error)),
     };
     open_authority(&authorities, name, path, grant_current_user)
 }
@@ -263,7 +279,7 @@ fn open_authority(
 ) -> anyhow::Result<Option<Dir>> {
     authorities
         .open_dir(name, &tree_rule(grant_current_user), Share::Shared)
-        .map_err(|error| untrusted_stored_identity(path, error.into()))
+        .map_err(|error| check_failure(path, error.into()))
 }
 
 /// Opens the `identity.json` at `path` in its checked `authority` directory, without changing it, once it passes the
@@ -298,7 +314,7 @@ fn open_identity_in(
     let name = path.file_name().context("identity path has no name")?;
     authority
         .open_file(name, &tree_rule(grant_current_user), options)
-        .map_err(|error| untrusted_stored_identity(path, error.into()))
+        .map_err(|error| check_failure(path, error.into()))
 }
 
 fn create_authority(path: &Utf8Path, grant_current_user: bool) -> anyhow::Result<Dir> {

crates/agent-identity/src/state.rs

@@ -3150,6 +3150,38 @@ mod tests {
         Ok(())
     }
 
+    // oracle: intent crates/agent-identity/INTENT.md §Local state
+    #[cfg(windows)]
+    #[test]
+    fn an_authority_directory_another_process_holds_is_not_untrusted() -> anyhow::Result<()> {
+        // Only an authority directory that fails the check stops all identity processing.
+        // A directory that another process, such as a backup or an antivirus scan, holds without sharing keeps the check
+        // from running, but passes the check once released.
+        use std::os::windows::fs::OpenOptionsExt as _;
+
+        use windows::Win32::Storage::FileSystem::FILE_FLAG_BACKUP_SEMANTICS;
+
+        let directory = TestDir::new()?;
+        let store = directory.store()?;
+        let identity = enroll_locally(&store, &directory.0)?;
+        let held = fs::OpenOptions::new()
+            .read(true)
+            .share_mode(0)
+            .custom_flags(FILE_FLAG_BACKUP_SEMANTICS.0)
+            .open(directory.authority(identity.authority_id))?;
+
+        let while_held = store.check_stored_identities();
+        drop(held);
+        let released = store.check_stored_identities();
+
+        let while_held = while_held
+            .err()
+            .context("the check passed while the directory was held")?;
+        assert_eq!(untrusted_path(&while_held), None, "{while_held:#}");
+        released?;
+        Ok(())
+    }
+
     // oracle: intent crates/agent-identity/INTENT.md §Local state
     #[test]
     fn start_up_stops_on_an_authority_directory_open_to_other_users_and_leaves_it_unchanged() -> anyhow::Result<()> {

devolutions-agent/src/identity/mod.rs

@@ -313,12 +313,38 @@ async fn run_pending_enrollments(
         ..
     } = shared;
     let mut retries: HashMap<Utf8PathBuf, RetryState> = HashMap::new();
+    let mut last_check_error = None;
     let mut last_list_error = None;
     loop {
         // `crates/agent-identity/INTENT.md` ("Local state"): an authority directory or `identity.json` that fails its
         // check stops all identity processing; start-up reports it and waits until it is fixed.
-        if store.check_stored_identities().is_err() {
-            return Stopped::Untrusted;
+        // Any other error only kept the check from running, so the poll retries it rather than stop every runner and its
+        // streams.
+        // UNTESTED: both outcomes are logged with the error, because stopping every identity is otherwise unexplained
+        // and a check that keeps failing otherwise holds pending enrollment silently.
+        // No test captures service logs; when changing this, keep the error in both logs.
+        if let Err(error) = store.check_stored_identities() {
+            if let Some(untrusted) = error
+                .chain()
+                .find_map(|cause| cause.downcast_ref::<UntrustedStoredIdentity>())
+            {
+                warn!(
+                    error = format!("{error:#}"),
+                    path = %untrusted.path,
+                    "Stopping every Agent Identity because a stored identity failed its check"
+                );
+                return Stopped::Untrusted;
+            }
+            if log_is_due(&mut last_check_error) {
+                warn!(
+                    error = format!("{error:#}"),
+                    "Cannot check the stored Agent Identities; retrying at the next poll"
+                );
+            }
+            if wait_or_stop(shutdown, poll_interval).await {
+                return Stopped::Shutdown;
+            }
+            continue;
         }
         match pending::list(data_dir) {
             Ok(paths) => {
@@ -630,22 +656,37 @@ mod tests {
         }))?)
     }
 
+    fn test_shared(data_dir: &TestDataDir) -> anyhow::Result<Arc<Shared>> {
+        let conf = test_conf()?;
+        let settings = conf.debug.identity.clone().unwrap_or_default();
+        Ok(Arc::new(Shared {
+            store: data_dir.store()?,
+            client: Client::new(conf.proxy.to_proxy_config(), None)?,
+            channels: channel_client(&conf, &settings)?,
+            settings,
+        }))
+    }
+
+    /// Returns a token for a loopback authority that refuses connections, which keeps an enrollment local.
+    fn loopback_token() -> anyhow::Result<Token> {
+        use base64::Engine as _;
+
+        Token::parse(&format!(
+            "dvaet1.{}.{}",
+            base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"u":"https://127.0.0.1:9"}"#),
+            "A".repeat(43)
+        ))
+    }
+
     #[tokio::test]
     async fn an_untrusted_authority_directory_holds_pending_enrollment_until_it_is_fixed() -> anyhow::Result<()> {
         // oracle: intent crates/agent-identity/INTENT.md §Local state
         // An authority directory that fails the check stops all identity processing until it is fixed, so no pending
         // file is processed meanwhile, and processing resumes without a restart.
-        use base64::Engine as _;
-
         let data_dir = TestDataDir::new()?;
         drop(data_dir.store()?);
         let untrusted = data_dir.put_untrusted_authority_directory()?;
-        // A loopback authority that refuses connections keeps the enrollment local.
-        let token = Token::parse(&format!(
-            "dvaet1.{}.{}",
-            base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(r#"{"u":"https://127.0.0.1:9"}"#),
-            "A".repeat(43)
-        ))?;
+        let token = loopback_token()?;
         pending::write(&data_dir.0, &token, true)?;
         let in_progress = pending::in_progress_path(&data_dir.0, &token.sha256_hex())?;
         let (shutdown_handle, shutdown) = devolutions_gateway_task::ShutdownHandle::new();
@@ -686,14 +727,7 @@ mod tests {
         // A stored identity that fails the check after start-up stops all identity processing too; the caller then
         // stops every runner and its streams.
         let data_dir = TestDataDir::new()?;
-        let conf = test_conf()?;
-        let settings = conf.debug.identity.clone().unwrap_or_default();
-        let shared = Arc::new(Shared {
-            store: data_dir.store()?,
-            client: Client::new(conf.proxy.to_proxy_config(), None)?,
-            channels: channel_client(&conf, &settings)?,
-            settings,
-        });
+        let shared = test_shared(&data_dir)?;
         let (_shutdown_handle, mut shutdown) = devolutions_gateway_task::ShutdownHandle::new();
         let mut runners = Runners::new(Arc::clone(&shared), shutdown.clone());
         put_untrusted(&data_dir)?;
@@ -715,6 +749,70 @@ mod tests {
         Ok(())
     }
 
+    #[tokio::test]
+    async fn a_check_that_cannot_run_neither_stops_the_pending_poll_nor_holds_enrollment_once_it_runs_again()
+    -> anyhow::Result<()> {
+        // oracle: intent crates/agent-identity/INTENT.md §Local state
+        // Only an authority directory or `identity.json` that fails the check stops all identity processing.
+        // An error that keeps the check from running, here listing the authority directories, is not such a failure,
+        // so the poll keeps running, which keeps its caller from stopping every runner and its streams, and processes
+        // pending files again once the check runs.
+        let data_dir = TestDataDir::new()?;
+        let shared = test_shared(&data_dir)?;
+        let (_shutdown_handle, mut shutdown) = devolutions_gateway_task::ShutdownHandle::new();
+        let mut runners = Runners::new(Arc::clone(&shared), shutdown.clone());
+        let token = loopback_token()?;
+        pending::write(&data_dir.0, &token, true)?;
+        let in_progress = pending::in_progress_path(&data_dir.0, &token.sha256_hex())?;
+        let authorities = data_dir.0.join("identity").join("authorities");
+        std::fs::remove_dir(&authorities)?;
+        std::fs::write(&authorities, b"")?;
+        let check = shared.store.check_stored_identities();
+        assert!(
+            check
+                .as_ref()
+                .is_err_and(|error| !error.chain().any(|cause| cause.is::<UntrustedStoredIdentity>())),
+            "listing a file as the authority directories must fail without an untrusted identity: {check:?}"
+        );
+
+        let poll = run_pending_enrollments(
+            &shared,
+            &mut runners,
+            &data_dir.0,
+            Duration::from_millis(10),
+            &mut shutdown,
+        );
+        tokio::pin!(poll);
+        let stopped_while_failing = tokio::select! {
+            _ = &mut poll => true,
+            _ = sleep(Duration::from_millis(500)) => false,
+        };
+        // A poll that returned cannot be polled again, so this is checked before the check runs again.
+        assert!(
+            !stopped_while_failing,
+            "the pending poll stopped while the check could not run"
+        );
+        std::fs::remove_file(&authorities)?;
+        let resumed = tokio::time::timeout(Duration::from_secs(10), async {
+            tokio::select! {
+                _ = &mut poll => false,
+                _ = async {
+                    while !in_progress.exists() {
+                        sleep(Duration::from_millis(10)).await;
+                    }
+                } => true,
+            }
+        })
+        .await;
+
+        assert_eq!(
+            resumed.ok(),
+            Some(true),
+            "the pending poll did not process its pending file once the check ran again"
+        );
+        Ok(())
+    }
+
     #[test]
     fn the_n_th_consecutive_failure_waits_2_to_the_n_seconds_times_jitter_then_at_most_the_maximum() {
         // oracle: intent crates/agent-identity/INTENT.md §Schedules