[pull] main from danny-avila:main

.github/scripts/verify-playwright-ffmpeg.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+# Verifies that Playwright's ffmpeg download produced a usable binary.
+#
+# `playwright install` is not trustworthy on its own here: under the Node 24.16.0
+# yauzl/extract-zip regression (Playwright < 1.60.0) it would hang mid-extraction
+# and leave a truncated `ffmpeg-linux` behind with no INSTALLATION_COMPLETE marker,
+# so the exit code said nothing about whether ffmpeg actually worked.
+#
+# CI caches the download, so this runs before the cache is saved: checking both the
+# marker and that the binary actually executes is what keeps a partial extraction
+# from being promoted into a cache that every later job would restore. The install
+# directory is read back from Playwright so this stays correct across version bumps
+# and never lets a stale revision vouch for the one actually required.
+
+set -uo pipefail
+
+install_dir=$(npx playwright install --dry-run ffmpeg 2>/dev/null |
+  sed -n 's/^[[:space:]]*Install location:[[:space:]]*//p' | head -1)
+
+if [ -z "${install_dir}" ]; then
+  echo "::warning::Could not determine Playwright's ffmpeg install location; skipping cache save."
+  exit 1
+fi
+
+if [ ! -f "${install_dir}/INSTALLATION_COMPLETE" ]; then
+  echo "::warning::${install_dir} has no INSTALLATION_COMPLETE marker; the download did not finish."
+  exit 1
+fi
+
+binary="${install_dir}/ffmpeg-linux"
+
+if [ ! -x "${binary}" ]; then
+  echo "::warning::${binary} is missing or not executable."
+  exit 1
+fi
+
+if ! "${binary}" -version >/dev/null 2>&1; then
+  echo "::warning::${binary} is present but does not execute; treating it as a partial extraction."
+  exit 1
+fi
+
+echo "Verified Playwright ffmpeg at ${binary}"

.github/workflows/agents-integration-tests.yml

@@ -1,7 +1,7 @@
 name: Agents Integration Tests
 
 # Runs the packages/api `src/agents/**` integration specs (e.g. the durable HITL
-# checkpointer against a real in-process MongoDB via mongodb-memory-server). These
+# checkpointer and cross-replica subagent delivery against real MongoDB and Redis). These
 # are `*.integration.spec.ts`, which `test:ci` deliberately excludes — without this
 # job they run nowhere and their regressions guard nothing.
 on:
@@ -33,10 +33,21 @@ concurrency:
 
 jobs:
   agents_integration_tests:
-    name: Integration Tests that use in-process MongoDB
+    name: Integration Tests (MongoDB and Redis)
     timeout-minutes: 20
     runs-on: ubuntu-latest
 
+    services:
+      redis:
+        image: redis:7-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
@@ -95,8 +106,9 @@ jobs:
         if: steps.cache-api.outputs.cache-hit != 'true'
         run: npm run build:api
 
-      - name: Run agents integration tests (in-process MongoDB)
+      - name: Run agents integration tests
         working-directory: packages/api
         env:
           NODE_ENV: test
+          REDIS_URI: redis://127.0.0.1:6379
         run: npm run test:agents-integration

.github/workflows/codegraph-e2e-votes.yml

@@ -172,11 +172,37 @@ jobs:
         run: google-chrome --version
 
       # ffmpeg for retry video — see the note in playwright-mock.yml.
-      - name: Install Playwright ffmpeg (best effort)
+      - name: Resolve Playwright version
+        id: playwright-version
+        if: steps.tiers.outputs.count != '0'
+        run: |
+          version=$(node -p "require('./package-lock.json').packages['node_modules/playwright-core'].version")
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+      - name: Restore Playwright ffmpeg cache
+        id: cache-ffmpeg
         if: steps.tiers.outputs.count != '0'
+        uses: actions/cache/restore@v5
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
+
+      - name: Install Playwright ffmpeg (best effort)
+        id: install-ffmpeg
+        if: steps.tiers.outputs.count != '0' && steps.cache-ffmpeg.outputs.cache-hit != 'true'
         timeout-minutes: 3
         continue-on-error: true
-        run: timeout -k 10 90 npx playwright install ffmpeg
+        run: |
+          timeout -k 10 60 npx playwright install ffmpeg
+          .github/scripts/verify-playwright-ffmpeg.sh
+
+      - name: Save Playwright ffmpeg cache
+        if: steps.tiers.outputs.count != '0' && steps.install-ffmpeg.outcome == 'success'
+        continue-on-error: true
+        uses: actions/cache/save@v5
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
 
       # Optional fonts only — see the note in playwright-mock.yml.
       - name: Install optional Playwright font dependencies (best effort)

.github/workflows/playwright-mock.yml

@@ -146,13 +146,48 @@ jobs:
 
       # `video: 'on-first-retry'` needs ffmpeg; without it the first retry dies in
       # browserContext.newPage before the test body runs, so a flaky test loses the
-      # retry that would have recovered it. The CLI can hang after the download
-      # finishes on these runners, so bound it and keep it non-fatal — worst case is
-      # today's behaviour of retrying without video.
+      # retry that would have recovered it.
+      #
+      # This step used to burn its full 90s bound on every job. Playwright's bundled
+      # extractor hangs on Node 24.16.0 (a yauzl/extract-zip regression fixed in
+      # Playwright 1.60.0): the 2.3MB download finished in under a second, then
+      # extraction stalled and the timeout reaped it, leaving a truncated binary and
+      # no INSTALLATION_COMPLETE marker — so ffmpeg was never actually installed and
+      # retries never got video. With Playwright bumped past the fix the install
+      # takes about a second, and a restored cache skips it outright.
+      #
+      # The cache is only saved once the binary is verified to run, so a partial
+      # extraction can never be promoted into a cache every later job restores.
+      # Kept non-fatal: retry video is a debugging aid, not something CI asserts on.
+      - name: Resolve Playwright version
+        id: playwright-version
+        run: |
+          version=$(node -p "require('./package-lock.json').packages['node_modules/playwright-core'].version")
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+      - name: Restore Playwright ffmpeg cache
+        id: cache-ffmpeg
+        uses: actions/cache/restore@v5
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
+
       - name: Install Playwright ffmpeg (best effort)
+        id: install-ffmpeg
+        if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
         timeout-minutes: 3
         continue-on-error: true
-        run: timeout -k 10 90 npx playwright install ffmpeg
+        run: |
+          timeout -k 10 60 npx playwright install ffmpeg
+          .github/scripts/verify-playwright-ffmpeg.sh
+
+      - name: Save Playwright ffmpeg cache
+        if: steps.install-ffmpeg.outcome == 'success'
+        continue-on-error: true
+        uses: actions/cache/save@v5
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
 
       # The runner's Chrome is an apt package, so its real library dependencies are
       # already satisfied; all `install-deps` adds here are optional CJK/Thai/Cyrillic
@@ -246,10 +281,35 @@ jobs:
         run: google-chrome --version
 
       # ffmpeg for retry video — see the note in the e2e_shards job.
+      - name: Resolve Playwright version
+        id: playwright-version
+        run: |
+          version=$(node -p "require('./package-lock.json').packages['node_modules/playwright-core'].version")
+          echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+      - name: Restore Playwright ffmpeg cache
+        id: cache-ffmpeg
+        uses: actions/cache/restore@v5
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
+
       - name: Install Playwright ffmpeg (best effort)
+        id: install-ffmpeg
+        if: steps.cache-ffmpeg.outputs.cache-hit != 'true'
         timeout-minutes: 3
         continue-on-error: true
-        run: timeout -k 10 90 npx playwright install ffmpeg
+        run: |
+          timeout -k 10 60 npx playwright install ffmpeg
+          .github/scripts/verify-playwright-ffmpeg.sh
+
+      - name: Save Playwright ffmpeg cache
+        if: steps.install-ffmpeg.outcome == 'success'
+        continue-on-error: true
+        uses: actions/cache/save@v5
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-ffmpeg-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}
 
       # This job deliberately skips the optional font install: its bounded
       # Playwright apt process can outlive the wrapper on a slow mirror and

AGENTS.md

@@ -10,4 +10,8 @@ instead of copying classes into a feature. Keep genuine layout and behavior loca
 why any new custom CSS cannot be expressed by the shared system. See the detailed policy in
 `CLAUDE.md` under “Theming and styling.”
 
-When adding or changing code that mutates user documents, invalidate the auth user document cache for affected users. This includes single-user updates and bulk role/user mutations; otherwise OpenID JWT request burst caching can serve a stale `req.user` until its TTL expires.
+## Backend auth cache
+
+When adding or changing code that mutates user documents, invalidate the auth user document cache
+for affected users, including bulk role and user mutations. See the detailed policy in `CLAUDE.md`
+under “Auth cache invalidation”.

CLAUDE.md

@@ -13,7 +13,8 @@ LibreChat is a monorepo with the following key workspaces:
 | `/client` | TypeScript/React | Frontend | `packages/data-provider`, `packages/client` | Frontend SPA |
 | `/packages/client` | TypeScript | Frontend | `packages/data-provider` | Shared frontend utilities |
 
-The source code for `@librechat/agents` (major backend dependency, same team) is at `/home/danny/agentus`.
+The source code for `@librechat/agents` (major backend dependency, same team) lives at
+<https://github.com/danny-avila/agents>.
 
 ---
 
@@ -170,6 +171,16 @@ Multi-line imports count total character length across all lines. Consolidate va
 
 ---
 
+## Backend Rules (`api/**`, `packages/api/**`)
+
+### Auth cache invalidation
+
+When adding or changing code that mutates user documents, invalidate the auth user document cache
+for the affected users. This covers single-user updates as well as bulk role and user mutations.
+Without it, OpenID JWT request burst caching can serve a stale `req.user` until its TTL expires.
+
+---
+
 ## Development Commands
 
 | Command | Purpose |

api/db/indexSync.js

@@ -230,34 +230,54 @@ async function performSync(flowManager, flowId, flowType) {
       await batchResetMeiliFlags(Conversation.collection);
     }
 
-    // Check if we need to sync messages
-    logger.info('[indexSync] Requesting message sync progress...');
-    const messageProgress = await Message.getSyncProgress();
-    if (!messageProgress.isComplete || settingsUpdated) {
-      logger.info(
-        `[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
-      );
+    let messageSyncError;
+    try {
+      // Check if we need to sync messages
+      logger.info('[indexSync] Requesting message sync progress...');
+      const messageProgress = await Message.getSyncProgress();
+      if (!messageProgress.isComplete || settingsUpdated) {
+        logger.info(
+          `[indexSync] Messages need syncing: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments} indexed`,
+        );
 
-      const messageCount = messageProgress.totalDocuments;
-      const messagesIndexed = messageProgress.totalProcessed;
-      const unindexedMessages = messageCount - messagesIndexed;
-      const noneIndexed = messagesIndexed === 0 && unindexedMessages > 0;
+        const messageCount = messageProgress.totalDocuments;
+        const messagesIndexed = messageProgress.totalProcessed;
+        const unindexedMessages = messageCount - messagesIndexed;
+        const messagesPendingCleanup = messageProgress.pendingCleanup ?? 0;
+        const noneIndexed = messagesIndexed === 0 && unindexedMessages > 0;
 
-      if (settingsUpdated || noneIndexed || unindexedMessages > syncThreshold) {
-        if (noneIndexed && !settingsUpdated) {
-          logger.info('[indexSync] No messages marked as indexed, forcing full sync');
+        if (settingsUpdated || noneIndexed || unindexedMessages > syncThreshold) {
+          if (noneIndexed && !settingsUpdated) {
+            logger.info('[indexSync] No messages marked as indexed, forcing full sync');
+          }
+          logger.info(
+            messagesPendingCleanup > 0
+              ? `[indexSync] Starting message sync (${unindexedMessages} unindexed, ${messagesPendingCleanup} pending cleanup)`
+              : `[indexSync] Starting message sync (${unindexedMessages} unindexed)`,
+          );
+          await Message.syncWithMeili();
+          messagesSync = true;
+        } else if (messagesPendingCleanup > 0) {
+          logger.info(
+            `[indexSync] Cleaning ${messagesPendingCleanup} excluded messages from search`,
+          );
+          await Message.cleanupExcludedMeiliIndex();
+          messagesSync = true;
+        } else if (unindexedMessages > 0) {
+          logger.info(
+            `[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
+          );
         }
-        logger.info(`[indexSync] Starting message sync (${unindexedMessages} unindexed)`);
-        await Message.syncWithMeili();
-        messagesSync = true;
-      } else if (unindexedMessages > 0) {
+      } else {
         logger.info(
-          `[indexSync] ${unindexedMessages} messages unindexed (below threshold: ${syncThreshold}, skipping)`,
+          `[indexSync] Messages are fully synced: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments}`,
         );
       }
-    } else {
-      logger.info(
-        `[indexSync] Messages are fully synced: ${messageProgress.totalProcessed}/${messageProgress.totalDocuments}`,
+    } catch (error) {
+      messageSyncError = error;
+      logger.error(
+        '[indexSync] Message reconciliation failed; continuing with conversations:',
+        error,
       );
     }
 
@@ -271,15 +291,26 @@ async function performSync(flowManager, flowId, flowType) {
       const convoCount = convoProgress.totalDocuments;
       const convosIndexed = convoProgress.totalProcessed;
       const unindexedConvos = convoCount - convosIndexed;
+      const convosPendingCleanup = convoProgress.pendingCleanup ?? 0;
       const noneConvosIndexed = convosIndexed === 0 && unindexedConvos > 0;
 
       if (settingsUpdated || noneConvosIndexed || unindexedConvos > syncThreshold) {
         if (noneConvosIndexed && !settingsUpdated) {
           logger.info('[indexSync] No conversations marked as indexed, forcing full sync');
         }
-        logger.info(`[indexSync] Starting convos sync (${unindexedConvos} unindexed)`);
+        logger.info(
+          convosPendingCleanup > 0
+            ? `[indexSync] Starting convos sync (${unindexedConvos} unindexed, ${convosPendingCleanup} pending cleanup)`
+            : `[indexSync] Starting convos sync (${unindexedConvos} unindexed)`,
+        );
         await Conversation.syncWithMeili();
         convosSync = true;
+      } else if (convosPendingCleanup > 0) {
+        logger.info(
+          `[indexSync] Cleaning ${convosPendingCleanup} excluded conversations from search`,
+        );
+        await Conversation.cleanupExcludedMeiliIndex();
+        convosSync = true;
       } else if (unindexedConvos > 0) {
         logger.info(
           `[indexSync] ${unindexedConvos} convos unindexed (below threshold: ${syncThreshold}, skipping)`,
@@ -291,6 +322,10 @@ async function performSync(flowManager, flowId, flowType) {
       );
     }
 
+    if (messageSyncError) {
+      throw messageSyncError;
+    }
+
     return { messagesSync, convosSync };
   } finally {
     if (indexingDisabled === true) {