Fix #5 #6 #7 #8: UTF-8 corruption, flush, cmd.exe mangling, redirected-stdio

.github/workflows/CI.yml

@@ -46,6 +46,19 @@ jobs:
           }
           Pop-Location
 
+      - name: Install oh-my-posh (Windows multi-byte stress test)
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          winget install JanDeDobbeleer.OhMyPosh --accept-source-agreements --accept-package-agreements --silent
+          $poshPath = "$env:LOCALAPPDATA\Programs\oh-my-posh\bin"
+          if (Test-Path $poshPath) {
+            Add-Content -LiteralPath $env:GITHUB_PATH -Value $poshPath
+            Write-Host "oh-my-posh installed"
+          } else {
+            Write-Warning "oh-my-posh installation failed; stress test will skip"
+          }
+
       - name: Bootstrap development dependencies
         shell: pwsh
         run: ./build.ps1 -Bootstrap

src/Bsn.Host/Bsn.Host.psd1

@@ -1,6 +1,6 @@
 @{
     RootModule = 'Bsn.Host.psm1'
-    ModuleVersion = '0.1.0'
+    ModuleVersion = '0.2.0'
     GUID = '9d84a7b8-1f2b-4d60-a4bd-3d7ef1d9f337'
     Author = 'BSVN'
     CompanyName = 'BSVN'

src/Bsn.Host/Private/Get-BsnRecordingCommandLine.ps1

@@ -3,13 +3,16 @@
     Builds the Windows process command line used for a recording.
 
 .DESCRIPTION
-    Wraps the recorded command in 'cmd.exe /d /c' so arbitrary command
-    lines can be recorded. Without -Command, the default interactive shell
-    is returned; when that shell is pwsh or powershell, Bsn.Host is
-    auto-imported into the nested session so Stop-InteractiveTranscript is
-    available inside the recording. With -ShowRecordingIndicator, the
-    nested session additionally gets a recording-aware prompt
-    ([char]0x23FA + ' REC PS <path>> ').
+    When -Command is given, the command is wrapped in pwsh -EncodedCommand
+    (Base64-encoded UTF-16LE) instead of cmd.exe /d /c. This avoids all
+    cmd.exe metacharacter issues (|, &, <, >, ^, quotes) that mangle the
+    recorded command (issue #7): the exact command text is preserved
+    bit-for-bit through the encoding.
+
+    Without -Command, the default interactive shell is returned; when that
+    shell is pwsh or powershell, Bsn.Host is auto-imported into the nested
+    session so Stop-InteractiveTranscript is available inside the recording,
+    and the recording-aware prompt is installed when requested.
 #>
 function Get-BsnRecordingCommandLine {
     [CmdletBinding()]
@@ -21,7 +24,14 @@ function Get-BsnRecordingCommandLine {
     )
 
     if ($Command) {
-        return ('cmd.exe /d /c "{0}"' -f $Command)
+        # Base64-encode the command to survive the CreateProcess command line
+        # without any shell metacharacter interpretation (issue #7).
+        $shellPath = Get-BsnDefaultShell
+        if ($shellPath -match ' ') {
+            $shellPath = '"{0}"' -f $shellPath
+        }
+        $encoded = [Convert]::ToBase64String([System.Text.Encoding]::Unicode.GetBytes($Command))
+        return ('{0} -NoProfile -EncodedCommand {1}' -f $shellPath, $encoded)
     }
 
     $shellPath = Get-BsnDefaultShell

src/Bsn.Host/Private/Test-BsnRecordingEnvironment.ps1

@@ -41,8 +41,18 @@ function Test-BsnRecordingEnvironment {
         }
     }
 
-    if ([Console]::IsInputRedirected -or [Console]::IsOutputRedirected) {
-        Write-Warning -Message 'The calling process has redirected standard I/O. The recorded process output may bypass the pseudo-console; run Start-InteractiveTranscript from an interactive console session for full capture.'
+    if ([Console]::IsInputRedirected -and [Console]::IsOutputRedirected) {
+        # Issue #8: on Windows, when BOTH stdin and stdout are redirected, the
+        # child process's output bypasses the pseudo console entirely (routed to
+        # the caller's pipes), producing an empty but valid-looking cast.
+        # There is no meaningful interactive recording in this shape - error
+        # with guidance rather than silently produce a useless recording.
+        if ($Platform -eq [BsnOperatingSystem]::Windows) {
+            throw 'Interactive transcripts cannot capture output when both stdin and stdout are redirected on Windows: the recorded process bypasses the pseudo console and the cast would be empty. Run Start-InteractiveTranscript from a real console session, or from a host with at least one interactive standard handle.'
+        }
+    }
+    elseif ([Console]::IsInputRedirected -or [Console]::IsOutputRedirected) {
+        Write-Warning -Message 'The calling process has partially redirected standard I/O. The recorded process output may bypass the pseudo-console; run Start-InteractiveTranscript from an interactive console session for full capture.'
     }
 
     if ($Host.Name -ne 'ConsoleHost') {

src/Bsn.Host/src/PtyRecorder.cs

@@ -405,13 +405,23 @@ internal class CastWriter : IDisposable
         private readonly object sync = new object();
         private int castVersion;
         private double previousElapsed;
+        private Decoder outputDecoder;
+        private Decoder inputDecoder;
 
         public CastWriter(string path, int width, int height, string shellName, bool recordedInput, int castVersion)
         {
             writer = new StreamWriter(path, false, new UTF8Encoding(false));
             writer.NewLine = "\n";
+            // AutoFlush: every line (header + events) goes to disk immediately.
+            // Without this, an abnormally terminated recording loses buffered
+            // data - a 0-byte cast or truncated final event (issue #6).
+            writer.AutoFlush = true;
             this.castVersion = castVersion;
             previousElapsed = 0.0;
+            // Persistent decoders: handle multi-byte UTF-8 sequences that span
+            // chunk boundaries without corrupting them to U+FFFD (issue #5).
+            outputDecoder = Encoding.UTF8.GetDecoder();
+            inputDecoder = Encoding.UTF8.GetDecoder();
             long timestamp = (long)(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds;
             string title = "Bsn.Host interactive transcript";
             if (recordedInput)
@@ -454,7 +464,13 @@ private string FormatTime(double seconds)
 
         public void WriteEvent(double seconds, bool isInput, byte[] chunk)
         {
-            string text = Escape(Encoding.UTF8.GetString(chunk));
+            // Persistent decoder per stream type: multi-byte UTF-8 sequences that
+            // span chunk boundaries are completed by the decoder across calls,
+            // producing complete characters instead of U+FFFD replacement (issue #5).
+            Decoder decoder = isInput ? inputDecoder : outputDecoder;
+            char[] charBuffer = new char[chunk.Length];
+            int charCount = decoder.GetChars(chunk, 0, chunk.Length, charBuffer, 0);
+            string text = Escape(new string(charBuffer, 0, charCount));
             lock (sync)
             {
                 if (writer == null) { return; }

src/Bsn.Host/tests/system/InteractiveTranscript.System.Tests.ps1

@@ -437,6 +437,203 @@ Describe 'InteractiveTranscript system tests' -Tag 'SystemTest' {
         }
     }
 
+    Context 'Issue #5: large multi-byte burst survives without U+FFFD corruption' {
+        # https://github.com/BSVN/Lmt.Commons/issues/5
+        It 'records 60000 multi-byte characters with zero U+FFFD in the cast' {
+            if (-not $script:CanRunSystemTests) {
+                Set-ItResult -Skipped -Because 'pwsh is required to run recorded scenarios and was not found.'
+            }
+            if ($PSVersionTable.PSVersion.Major -lt 7) {
+                Set-ItResult -Skipped -Because 'system tests require PowerShell 7 console infrastructure.'
+            }
+            if (-not $script:IsWindowsPlatform -and ([Console]::IsInputRedirected -or [Console]::IsOutputRedirected)) {
+                Set-ItResult -Skipped -Because 'the Unix script(1) engine is unreliable under CI redirected stdio; run locally.'
+            }
+
+            # Scenario: write 60000 copies of a 3-byte UTF-8 character (U+2580).
+            # Total: 180KB — far exceeds the 4096-byte read buffer, guaranteeing
+            # many multi-byte sequences span chunk boundaries.
+            $scenarioDir = Join-Path -Path $TestDrive -ChildPath 'utf8-burst'
+            $null = New-Item -ItemType Directory -Path $scenarioDir -Force
+            $scenarioPath = Join-Path -Path $scenarioDir -ChildPath 'scenario-utf8.ps1'
+            @'
+$ErrorActionPreference = 'Stop'
+$blockChar = [string][char]0x2580
+[Console]::Out.Write($blockChar * 60000)
+Write-Output ''
+Write-Output 'UTF8-BURST-DONE'
+'@ | Set-Content -LiteralPath $scenarioPath -Encoding UTF8
+
+            $castPath = Join-Path -Path $TestDrive -ChildPath 'utf8-burst.cast'
+            $result = Invoke-BsnSystemRecording -ScenarioPath $scenarioPath -CastPath $castPath
+            $result.ExitCode | Should -Be 0
+
+            # Parse every event and check for U+FFFD
+            $null = & $script:HostModule { param($p) Get-BsnCastTimeline -Path $p } $castPath
+            $rawStream = Get-BsnCastRawStream -CastPath $castPath
+
+            # The marker must be present (the recording captured the output)
+            $rawStream.IndexOf('UTF8-BURST-DONE') | Should -BeGreaterThan (-1) -Because 'the burst completion marker must be captured'
+
+            # Count U+FFFD occurrences: must be ZERO
+            $replacementCount = ([regex]::Matches($rawStream, [regex]::Escape([string][char]0xFFFD))).Count
+            $replacementCount | Should -Be 0 -Because ("cast must contain zero U+FFFD replacement characters (found {0}); multi-byte UTF-8 sequences must not be corrupted at chunk boundaries (issue #5)" -f $replacementCount)
+
+            # The multi-byte character must actually be present (not just absent of corruption)
+            $blockCharString = [string][char]0x2580
+            $rawStream.Contains($blockCharString) | Should -BeTrue -Because 'the block character U+2580 must appear in the cast'
+        }
+    }
+
+    Context 'Issue #5: oh-my-posh prompt rendering (real-world multi-byte stress test)' {
+        # https://github.com/BSVN/Lmt.Commons/issues/5
+        # oh-my-posh renders powerline/nerd-font glyphs (3-4 byte UTF-8 sequences
+        # in the Supplementary Private Use Area). A single prompt with many segments
+        # easily exceeds the 4096-byte read buffer, guaranteeing chunk-boundary
+        # splits mid-glyph. This is the real-world case the issue was filed for.
+        It 'records an oh-my-posh prompt burst with zero U+FFFD in the cast' {
+            if (-not $script:CanRunSystemTests) {
+                Set-ItResult -Skipped -Because 'pwsh is required to run recorded scenarios and was not found.'
+            }
+            if ($PSVersionTable.PSVersion.Major -lt 7) {
+                Set-ItResult -Skipped -Because 'system tests require PowerShell 7 console infrastructure.'
+            }
+            if (-not $script:IsWindowsPlatform) {
+                Set-ItResult -Skipped -Because 'oh-my-posh test is Windows-only (Linux uses vim TUI test instead).'
+            }
+
+            # oh-my-posh must be on PATH (CI installs it; locally: winget install JanDeDobbeleer.OhMyPosh)
+            $ohMyPosh = Get-Command -Name 'oh-my-posh' -ErrorAction 'SilentlyContinue'
+            if (-not $ohMyPosh) {
+                Set-ItResult -Skipped -Because 'oh-my-posh is not on PATH. Install it (winget install JanDeDobbeleer.OhMyPosh) to run this stress test.'
+            }
+
+            # Scenario: render multiple oh-my-posh prompts (each produces multi-byte glyphs)
+            $scenarioDir = Join-Path -Path $TestDrive -ChildPath 'ohmyposh'
+            $null = New-Item -ItemType Directory -Path $scenarioDir -Force
+            $scenarioPath = Join-Path -Path $scenarioDir -ChildPath 'scenario-omp.ps1'
+
+            $themePath = if ($env:POSH_THEMES_PATH) {
+                Join-Path -Path $env:POSH_THEMES_PATH -ChildPath 'atomic.omp.json'
+            } else {
+                # Try a known install location
+                Join-Path -Path "$env:LOCALAPPDATA\Programs\oh-my-posh\themes" -ChildPath 'atomic.omp.json'
+            }
+
+            $themeBlock = if (Test-Path -LiteralPath $themePath) {
+                " --config `"$themePath`""
+            } else {
+                ''  # use the default theme
+            }
+
+            @"
+`$ErrorActionPreference = 'Stop'
+# Render multiple prompts via oh-my-posh print primary (produces the glyph-laden output)
+# Each prompt is typically 200-600 bytes of multi-byte VT sequences
+1..20 | ForEach-Object { oh-my-posh print primary --shell pwsh$themeBlock }
+Write-Output 'OMP-STRESS-DONE'
+"@ | Set-Content -LiteralPath $scenarioPath -Encoding UTF8
+
+            $castPath = Join-Path -Path $TestDrive -ChildPath 'omp-stress.cast'
+            $result = Invoke-BsnSystemRecording -ScenarioPath $scenarioPath -CastPath $castPath
+            $result.ExitCode | Should -Be 0
+
+            $rawStream = Get-BsnCastRawStream -CastPath $castPath
+
+            # The marker must be present
+            $rawStream.IndexOf('OMP-STRESS-DONE') | Should -BeGreaterThan (-1) -Because 'the oh-my-posh stress test completion marker must be captured'
+
+            # Count U+FFFD occurrences: must be ZERO
+            $replacementCount = ([regex]::Matches($rawStream, [regex]::Escape([string][char]0xFFFD))).Count
+            $replacementCount | Should -Be 0 -Because ("cast must contain zero U+FFFD replacement characters after oh-my-posh prompt rendering (found {0}); nerd-font glyphs must survive chunk boundaries without corruption (issue #5)" -f $replacementCount)
+        }
+    }
+
+    Context 'Issue #5: vim TUI session (hardest Linux TUI case)' {
+        # https://github.com/BSVN/Lmt.Commons/issues/5
+        # vim is one of the hardest TUI cases: full-screen rendering, alternate
+        # screen buffer, cursor positioning, syntax highlighting, status line,
+        # and multi-byte content. This test validates that the recording captures
+        # a real vim session correctly.
+        It 'records a vim session and captures the TUI content' {
+            if (-not $script:CanRunSystemTests) {
+                Set-ItResult -Skipped -Because 'pwsh is required to run recorded scenarios and was not found.'
+            }
+            if ($PSVersionTable.PSVersion.Major -lt 7) {
+                Set-ItResult -Skipped -Because 'system tests require PowerShell 7 console infrastructure.'
+            }
+            if ($script:IsWindowsPlatform) {
+                Set-ItResult -Skipped -Because 'vim TUI test is Linux-only (Windows uses oh-my-posh test instead).'
+            }
+            if ([Console]::IsInputRedirected -or [Console]::IsOutputRedirected) {
+                Set-ItResult -Skipped -Because 'requires an interactive terminal (CI has redirected stdio); run locally.'
+            }
+
+            $vimCommand = Get-Command -Name 'vim' -ErrorAction 'SilentlyContinue'
+            if (-not $vimCommand) {
+                # Try vi as a fallback
+                $vimCommand = Get-Command -Name 'vi' -ErrorAction 'SilentlyContinue'
+            }
+            if (-not $vimCommand) {
+                Set-ItResult -Skipped -Because 'vim/vi is not installed. Install it to run this TUI stress test.'
+            }
+
+            # Create a file with distinctive content for vim to render
+            $contentFile = Join-Path -Path $TestDrive -ChildPath 'vim-content.txt'
+            @'
+VIM-TUI-LINE-1: The quick brown fox
+VIM-TUI-LINE-2: jumps over the lazy dog
+VIM-TUI-LINE-3: pack my box with five dozen liquor jugs
+VIM-TUI-LINE-4: ================================
+VIM-TUI-LINE-5: [SUCCESS] All systems operational
+VIM-TUI-LINE-6: © 2026 UTF-8 test — ¥€£¢
+VIM-TUI-LINE-7: ┌──────────────────────────┐
+VIM-TUI-LINE-8: │ Box-drawing characters  │
+VIM-TUI-LINE-9: └──────────────────────────┘
+'@ | Set-Content -LiteralPath $contentFile -Encoding UTF8
+
+            # Scenario: open vim with the content file, wait briefly, quit
+            # vim enters alt-screen, renders the file with line numbers,
+            # status line, and the content - then exits alt-screen on :q
+            $scenarioDir = Join-Path -Path $TestDrive -ChildPath 'vim-tui'
+            $null = New-Item -ItemType Directory -Path $scenarioDir -Force
+            $scenarioPath = Join-Path -Path $scenarioDir -ChildPath 'scenario-vim.sh'
+
+            $vimScript = @"
+#!/bin/sh
+# Open vim with the content file, auto-quit after loading
+vim -R -c 'sleep 1' -c 'q' '$contentFile' 2>/dev/null || vi -R -c 'q' '$contentFile' 2>/dev/null
+echo 'VIM-TUI-DONE'
+"@
+            $vimScript | Set-Content -LiteralPath $scenarioPath -Encoding UTF8
+            $null = & chmod +x $scenarioPath
+
+            $castPath = Join-Path -Path $TestDrive -ChildPath 'vim-tui.cast'
+            $result = Invoke-BsnSystemRecording -ScenarioPath $scenarioPath -CastPath $castPath
+            $result.ExitCode | Should -Be 0
+
+            $rawStream = Get-BsnCastRawStream -CastPath $castPath
+            $plainText = Convert-BsnCastPlainText -RawStream $rawStream
+
+            # TUI capture: alternate screen enter/exit must be present
+            $esc = [char]27
+            $rawStream.Contains($esc + '[?1049h') | Should -BeTrue -Because 'vim must enter the alternate screen buffer'
+            $rawStream.Contains($esc + '[?1049l') | Should -BeTrue -Because 'vim must exit the alternate screen buffer'
+
+            # Content capture: vim rendered the file content on screen
+            $plainText.IndexOf('VIM-TUI-LINE-1') | Should -BeGreaterThan (-1) -Because 'vim rendered the first line'
+            $plainText.IndexOf('VIM-TUI-LINE-5') | Should -BeGreaterThan (-1) -Because 'vim rendered the success line'
+
+            # Multi-byte content: the UTF-8 and box-drawing lines must survive
+            $plainText.IndexOf('UTF-8 test') | Should -BeGreaterThan (-1) -Because 'vim rendered multi-byte UTF-8 content'
+            $plainText.IndexOf('Box-drawing') | Should -BeGreaterThan (-1) -Because 'vim rendered box-drawing characters'
+
+            # No corruption
+            $replacementCount = ([regex]::Matches($rawStream, [regex]::Escape([string][char]0xFFFD))).Count
+            $replacementCount | Should -Be 0 -Because ("vim TUI recording must contain zero U+FFFD (found {0}); multi-byte content must survive chunk boundaries (issue #5)" -f $replacementCount)
+        }
+    }
+
     Context 'session with cmdlets and a TUI application' {
         It 'records the session and the cast matches the real view' {
             if (-not $script:CanRunSystemTests) {