Expect the substitute, not the lookalike, when the codepage is short

.github/actions/smoke-test/action.yml

@@ -210,7 +210,10 @@ runs:
         if ("$so" -notmatch 'MBCS->UTF-8 ok on 2 checks') {
           throw "selftest did not exercise the MBCS->UTF-8 conversion"
         }
-        if ("$so" -notmatch 'catalog decoding ok on 3 checks') {
+        # The suffix proves the best-fit case can fail: cp1252 offers a lookalike for U+0100.
+        # Where a codepage offers none, that case cannot catch anything. The last two checks
+        # run only there, so the count and the suffix move together.
+        if ("$so" -notmatch 'catalog decoding ok on 6 checks \(best-fit acp\)') {
           throw 'selftest did not report catalog decoding: a legacy catalog may be silently mangled'
         }
         # Swapping decode and unescape is invisible on this cp1252 runner, so this drives CP932.

.github/workflows/windows-build.yml

@@ -724,7 +724,12 @@ jobs:
           # --selftest throws one exception on purpose.
           $p = Start-Process (Join-Path $bin 'WinHTTrack.exe') -ArgumentList '--selftest' -Wait -PassThru `
                  -RedirectStandardOutput "$PWD\crash-st.txt" -RedirectStandardError "$PWD\crash-st.err"
-          if ($p.ExitCode -ne 0) { throw "--selftest exited $($p.ExitCode) from the staged tree" }
+          if ($p.ExitCode -ne 0) {
+            # The selftest names the failing check on stderr, and the exit code alone does not.
+            Write-Host "--- selftest stdout ---`n$(Get-Content "$PWD\crash-st.txt" -Raw -ErrorAction SilentlyContinue)"
+            Write-Host "--- selftest stderr ---`n$(Get-Content "$PWD\crash-st.err" -Raw -ErrorAction SilentlyContinue)"
+            throw "--selftest exited $($p.ExitCode) from the staged tree"
+          }
           if (-not (Test-Path $report)) { throw 'no crash report written: the first-chance hook never ran' }
           $trace = Get-Content $report -Raw
           Write-Host "--- crash report ---`n$trace"

WinHTTrack/Shell.cpp

@@ -1715,6 +1715,23 @@ void CopyTextUTF8ToCP(LPSTR dest, int destSize, LPCSTR lpString) {
   freet(cp);
 }
 
+// Contract in Shell.h.
+BOOL CopyTextWideToCPExact(LPSTR dest, int destSize, LPCWSTR wide) {
+  // A best-fit substitute leaves lpUsedDefaultChar clear, so without WC_NO_BEST_FIT_CHARS an
+  // approximation comes back and reads as held. CP_UTF8 rejects both that flag and the pointer.
+  const BOOL utf8 = GetACP() == CP_UTF8;
+  BOOL lost = FALSE;
+  int n;
+
+  if (destSize <= 0)
+    return FALSE;
+  n = WideCharToMultiByte(CP_ACP, utf8 ? 0 : WC_NO_BEST_FIT_CHARS, wide, -1, dest, destSize,
+                          NULL, utf8 ? NULL : &lost);
+  if (n <= 0)
+    dest[0] = '\0';
+  return n > 0 && !lost;
+}
+
 bool ShellOpen(LPCSTR file, int nShowCmd) {
   return (INT_PTR) ShellExecute(NULL, "open", file, NULL, NULL, nShowCmd) > 32;
 }

WinHTTrack/Shell.h

@@ -159,6 +159,11 @@ char *strdupt_utf8(const char *const s);
 // TTN_NEEDTEXTA tooltip text. Falls back to the raw bytes if conversion fails.
 void CopyTextUTF8ToCP(LPSTR dest, int destSize, LPCSTR lpString);
 
+// The ANSI-codepage form of a wide string, always NUL-terminated, with best-fit substitutes
+// blocked as the engine's converter blocks them. --selftest needs its own oracle for that.
+// FALSE: the codepage substituted; dest is empty instead if it could not convert at all.
+BOOL CopyTextWideToCPExact(LPSTR dest, int destSize, LPCWSTR wide);
+
 // Drop a trailing '/' or '\\', reporting whether there was one. Empty-string safe.
 inline bool StripTrailingSlash(char* s) {
   const size_t len = (s != NULL) ? strlen(s) : 0;

WinHTTrack/WinHTTrack.cpp

@@ -345,13 +345,9 @@ BOOL CWinHTTrackApp::InitInstance()
     // Only reachable by typing into a dialog, so test the MBCS->UTF-8 conversion here instead.
     {
       static const WCHAR wide[] = { 'c', 'a', 'f', 0x00E9, 0 };   /* cafe-acute */
-      BOOL lost = FALSE;
-      // WideCharToMultiByte rejects a non-NULL lpUsedDefaultChar when the code page is CP_UTF8.
-      BOOL *const plost = (GetACP() == CP_UTF8) ? NULL : &lost;
       char ansi[16];
-      const int n = WideCharToMultiByte(CP_ACP, 0, wide, -1, ansi, sizeof(ansi),
-                                        NULL, plost);
-      if (n > 0 && !lost) {   /* skip where the ANSI codepage cannot hold it at all */
+      /* skip where the codepage cannot hold it exactly */
+      if (CopyTextWideToCPExact(ansi, sizeof(ansi), wide)) {
         int nchecks = 0;
         char *got = strdupt_utf8(ansi);   /* freet() nulls it, so not const */
         if (got == NULL || strcmp(got, "caf\xc3\xa9") != 0) {

WinHTTrack/newlang.cpp

@@ -363,41 +363,81 @@ void LANG_SELFTEST_ESCAPE_ORDER(void) {
 /* --selftest: the catalogs are the only non-ASCII data the GUI loads, and the
    English-only smoke walk cannot see a decoding regression. */
 void LANG_SELFTEST_DECODE(void) {
-  static const WCHAR wide[] = { 'c', 'a', 'f', 0x00E9, 0 };   /* cafe-acute */
-  BOOL lost = FALSE;
-  // WideCharToMultiByte rejects a non-NULL lpUsedDefaultChar when the code page is CP_UTF8.
-  BOOL *const plost = (GetACP() == CP_UTF8) ? NULL : &lost;
-  char want[16];
-  const int n = WideCharToMultiByte(CP_ACP, 0, wide, -1, want, sizeof(want), NULL, plost);
-
-  if (n > 0 && !lost) {   /* skip where the ANSI codepage cannot hold it at all */
-    static const struct { const char* in; const char* want; const char* what; } cases[] = {
-      /* A converted catalog reaches the ANSI codepage. */
-      { "caf\xc3\xa9", NULL, "utf-8" },
-      /* A catalog still in its legacy charset must survive byte for byte: drop the
-         UTF-8 gate and the engine's converter substitutes U+FFFD, giving "caf?". */
-      { "caf\xe9", "caf\xe9", "legacy" },
-      { "Options", "Options", "ascii" },
-      { NULL, NULL, NULL }
-    };
-    int nchecks = 0;
-    for(int k=0 ; cases[k].in != NULL ; k++) {
-      const char* const expect = (cases[k].want != NULL) ? cases[k].want : want;
-      const int len = (int) strlen(cases[k].in);
-      char got[64];
-      if (IsValidUTF8(cases[k].in, len))
-        CopyTextUTF8ToCP(got, sizeof(got), cases[k].in);
-      else
-        lstrcpynA(got, cases[k].in, sizeof(got));
-      if (strcmp(got, expect) != 0) {
-        fprintf(stderr, "FATAL: %s catalog text gave '%s', expected '%s'\n",
-                cases[k].what, got, expect);
+  static const WCHAR cafe[] = { 'c', 'a', 'f', 0x00E9, 0 };   /* cafe-acute */
+  static const WCHAR macron[] = { 0x0100, 0 };                /* A-macron, 'A' by best-fit on cp1252 */
+  static const struct { const char* in; const WCHAR* wide; const char* want; const char* what; } cases[] = {
+    /* A converted catalog reaches the ANSI codepage. */
+    { "caf\xc3\xa9", cafe, NULL, "utf-8" },
+    /* The engine blocks best-fit, so a character the codepage lacks must arrive as its
+       default character, not as a lookalike. */
+    { "\xc4\x80", macron, NULL, "best-fit" },
+    /* A catalog still in its legacy charset must survive byte for byte: drop the
+       UTF-8 gate and the engine's converter substitutes U+FFFD, giving "caf?". */
+    { "caf\xe9", NULL, "caf\xe9", "legacy" },
+    { "Options", NULL, "Options", "ascii" },
+    { NULL, NULL, NULL, NULL }
+  };
+  int nchecks = 0;
+
+  for(int k=0 ; cases[k].in != NULL ; k++) {
+    const int len = (int) strlen(cases[k].in);
+    char expect[64];
+    char got[64];
+    /* Expecting what this codepage holds, so none has to be skipped. */
+    if (cases[k].wide != NULL)
+      CopyTextWideToCPExact(expect, sizeof(expect), cases[k].wide);
+    else
+      lstrcpynA(expect, cases[k].want, sizeof(expect));
+    if (IsValidUTF8(cases[k].in, len))
+      CopyTextUTF8ToCP(got, sizeof(got), cases[k].in);
+    else
+      lstrcpynA(got, cases[k].in, sizeof(got));
+    if (strcmp(got, expect) != 0) {
+      fprintf(stderr, "FATAL: %s catalog text gave '%s', expected '%s'\n",
+              cases[k].what, got, expect);
+      fflush(stderr);
+      ExitProcess(3);
+    } else
+      nchecks++;
+  }
+
+  {
+    /* The case above only discriminates where the codepage has a lookalike to offer. Where it
+       does, pin the substitute to the codepage's own default character. A second
+       WideCharToMultiByte call would only prove the two agree. */
+    CPINFO cpinfo;
+    char def[MAX_DEFAULTCHAR + 1];
+    char expect[64], lookalike[64];
+    const int n = WideCharToMultiByte(CP_ACP, 0, macron, -1, lookalike, sizeof(lookalike),
+                                      NULL, NULL);
+    const BOOL held = CopyTextWideToCPExact(expect, sizeof(expect), macron);
+    const BOOL bestfit = n > 0 && strcmp(lookalike, expect) != 0;
+
+    if (bestfit) {
+      if (!GetCPInfo(CP_ACP, &cpinfo)) {
+        fprintf(stderr, "FATAL: no CPINFO for the ANSI codepage\n");
+        fflush(stderr);
+        ExitProcess(3);
+      }
+      memcpy(def, cpinfo.DefaultChar, MAX_DEFAULTCHAR);
+      def[MAX_DEFAULTCHAR] = '\0';
+      /* Counted apart: a helper that forgot its lpUsedDefaultChar check still substitutes,
+         so sharing one increment would leave that half unpinned. */
+      if (held) {
+        fprintf(stderr, "FATAL: U+0100 reported as held by a codepage that substitutes it\n");
+        fflush(stderr);
+        ExitProcess(3);
+      } else
+        nchecks++;
+      if (strcmp(expect, def) != 0) {
+        fprintf(stderr, "FATAL: U+0100 became '%s', expected the default character '%s'\n",
+                expect, def);
         fflush(stderr);
         ExitProcess(3);
       } else
         nchecks++;
     }
-    printf("catalog decoding ok on %d checks\n", nchecks);
+    printf("catalog decoding ok on %d checks%s\n", nchecks, bestfit ? " (best-fit acp)" : "");
   }
 }