keep callees of plain callers plain

src/app/pipeline.c

@@ -1256,6 +1256,7 @@ static int prepare_llvm_function_abis(const LoadedCodeSection* sections,
                                       u32 section_count,
                                       DolLLVMFunctionRange* ranges,
                                       u32 range_count, DolLLVMRuntime runtime,
+                                      DolLLVMNativeABIPolicy policy,
                                       const u32* entry_points,
                                       u32 entry_point_count,
                                       FILE* fallback_report,
@@ -1483,6 +1484,8 @@ static int prepare_llvm_function_abis(const LoadedCodeSection* sections,
         !dolllvm_propagate_function_abis(ranges, range_count, edges,
                                          edge_count))
         goto fail;
+    dolllvm_finish_function_abis(ranges, range_count, policy, runtime, edges,
+                                 edge_count);
     if (runtime == DOLLLVM_RUNTIME_MODERNGEKKO)
         for (u32 range_index = 0; range_index < range_count; range_index++)
             memset(ranges[range_index].escape_state, 0,
@@ -2099,16 +2102,15 @@ static int emit_code_sections_llvm(const LoadedCodeSection* sections,
         cache_dir[0] = '\0';
     if (!prepare_llvm_function_abis(
             sections, section_count, ranges, range_count,
-            (DolLLVMRuntime)options->llvm_runtime, entry_points,
+            (DolLLVMRuntime)options->llvm_runtime,
+            (DolLLVMNativeABIPolicy)options->llvm_native_abi, entry_points,
             entry_point_count, fallback_report, &native_entry_candidates,
             &native_entry_candidate_count, &native_entry_candidate_capacity))
         goto fail;
 #if defined(__GLIBC__)
     if (!llvm_fast_iteration())
         malloc_trim(0);
 #endif
-    dolllvm_apply_native_abi_policy(
-        ranges, range_count, (DolLLVMNativeABIPolicy)options->llvm_native_abi);
     for (u32 i = 0; i < patch_count; i++) {
         const DolLLVMFunctionRange* range =
             llvm_range_for(ranges, range_count, patches[i].start);

src/backend/llvm/abi_policy.cpp

@@ -43,6 +43,47 @@ u32 packedReturnLanes(const u64 *mask) {
   return lanes + static_cast<u32>(bits != 0);
 }
 
+// Plain callers can't call native ranges directly, so keep their callees plain.
+void demotePlainCallerTargets(DolLLVMFunctionRange *ranges, u32 rangeCount,
+                              const DolLLVMCallEdge *edges, u32 edgeCount) {
+  if (!ranges || (!edges && edgeCount))
+    return;
+  std::unordered_map<u32, u32> starts;
+  std::vector<u32> order(rangeCount);
+  for (u32 index = 0; index < rangeCount; index++) {
+    starts.emplace(ranges[index].start, index);
+    order[index] = index;
+  }
+  std::sort(order.begin(), order.end(), [&](u32 left, u32 right) {
+    return ranges[left].start < ranges[right].start;
+  });
+  auto containing = [&](u32 address) -> u32 {
+    auto position = std::upper_bound(
+        order.begin(), order.end(), address,
+        [&](u32 value, u32 index) { return value < ranges[index].start; });
+    if (position == order.begin())
+      return rangeCount;
+    const u32 index = *--position;
+    return address < ranges[index].end ? index : rangeCount;
+  };
+  const u32 native =
+      DOLLLVM_FUNCTION_ABI_NATIVE | DOLLLVM_FUNCTION_ABI_NATIVE_MEMORY;
+  for (bool changed = true; changed;) {
+    changed = false;
+    for (u32 edgeIndex = 0; edgeIndex < edgeCount; edgeIndex++) {
+      auto caller = starts.find(edges[edgeIndex].caller_start);
+      const u32 callee = containing(edges[edgeIndex].callee_address);
+      if (caller == starts.end() || callee == rangeCount ||
+          caller->second == callee ||
+          (ranges[caller->second].abi_flags & DOLLLVM_FUNCTION_ABI_NATIVE) ||
+          !(ranges[callee].abi_flags & DOLLLVM_FUNCTION_ABI_NATIVE))
+        continue;
+      ranges[callee].abi_flags &= ~native;
+      changed = true;
+    }
+  }
+}
+
 } // namespace
 
 extern "C" bool dolllvm_propagate_function_abis(DolLLVMFunctionRange *ranges,
@@ -213,3 +254,14 @@ extern "C" void dolllvm_apply_native_abi_policy(DolLLVMFunctionRange *ranges,
           ~(DOLLLVM_FUNCTION_ABI_NATIVE | DOLLLVM_FUNCTION_ABI_NATIVE_MEMORY);
   }
 }
+
+extern "C" void dolllvm_finish_function_abis(DolLLVMFunctionRange *ranges,
+                                             u32 rangeCount,
+                                             DolLLVMNativeABIPolicy policy,
+                                             DolLLVMRuntime runtime,
+                                             const DolLLVMCallEdge *edges,
+                                             u32 edgeCount) {
+  dolllvm_apply_native_abi_policy(ranges, rangeCount, policy);
+  if (runtime != DOLLLVM_RUNTIME_MODERNGEKKO)
+    demotePlainCallerTargets(ranges, rangeCount, edges, edgeCount);
+}

src/backend/llvm/llvm_backend.cpp

@@ -130,8 +130,8 @@ extern "C" bool dolllvm_emit_object(const DolIRModule *source,
     } else {
       ranges.assign(options.function_ranges, options.function_ranges + options.function_range_count);
       dolllvm::prepareModuleABIs(*source, ranges, options.runtime, &callEdges);
-      dolllvm_apply_native_abi_policy(ranges.data(), static_cast<u32>(ranges.size()),
-                                      options.native_abi_policy);
+      dolllvm_finish_function_abis(ranges.data(), static_cast<u32>(ranges.size()), options.native_abi_policy,
+                                   options.runtime, callEdges.data(), static_cast<u32>(callEdges.size()));
       options.function_ranges = ranges.data();
     }
     options.call_edges = callEdges.data();

src/backend/llvm/llvm_backend.h

@@ -159,6 +159,12 @@ void dolllvm_enable_native_services(DolLLVMFunctionRange *ranges,
 void dolllvm_apply_native_abi_policy(DolLLVMFunctionRange *ranges,
                                      u32 range_count,
                                      DolLLVMNativeABIPolicy policy);
+void dolllvm_finish_function_abis(DolLLVMFunctionRange *ranges,
+                                  u32 range_count,
+                                  DolLLVMNativeABIPolicy policy,
+                                  DolLLVMRuntime runtime,
+                                  const DolLLVMCallEdge *edges,
+                                  u32 edge_count);
 void dolllvm_report_abi_stats(const DolLLVMFunctionRange *ranges,
                               u32 range_count, const char *target_triple,
                               FILE *output);