[SYCL] Emit SYCL fat binary for host-side wrapping

clang/lib/CodeGen/CMakeLists.txt

@@ -2,6 +2,7 @@ set(LLVM_LINK_COMPONENTS
   ABI
   AggressiveInstCombine
   Analysis
+  BinaryFormat
   BitReader
   BitWriter
   CodeGen

clang/lib/CodeGen/CodeGenSYCL.cpp

@@ -14,7 +14,7 @@
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "clang/Basic/DiagnosticFrontend.h"
-#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Frontend/Offloading/OffloadWrapper.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/VirtualFileSystem.h"
@@ -112,14 +112,13 @@ llvm::Function *CodeGenModule::embedSYCLDeviceBinary() {
     return nullptr;
   }
   std::unique_ptr<llvm::MemoryBuffer> Buffer = std::move(BufferOrErr.get());
-  // A wrapper module in bitcode form already carries the registration
-  // descriptor the SYCL runtime expects. It is linked into the host module by
-  // loadLinkModules, so there is nothing to embed here. In SYCL CUDA
-  // compatibility mode the file is the CUDA fat binary, which CUDA registers.
-  if (getLangOpts().CUDA ||
-      llvm::isBitcode(
-          reinterpret_cast<const unsigned char *>(Buffer->getBufferStart()),
-          reinterpret_cast<const unsigned char *>(Buffer->getBufferEnd())))
+  // The SYCL runtime registers device images through a descriptor that also
+  // carries their properties, kernel entries and options. loadLinkModules
+  // wraps an offload binary container into such a descriptor, so there is
+  // nothing to embed here. In SYCL CUDA compatibility mode the file is the
+  // CUDA fat binary, which CUDA registers.
+  if (getLangOpts().CUDA || llvm::identify_magic(Buffer->getBuffer()) ==
+                                llvm::file_magic::offload_binary)
     return nullptr;
   llvm::Function *RegistrationFunc = nullptr;
   if (llvm::Error Err = llvm::offloading::wrapSYCLBinaries(

clang/lib/CodeGen/ModuleLinker.cpp

@@ -11,7 +11,9 @@
 #include "clang/Basic/CodeGenOptions.h"
 #include "clang/Basic/DiagnosticFrontend.h"
 #include "clang/Frontend/CompilerInstance.h"
+#include "llvm/BinaryFormat/Magic.h"
 #include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Frontend/Offloading/SYCLOffloadWrapper.h"
 #include "llvm/IR/Module.h"
 
 using namespace clang;
@@ -47,39 +49,45 @@ bool clang::loadLinkModules(CompilerInstance &CI, llvm::LLVMContext &Ctx,
                            F.Internalize, F.LinkFlags});
   }
 
-  // For SYCL no-RDC, link the per-TU wrapper module produced by
-  // clang-linker-wrapper --emit-fatbin-only into the host module so the SYCL
-  // runtime finds the device image at program startup. A raw device binary is
-  // embedded by CodeGenModule::embedSYCLDeviceBinary instead.
-  if (CI.getLangOpts().SYCLIsHost &&
-      !CI.getCodeGenOpts().OffloadBinaryToEmbedFile.empty()) {
-    auto BCBuf = CI.getFileManager().getBufferForFile(
-        CI.getCodeGenOpts().OffloadBinaryToEmbedFile);
-    if (!BCBuf) {
+  // For SYCL no-RDC, wrap the device images of the fat binary produced by
+  // clang-linker-wrapper --emit-fatbin-only into a module that registers them
+  // with the SYCL runtime, and link it into the host module. Other binaries
+  // are embedded by CodeGenModule::embedSYCLDeviceBinary instead.
+  StringRef FatbinFile = CI.getCodeGenOpts().OffloadBinaryToEmbedFile;
+  if (CI.getLangOpts().SYCLIsHost && !CI.getLangOpts().CUDA &&
+      !FatbinFile.empty()) {
+    auto FatbinBuf = CI.getFileManager().getBufferForFile(FatbinFile);
+    if (!FatbinBuf) {
       CI.getDiagnostics().Report(diag::err_cannot_open_file)
-          << CI.getCodeGenOpts().OffloadBinaryToEmbedFile
-          << BCBuf.getError().message();
+          << FatbinFile << FatbinBuf.getError().message();
       LinkModules.clear();
       return true;
     }
-    if (!llvm::isBitcode(
-            reinterpret_cast<const unsigned char *>((*BCBuf)->getBufferStart()),
-            reinterpret_cast<const unsigned char *>((*BCBuf)->getBufferEnd())))
+    if (llvm::identify_magic((*FatbinBuf)->getBuffer()) !=
+        llvm::file_magic::offload_binary)
       return false;
-    llvm::Expected<std::unique_ptr<llvm::Module>> MOrErr =
-        llvm::parseBitcodeFile((*BCBuf)->getMemBufferRef(), Ctx);
-    if (!MOrErr) {
-      llvm::handleAllErrors(MOrErr.takeError(), [&](llvm::ErrorInfoBase &EIB) {
+
+    auto ReportError = [&](llvm::Error Err) {
+      llvm::handleAllErrors(std::move(Err), [&](llvm::ErrorInfoBase &EIB) {
         CI.getDiagnostics().Report(diag::err_fe_linking_module)
-            << CI.getCodeGenOpts().OffloadBinaryToEmbedFile << EIB.message();
+            << FatbinFile << EIB.message();
       });
       LinkModules.clear();
       return true;
-    }
-    // The wrapper .bc from clang-linker-wrapper has no data layout set.
-    // Stamp the host layout so linkModules doesn't warn about a mismatch.
-    (*MOrErr)->setDataLayout(CI.getTarget().getDataLayoutString());
-    LinkModules.push_back({std::move(*MOrErr),
+    };
+    auto ImagesOrErr =
+        llvm::offloading::readSYCLImages((*FatbinBuf)->getMemBufferRef());
+    if (!ImagesOrErr)
+      return ReportError(ImagesOrErr.takeError());
+
+    auto WrapperM =
+        std::make_unique<llvm::Module>("offload.wrapper.object", Ctx);
+    WrapperM->setTargetTriple(CI.getTarget().getTriple());
+    WrapperM->setDataLayout(CI.getTarget().getDataLayoutString());
+    if (llvm::Error Err =
+            llvm::offloading::wrapSYCLBinaries(*WrapperM, *ImagesOrErr))
+      return ReportError(std::move(Err));
+    LinkModules.push_back({std::move(WrapperM),
                            /*PropagateAttrs=*/false,
                            /*Internalize=*/false,
                            /*LinkFlags=*/0});

clang/test/CodeGen/sycl-no-rdc-embed-binary.cpp

@@ -1,25 +1,48 @@
-// Tests for -foffload-include-binary with a SYCL wrapper module in bitcode form
-// (SYCL -fno-sycl-rdc compile-step device embedding via loadLinkModules).
+// Tests for -foffload-include-binary with a SYCL fat binary in offload binary
+// form (SYCL -fno-sycl-rdc compile-step device embedding via loadLinkModules).
 
 // Missing file: err_cannot_open_file
 // RUN: not %clang_cc1 -triple x86_64-unknown-linux-gnu -fsycl-is-host \
 // RUN:   -foffload-include-binary no-such-file.bc -emit-obj -o /dev/null %s 2>&1 \
 // RUN:   | FileCheck --check-prefix=CHECK-NO-FILE %s
 // CHECK-NO-FILE: fatal error: cannot open file 'no-such-file.bc'
 
-// Wrapper module bitcode: linked into the host module as is, so its
-// registration constructor is kept and no raw device binary is embedded.
+// Offload binary: each image is wrapped into the descriptor the SYCL runtime
+// registers, together with its target, options and kernel entries. The raw
+// container itself is not embedded.
 // RUN: split-file %s %t
-// RUN: llvm-as %t/wrapper.ll -o %t/wrapper.bc
+// RUN: echo -n 'FAKE_SYCL_DEVICE_IMAGE' > %t/image.bin
+// RUN: llvm-offload-binary -o %t/fatbin.syclfb \
+// RUN:   --image=file=%t/image.bin,kind=sycl,triple=spir64-unknown-unknown,arch=,sycl-target=spir64,sycl-image-format=0,symbols=_ZTS1K,compile-opts=-g,link-opts=-cl-opt
 // RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fsycl-is-host \
-// RUN:   -foffload-include-binary %t/wrapper.bc -emit-llvm %t/host.cpp -o - \
+// RUN:   -foffload-include-binary %t/fatbin.syclfb -emit-llvm %t/host.cpp -o - \
 // RUN:   | FileCheck --check-prefix=CHECK-LINK %s \
 // RUN:     --implicit-check-not='.sycl_fatbin' \
 // RUN:     --implicit-check-not='.sycl_offloading.binary'
-// CHECK-LINK: @llvm.global_ctors = {{.*}} ptr @sycl.descriptor_reg
+// CHECK-LINK-DAG: @.sycl_offloading.target.0 = internal unnamed_addr constant [7 x i8] c"spir64\00"
+// CHECK-LINK-DAG: @.sycl_offloading.opts.compile.0 = internal unnamed_addr constant [3 x i8] c"-g\00"
+// CHECK-LINK-DAG: @.sycl_offloading.opts.link.0 = internal unnamed_addr constant [8 x i8] c"-cl-opt\00"
+// CHECK-LINK-DAG: @.sycl_offloading.0.data = internal unnamed_addr constant [22 x i8] c"FAKE_SYCL_DEVICE_IMAGE"
+// CHECK-LINK-DAG: @__sycl_offload_entry_name = internal unnamed_addr constant [7 x i8] c"_ZTS1K\00"
+// CHECK-LINK-DAG: @llvm.global_ctors = {{.*}} ptr @sycl.descriptor_reg
 // CHECK-LINK: define internal void @sycl.descriptor_reg()
 // CHECK-LINK: call void @__sycl_register_lib(ptr @.sycl_offloading.descriptor)
 
+// Malformed runtime metadata is diagnosed.
+// RUN: llvm-offload-binary -o %t/no-format.syclfb \
+// RUN:   --image=file=%t/image.bin,kind=sycl,triple=spir64-unknown-unknown
+// RUN: not %clang_cc1 -triple x86_64-unknown-linux-gnu -fsycl-is-host \
+// RUN:   -foffload-include-binary %t/no-format.syclfb -emit-llvm %t/host.cpp \
+// RUN:   -o /dev/null 2>&1 | FileCheck --check-prefix=CHECK-NO-FORMAT %s
+// CHECK-NO-FORMAT: error: cannot link module '{{.*}}no-format.syclfb': invalid SYCL image format ''
+
+// RUN: llvm-offload-binary -o %t/openmp.syclfb \
+// RUN:   --image=file=%t/image.bin,kind=openmp,triple=spir64-unknown-unknown
+// RUN: not %clang_cc1 -triple x86_64-unknown-linux-gnu -fsycl-is-host \
+// RUN:   -foffload-include-binary %t/openmp.syclfb -emit-llvm %t/host.cpp \
+// RUN:   -o /dev/null 2>&1 | FileCheck --check-prefix=CHECK-NOT-SYCL %s
+// CHECK-NOT-SYCL: error: cannot link module '{{.*}}openmp.syclfb': offload binary entry 0 is not a SYCL device image
+
 // SYCL CUDA compatibility mode: the host compilation is also a CUDA host
 // compilation and the file is the CUDA fat binary. Only CUDA registers it; it
 // is neither linked nor embedded as a SYCL device image.
@@ -33,17 +56,6 @@
 // CHECK-SYCL-CUDA: section ".nv_fatbin"
 // CHECK-SYCL-CUDA: call ptr @__cudaRegisterFatBinary(
 
-//--- wrapper.ll
-@.sycl_offloading.descriptor = internal constant [1 x i8] zeroinitializer
-@llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 1, ptr @sycl.descriptor_reg, ptr null }]
-
-define internal void @sycl.descriptor_reg() {
-  call void @__sycl_register_lib(ptr @.sycl_offloading.descriptor)
-  ret void
-}
-
-declare void @__sycl_register_lib(ptr)
-
 //--- host.cpp
 void f() {}
 

clang/test/Driver/sycl-nordc-fatbin.cpp

@@ -1,42 +1,38 @@
 // REQUIRES: spirv-registered-target, x86-registered-target
 
-// TODO: With the intel/llvm SYCL driver the .sycl_fatbin section holds LLVM
-// bitcode rather than an offload binary wrapping a finalized SPIR-V module.
-// Enable once the -fno-sycl-rdc flow matches upstream.
-// XFAIL: *
+// REQUIRES: spirv-to-ir-wrapper, sycl-post-link
 
-/// Verify which section carries the SYCL device binary in the host object.
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl -fno-gpu-rdc \
+/// Verify that no-RDC emits image address/size records in .tgtimg, while RDC
+/// embeds device code for linking in .llvm.offloading.
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl --offload-new-driver --no-offloadlib -fno-gpu-rdc \
 // RUN:   -c %s -o %t.nordc.o
 // RUN: llvm-readelf -S %t.nordc.o \
 // RUN:   | FileCheck -check-prefix=NORDC %s --implicit-check-not='.llvm.offloading'
 
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl -fgpu-rdc \
+/// -fsycl-link also forwards --sycl-device-link to the per-TU no-RDC job.
+/// Fatbin-only completion must skip the device-link copy branch.
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl --offload-new-driver --no-offloadlib -fno-gpu-rdc \
+// RUN:   -fsycl-link -c %s -o %t.nordc.devlink.o
+// RUN: llvm-readelf -S %t.nordc.devlink.o \
+// RUN:   | FileCheck -check-prefix=NORDC %s --implicit-check-not='.llvm.offloading'
+
+// RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl --offload-new-driver --no-offloadlib -fgpu-rdc \
 // RUN:   -c %s -o %t.rdc.o
 // RUN: llvm-readelf -S %t.rdc.o \
-// RUN:   | FileCheck -check-prefix=RDC %s --implicit-check-not='.sycl_fatbin'
+// RUN:   | FileCheck -check-prefix=RDC %s --implicit-check-not='.tgtimg'
 
 /// The choice does not depend on the object format.
-// RUN: %clangxx --target=x86_64-pc-windows-msvc -fsycl -fno-gpu-rdc \
+// RUN: %clangxx --target=x86_64-pc-windows-msvc -fsycl --offload-new-driver --no-offloadlib -fno-gpu-rdc \
 // RUN:   -c %s -o %t.nordc.obj
 // RUN: llvm-readobj --sections %t.nordc.obj \
 // RUN:   | FileCheck -check-prefix=NORDC %s --implicit-check-not='.llvm.offloading'
 
-// RUN: %clangxx --target=x86_64-pc-windows-msvc -fsycl -fgpu-rdc \
+// RUN: %clangxx --target=x86_64-pc-windows-msvc -fsycl --offload-new-driver --no-offloadlib -fgpu-rdc \
 // RUN:   -c %s -o %t.rdc.obj
 // RUN: llvm-readobj --sections %t.rdc.obj \
-// RUN:   | FileCheck -check-prefix=RDC %s --implicit-check-not='.sycl_fatbin'
+// RUN:   | FileCheck -check-prefix=RDC %s --implicit-check-not='.tgtimg'
 
-// NORDC: .sycl_fatbin
+// NORDC: .tgtimg
 // RDC: .llvm.offloading
 
-/// The section holds an offload binary (magic 0x10FF10AD) whose image is a
-/// finalized SPIR-V module (magic 0x07230203), both shown little endian by the
-/// hex dump.
-// RUN: llvm-readelf --hex-dump=.sycl_fatbin %t.nordc.o \
-// RUN:   | FileCheck -check-prefix=BINARY %s
-
-// BINARY: 10ff10ad
-// BINARY: 03022307
-
 void f() {}

clang/test/Driver/sycl-nordc-wrapper-object.cpp

@@ -1,39 +1,15 @@
-// REQUIRES: spirv-registered-target, x86-registered-target
-// REQUIRES: spirv-to-ir-wrapper, sycl-post-link
+// REQUIRES: sycl-post-link
 
-/// intel/llvm counterpart of LLORG's sycl-nordc-fatbin.cpp. In intel/llvm the
-/// non-RDC host object carries the wrapper module produced at compile time:
-/// the device image in ".tgtimg" plus the __sycl_register_lib registration,
-/// and no ".llvm.offloading", so the final link does not device-link it again.
-/// The RDC host object is the reverse.
+/// Verify that the no-RDC driver flow registers and unregisters the descriptor
+/// expected by the Intel SYCL runtime.
 // RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl --offload-new-driver \
-// RUN:   --no-offloadlib -fno-gpu-rdc -c %s -o %t.nordc.o
-// RUN: llvm-readelf -S %t.nordc.o \
-// RUN:   | FileCheck -check-prefix=NORDC %s --implicit-check-not='.llvm.offloading'
-// RUN: llvm-nm %t.nordc.o | FileCheck -check-prefix=REG %s
-
-// RUN: %clangxx --target=x86_64-unknown-linux-gnu -fsycl --offload-new-driver \
-// RUN:   --no-offloadlib -fgpu-rdc -c %s -o %t.rdc.o
-// RUN: llvm-readelf -S %t.rdc.o \
-// RUN:   | FileCheck -check-prefix=RDC %s --implicit-check-not='.tgtimg'
-// RUN: llvm-nm %t.rdc.o | FileCheck -check-prefix=NOREG %s
-
-/// The choice does not depend on the object format.
-// RUN: %clangxx --target=x86_64-pc-windows-msvc -fsycl --offload-new-driver \
-// RUN:   --no-offloadlib -fno-gpu-rdc -c %s -o %t.nordc.obj
-// RUN: llvm-readobj --sections %t.nordc.obj \
-// RUN:   | FileCheck -check-prefix=NORDC %s --implicit-check-not='.llvm.offloading'
-// RUN: llvm-nm %t.nordc.obj | FileCheck -check-prefix=REG %s
-
+// RUN:   --no-offloadlib -fno-gpu-rdc -S -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
 // RUN: %clangxx --target=x86_64-pc-windows-msvc -fsycl --offload-new-driver \
-// RUN:   --no-offloadlib -fgpu-rdc -c %s -o %t.rdc.obj
-// RUN: llvm-readobj --sections %t.rdc.obj \
-// RUN:   | FileCheck -check-prefix=RDC %s --implicit-check-not='.tgtimg'
-// RUN: llvm-nm %t.rdc.obj | FileCheck -check-prefix=NOREG %s
+// RUN:   --no-offloadlib -fno-gpu-rdc -S -emit-llvm %s -o - \
+// RUN:   | FileCheck %s
 
-// NORDC: .tgtimg
-// RDC: .llvm.offloading
-// REG: U __sycl_register_lib
-// NOREG-NOT: __sycl_register_lib
+// CHECK: call void @__sycl_register_lib(ptr @.sycl_offloading.descriptor)
+// CHECK: call void @__sycl_unregister_lib(ptr @.sycl_offloading.descriptor)
 
 void f() {}

clang/test/OffloadTools/clang-linker-wrapper/linker-wrapper-sycl-fatbin.cpp

@@ -0,0 +1,53 @@
+// REQUIRES: system-linux
+
+// Fatbin-only output must not generate or compile a host wrapper.
+// RUN: touch %t.o
+// RUN: llvm-offload-binary -o %t.pack --image=file=%t.o,triple=spir64-unknown-unknown,arch=generic,kind=sycl
+// RUN: clang-linker-wrapper --dry-run --emit-fatbin-only --linker-path=/usr/bin/ld %t.pack -o %t.fatbin 2>&1 | FileCheck %s
+// CHECK: sycl-post-link
+// CHECK: llvm-spirv
+// CHECK-NOT: offload-wrapper
+// CHECK-NOT: "{{.*}}clang"{{.*}} -c
+// CHECK-NOT: "/usr/bin/ld"
+// RUN: llvm-objdump --offloading %t.fatbin | FileCheck %s --check-prefix=CONTAINER
+// CONTAINER: OFFLOADING IMAGE [0]:
+// CONTAINER: kind elf
+// CONTAINER: triple spir64-unknown-unknown
+// CONTAINER: producer sycl
+
+// Fatbin-only completion takes precedence over the device-link copy branch.
+// RUN: clang-linker-wrapper --dry-run --emit-fatbin-only --sycl-device-link --linker-path=/usr/bin/ld %t.pack -o %t.devlink.fatbin 2>&1 | FileCheck %s
+// RUN: llvm-objdump --offloading %t.devlink.fatbin | FileCheck %s --check-prefix=CONTAINER
+
+// Keep all independent no-RDC modules in a single container.
+// RUN: llvm-offload-binary -o %t.multi.pack --image=file=%t.o,triple=spir64-unknown-unknown,arch=generic,kind=sycl --image=file=%t.o,triple=spir64-unknown-unknown,arch=generic,kind=sycl
+// RUN: clang-linker-wrapper --dry-run --emit-fatbin-only --no-sycl-rdc --linker-path=/usr/bin/ld %t.multi.pack -o %t.multi.fatbin 2>&1 | FileCheck %s
+// RUN: llvm-objdump --offloading %t.multi.fatbin | FileCheck %s --check-prefix=MULTI
+// MULTI: OFFLOADING IMAGE [0]:
+// MULTI: producer sycl
+// MULTI: OFFLOADING IMAGE [1]:
+// MULTI: producer sycl
+// MULTI-NOT: OFFLOADING IMAGE
+
+// Reject modes that add host objects through the pipeline callback rather
+// than allowing them to overwrite the device-image container.
+// RUN: llvm-offload-binary -o %t.native.pack --image=file=%t.o,triple=native_cpu,arch=generic,kind=sycl
+// RUN: not clang-linker-wrapper --dry-run --emit-fatbin-only --linker-path=/usr/bin/ld %t.native.pack -o %t.native.fatbin 2>&1 | FileCheck %s --check-prefix=NATIVE
+// RUN: test ! -e %t.native.fatbin
+// NATIVE-NOT: sycl-post-link
+// NATIVE: error: --emit-fatbin-only is not supported for NativeCPU
+// NATIVE-NOT: sycl-post-link
+
+// RUN: llvm-offload-binary -o %t.nvptx.pack --image=file=%t.o,triple=nvptx64-nvidia-cuda,arch=sm_50,kind=sycl
+// RUN: not clang-linker-wrapper --dry-run --emit-fatbin-only --sycl-embed-ir --linker-path=/usr/bin/ld %t.nvptx.pack -o %t.nvptx.fatbin 2>&1 | FileCheck %s --check-prefix=EMBED
+// RUN: test ! -e %t.nvptx.fatbin
+// RUN: llvm-offload-binary -o %t.amdgcn.pack --image=file=%t.o,triple=amdgcn-amd-amdhsa,arch=gfx908,kind=sycl
+// RUN: not clang-linker-wrapper --dry-run --emit-fatbin-only --sycl-embed-ir --linker-path=/usr/bin/ld %t.amdgcn.pack -o %t.amdgcn.fatbin 2>&1 | FileCheck %s --check-prefix=EMBED
+// RUN: test ! -e %t.amdgcn.fatbin
+// EMBED-NOT: sycl-post-link
+// EMBED: error: --emit-fatbin-only is not supported with --sycl-embed-ir for NVPTX or AMDGCN
+// EMBED-NOT: sycl-post-link
+
+// Embedding IR does not add host objects for SPIR targets.
+// RUN: clang-linker-wrapper --dry-run --emit-fatbin-only --sycl-embed-ir --linker-path=/usr/bin/ld %t.pack -o %t.spir.fatbin 2>&1 | FileCheck %s
+// RUN: llvm-objdump --offloading %t.spir.fatbin | FileCheck %s --check-prefix=CONTAINER