[WIP][Driver][SYCL] Link device libraries at compile time for 3rd party

clang/lib/Driver/Driver.cpp

@@ -5859,8 +5859,7 @@ class OffloadingActionBuilder final {
       // specific libs and doing a separate directory search.  Each toolchain
       // has their own getDeviceLibs that we can potentially use.
       DeviceLibraries =
-          SYCLTC.getDeviceLibNames(C.getDriver(), Args, TC->getTriple());
-
+          clang::driver::getSYCLDeviceLibNames(C.getDriver(), Args, *TC);
       for (const auto &DeviceLib : DeviceLibraries) {
         for (const auto &LLCandidate : LibLocCandidates) {
           SmallString<128> LibName(LLCandidate);

clang/lib/Driver/ToolChains/Clang.cpp

@@ -11773,87 +11773,10 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Add any SYCL offloading specific options to the clang-linker-wrapper
   if (C.hasOffloadToolChain<Action::OFK_SYCL>()) {
-
-    if (Args.hasArg(options::OPT_fsycl_link_EQ))
-      CmdArgs.push_back(Args.MakeArgString("--sycl-device-link"));
-
-    // -sycl-device-library-location=<dir> provides the location in which the
-    // SYCL device libraries can be found.
-    SmallString<128> DeviceLibDir(D.Dir);
-    llvm::sys::path::append(DeviceLibDir, "..", "lib");
-    // Check the library location candidates for the the libsycl-crt library
-    // and use that location.  Base the location on relative to driver if this
-    // is not resolved.
-    SmallVector<SmallString<128>, 4> LibLocCandidates;
     SYCLInstallationDetector SYCLInstallation(D, getToolChain().getTriple(),
                                               Args);
-    SYCLInstallation.getSYCLDeviceLibPath(LibLocCandidates);
-    SmallString<128> LibName("libsycl-crt");
-    bool IsNewOffload = D.getUseNewOffloadingDriver();
-    StringRef LibSuffix = TheTriple.isWindowsMSVCEnvironment()
-                              ? (IsNewOffload ? ".new.obj" : ".obj")
-                              : (IsNewOffload ? ".new.o" : ".o");
-    llvm::sys::path::replace_extension(LibName, LibSuffix);
-    for (const auto &LibLoc : LibLocCandidates) {
-      SmallString<128> FullLibName(LibLoc);
-      llvm::sys::path::append(FullLibName, LibName);
-      if (llvm::sys::fs::exists(FullLibName)) {
-        DeviceLibDir = LibLoc;
-        break;
-      }
-    }
-
-    // Device libraries for SYCL offloading are specified using
-    // --bitcode-library=<triple>=<path to bc file> for each library.
-    // The full path must be provided (not relative to
-    // -sycl-device-library-location).
-    //
-    // Note: SPIR/SPIRV device libraries are linked at compile time using
-    // -mlink-builtin-bitcode, so they are not passed here. Only non-SPIR
-    // targets (NVPTX, AMD) pass device libraries to clang-linker-wrapper.
-    SmallVector<std::string, 4> BCLibList;
-
-    auto appendToList = [](SmallString<256> &List, const Twine &Arg) {
-      if (List.size() > 0)
-        List += ",";
-      List += Arg.str();
-    };
-
-    auto ToolChainRange = C.getOffloadToolChains<Action::OFK_SYCL>();
-    for (const auto &[Kind, TC] :
-         llvm::make_range(ToolChainRange.first, ToolChainRange.second)) {
-      llvm::Triple TargetTriple = TC->getTriple();
-      const toolchains::SYCLToolChain &SYCLTC =
-          static_cast<const toolchains::SYCLToolChain &>(*TC);
-      SmallVector<ToolChain::BitCodeLibraryInfo, 8> SYCLDeviceLibs;
-      // SPIR or SPIR-V device libraries are compiled into the device compile
-      // step.
-      if (!TargetTriple.isSPIROrSPIRV())
-        SYCLDeviceLibs.append(SYCLTC.getDeviceLibNames(D, Args, TargetTriple));
-      for (const auto &AddLib : SYCLDeviceLibs) {
-        if (llvm::sys::path::extension(AddLib.Path) == ".bc") {
-          SmallString<256> LibPath(DeviceLibDir);
-          llvm::sys::path::append(LibPath, AddLib.Path);
-          BCLibList.push_back(
-              (Twine(TC->getTriple().str()) + "=" + LibPath).str());
-          continue;
-        }
-      }
-
-      if (TC->getTriple().isNVPTX())
-        if (const char *LibSpirvFile = SYCLInstallation.findLibspirvPath(
-                TC->getTriple(), Args, *TC->getAuxTriple()))
-          BCLibList.push_back(
-              (Twine(TC->getTriple().str()) + "=" + LibSpirvFile).str());
-    }
-
-    if (BCLibList.size())
-      for (const std::string &Lib : BCLibList)
-        CmdArgs.push_back(
-            Args.MakeArgString(Twine("--bitcode-library=") + Lib));
-
-    CmdArgs.push_back(Args.MakeArgString(
-        Twine("-sycl-device-library-location=") + DeviceLibDir));
+    if (Args.hasArg(options::OPT_fsycl_link_EQ))
+      CmdArgs.push_back(Args.MakeArgString("--sycl-device-link"));
 
     if (C.getDriver().isSaveOffloadCodeEnabled()) {
       SmallString<128> DumpDir;
@@ -11936,9 +11859,46 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
     // -Xsycl-target-linker are forwarded using --device-linker.
     const toolchains::SYCLToolChain &SYCLTC =
         static_cast<const toolchains::SYCLToolChain &>(getToolChain());
+    auto ToolChainRange = C.getOffloadToolChains<Action::OFK_SYCL>();
     for (auto &ToolChainMember :
          llvm::make_range(ToolChainRange.first, ToolChainRange.second)) {
       const ToolChain *TC = ToolChainMember.second;
+      // Handle native_cpu separately to add device library via
+      // --device-compiler
+      if (TC->getTriple().isNativeCPU()) {
+        SmallString<64> NativeUtilsLib("libsycl-nativecpu_utils.bc");
+        SYCLInstallationDetector SYCLInstall(TC->getDriver(), TC->getTriple(),
+                                             Args);
+        SmallVector<SmallString<128>, 4> DeviceLibLocCandidates;
+        SYCLInstall.getSYCLDeviceLibPath(DeviceLibLocCandidates);
+        for (const auto &DeviceLibLoc : DeviceLibLocCandidates) {
+          SmallString<128> FullLibName(DeviceLibLoc);
+          llvm::sys::path::append(FullLibName, NativeUtilsLib);
+          if (llvm::sys::fs::exists(FullLibName)) {
+            CmdArgs.push_back(Args.MakeArgString(
+                "--device-compiler=" +
+                Action::GetOffloadKindName(Action::OFK_SYCL) + ":" +
+                TC->getTripleString() + "=-Xclang"));
+            CmdArgs.push_back(Args.MakeArgString(
+                "--device-compiler=" +
+                Action::GetOffloadKindName(Action::OFK_SYCL) + ":" +
+                TC->getTripleString() + "=-mlink-bitcode-file"));
+            SmallString<256> BackendOptString;
+            BackendOptString = FullLibName;
+
+            CmdArgs.push_back(Args.MakeArgString(
+                "--device-compiler=" +
+                Action::GetOffloadKindName(Action::OFK_SYCL) + ":" +
+                TC->getTripleString() + "=-Xclang"));
+            CmdArgs.push_back(Args.MakeArgString(
+                "--device-compiler=" +
+                Action::GetOffloadKindName(Action::OFK_SYCL) + ":" +
+                TC->getTripleString() + "=" + BackendOptString));
+            break;
+          }
+        }
+        continue; // Don't process native_cpu through the SPIR/SPIRV path below
+      }
       if (!TC->getTriple().isSPIROrSPIRV())
         continue;
       ArgStringList BuildArgs;

clang/lib/Driver/ToolChains/Cuda.cpp

@@ -1003,9 +1003,6 @@ void CudaToolChain::addClangTargetOptions(
         CC1Args.append(
             {"-mllvm", "-nvvm-reflect-add=__CUDA_ID_QUERIES_FIT_IN_UINT=1"});
     }
-
-    SYCLInstallation.addLibspirvLinkArgs(getEffectiveTriple(), DriverArgs,
-                                         HostTC.getTriple(), CC1Args);
   } else {
     CC1Args.append({"-fcuda-is-device", "-mllvm",
                     "-enable-memcpyopt-without-libcalls",
@@ -1021,14 +1018,48 @@ void CudaToolChain::addClangTargetOptions(
                          options::OPT_fno_cuda_short_ptr, false))
     CC1Args.append({"-mllvm", "--nvptx-short-ptr"});
 
-  if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
-                          true))
-    return;
-
   if (DeviceOffloadingKind == Action::OFK_OpenMP &&
       DriverArgs.hasArg(options::OPT_S))
     return;
 
+  // For SYCL offloading, add SYCL device libraries.
+  if (DeviceOffloadingKind == Action::OFK_SYCL) {
+    if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
+                            true)) {
+      llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12> BCLibs;
+
+      llvm::SmallVector<llvm::SmallString<128>, 4> LibraryPaths;
+      SYCLInstallation.getSYCLDeviceLibPath(LibraryPaths);
+
+      // Get SYCL device library names for NVPTX target
+      llvm::SmallVector<BitCodeLibraryInfo, 8> DeviceLibs =
+          clang::driver::getSYCLDeviceLibNames(getDriver(), DriverArgs, *this);
+
+      // Create full path names to each device library
+      for (const auto &DeviceLib : DeviceLibs) {
+        for (const auto &LibraryPath : LibraryPaths) {
+          llvm::SmallString<128> FullLibName(LibraryPath);
+          llvm::sys::path::append(FullLibName, DeviceLib.Path);
+          if (llvm::sys::fs::exists(FullLibName)) {
+            BitCodeLibraryInfo BitCodeLibrary(
+                {FullLibName, DeviceLib.ShouldInternalize});
+            BCLibs.emplace_back(BitCodeLibrary);
+            break;
+          }
+        }
+      }
+      for (const auto &BCLib : BCLibs) {
+        CC1Args.push_back("-mlink-builtin-bitcode");
+        CC1Args.push_back(DriverArgs.MakeArgString(BCLib.Path));
+      }
+    }
+    SYCLInstallation.addLibspirvLinkArgs(getEffectiveTriple(), DriverArgs,
+                                         HostTC.getTriple(), CC1Args);
+  }
+  if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
+                          true))
+    return;
+
   std::string LibDeviceFile = CudaInstallation.getLibDeviceFile(GpuArch);
   if (LibDeviceFile.empty()) {
     getDriver().Diag(diag::err_drv_no_cuda_libdevice) << GpuArch;

clang/lib/Driver/ToolChains/HIPAMD.cpp

@@ -289,17 +289,16 @@ void HIPAMDToolChain::addClangTargetOptions(
     return; // No DeviceLibs for SPIR-V.
   }
 
-  if (DeviceOffloadingKind == Action::OFK_SYCL) {
-    SYCLInstallation.addSYCLIncludeArgs(DriverArgs, CC1Args);
-    SYCLInstallation.addLibspirvLinkArgs(getEffectiveTriple(), DriverArgs,
-                                         HostTC.getTriple(), CC1Args);
-  }
-
   for (auto BCFile : getDeviceLibs(DriverArgs, DeviceOffloadingKind)) {
     CC1Args.push_back(BCFile.ShouldInternalize ? "-mlink-builtin-bitcode"
                                                : "-mlink-bitcode-file");
     CC1Args.push_back(DriverArgs.MakeArgString(BCFile.Path));
   }
+  if (DeviceOffloadingKind == Action::OFK_SYCL) {
+    SYCLInstallation.addSYCLIncludeArgs(DriverArgs, CC1Args);
+    SYCLInstallation.addLibspirvLinkArgs(getEffectiveTriple(), DriverArgs,
+                                         HostTC.getTriple(), CC1Args);
+  }
 }
 
 llvm::opt::DerivedArgList *
@@ -417,6 +416,30 @@ HIPAMDToolChain::getDeviceLibs(const llvm::opt::ArgList &DriverArgs,
 
   addDirectoryList(DriverArgs, LibraryPaths, "", "HIP_DEVICE_LIB_PATH");
 
+  // For SYCL offloading, add SYCL device libraries.
+  if (DeviceOffloadingKind == Action::OFK_SYCL) {
+    llvm::SmallVector<llvm::SmallString<128>, 4> LibraryPaths;
+    SYCLInstallation.getSYCLDeviceLibPath(LibraryPaths);
+
+    // Get SYCL device library names for NVPTX target
+    llvm::SmallVector<BitCodeLibraryInfo, 8> DeviceLibs =
+        clang::driver::getSYCLDeviceLibNames(getDriver(), DriverArgs, *this);
+
+    // Create full path names to each device library
+    for (const auto &DeviceLib : DeviceLibs) {
+      for (const auto &LibraryPath : LibraryPaths) {
+        llvm::SmallString<128> FullLibName(LibraryPath);
+        llvm::sys::path::append(FullLibName, DeviceLib.Path);
+        if (llvm::sys::fs::exists(FullLibName)) {
+          BitCodeLibraryInfo BitCodeLibrary(
+              {FullLibName, DeviceLib.ShouldInternalize});
+          BCLibs.emplace_back(BitCodeLibrary);
+          break;
+        }
+      }
+    }
+  }
+
   // Maintain compatability with --hip-device-lib.
   auto BCLibArgs = DriverArgs.getAllArgValues(options::OPT_hip_device_lib_EQ);
   if (!BCLibArgs.empty()) {

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -567,35 +567,36 @@ static void addSYCLDeviceSanitizerLibs(
 
 // Returns the list of SYCL device library names for the given target.
 SmallVector<ToolChain::BitCodeLibraryInfo, 8>
-SYCLToolChain::getDeviceLibNames(const Driver &D,
-                                 const llvm::opt::ArgList &Args,
-                                 const llvm::Triple &TargetTriple) const {
+clang::driver::getSYCLDeviceLibNames(const Driver &D,
+                                     const llvm::opt::ArgList &Args,
+                                     const ToolChain &TC) {
   SmallVector<ToolChain::BitCodeLibraryInfo, 8> LibraryList;
+  const llvm::Triple TargetTriple(TC.getTriple());
   bool NoOffloadLib =
       !Args.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib, true);
   // Default internalization to 'true' for these libraries, as they are
   // expected to link with -mlink-builtin-bitcode.
   auto addLibToList = [&LibraryList](StringRef LibName,
                                      bool Internalize = true) {
-    BitCodeLibraryInfo BitCodeLibrary({LibName, Internalize});
+    ToolChain::BitCodeLibraryInfo BitCodeLibrary({LibName, Internalize});
     LibraryList.emplace_back(BitCodeLibrary);
   };
   if (TargetTriple.isNVPTX()) {
     if (!NoOffloadLib)
-      addLibToList("devicelib-nvptx64-nvidia-cuda.bc");
+      addLibToList("devicelib-nvptx64-nvidia-cuda.bc", false);
     return LibraryList;
   }
 
   if (TargetTriple.isAMDGCN()) {
     if (!NoOffloadLib)
-      addLibToList("devicelib-amdgcn-amd-amdhsa.bc");
+      addLibToList("devicelib-amdgcn-amd-amdhsa.bc", false);
     return LibraryList;
   }
 
   // Ignore no-offloadlib for NativeCPU device library, it provides some
   // critical builtins which must be linked with user's device image.
   if (TargetTriple.isNativeCPU()) {
-    addLibToList("libsycl-nativecpu_utils.bc");
+    addLibToList("libsycl-nativecpu_utils.bc", false);
     return LibraryList;
   }
 
@@ -628,7 +629,7 @@ SYCLToolChain::getDeviceLibNames(const Driver &D,
       "libsycl-native-bfloat16"};
   bool NativeBfloatLibs;
   bool NeedBfloatLibs =
-      selectBfloatLibs(Args, TargetTriple, *this, NativeBfloatLibs);
+      selectBfloatLibs(Args, TargetTriple, TC, NativeBfloatLibs);
   if (NeedBfloatLibs && !NoOffloadLib) {
     // Add native or fallback bfloat16 library.
     if (NativeBfloatLibs)
@@ -1439,11 +1440,18 @@ void SYCLToolChain::addClangTargetOptions(
     SYCLInstallation.addLibspirvLinkArgs(getEffectiveTriple(), DriverArgs,
                                          HostTC.getTriple(), CC1Args);
   }
-  // Only link device libraries at compile time for SPIR/SPIRV targets.
-  // Other targets (NVPTX, AMD) link at link time via clang-linker-wrapper.
-  // This is only done with the new offloading model, to fit with LLVM community
-  // implementation.
-  if (!getDriver().getUseNewOffloadingDriver() || !getTriple().isSPIROrSPIRV())
+  // Only link device libraries at compile time for the new offloading model.
+  if (!getDriver().getUseNewOffloadingDriver())
+    return;
+
+  // --no-offloadlib does not pull in device libraries.
+  if (!DriverArgs.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib,
+                          true))
+    return;
+
+  // For Native CPU, the device library is linked in with the
+  // clang-linker-wrapper call.
+  if (getTriple().isNativeCPU())
     return;
 
   llvm::SmallVector<BitCodeLibraryInfo, 12> BCLibs;
@@ -1910,7 +1918,7 @@ SYCLToolChain::getDeviceLibs(
 
   // Formulate all of the device libraries needed for this compilation.
   SmallVector<BitCodeLibraryInfo, 8> DeviceLibs =
-      getDeviceLibNames(getDriver(), DriverArgs, getTriple());
+      clang::driver::getSYCLDeviceLibNames(getDriver(), DriverArgs, *this);
 
   // Create full path names to each device library.  If found, add to the list
   // of device libraries that will be linked against.

clang/lib/Driver/ToolChains/SYCL.h

@@ -21,6 +21,12 @@ namespace driver {
 // AOT compiler).
 StringRef mapIntelGPUArchName(StringRef ArchName);
 
+// Provides a vector of SYCL device library names for the given target triple.
+// This can be used by non-SYCL toolchains when SYCL offloading is enabled.
+llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 8>
+getSYCLDeviceLibNames(const Driver &D, const llvm::opt::ArgList &Args,
+                      const ToolChain &TC);
+
 class Command;
 
 namespace tools {
@@ -154,11 +160,6 @@ class LLVM_LIBRARY_VISIBILITY SYCLToolChain : public ToolChain {
                           llvm::opt::OptSpecifier Opt,
                           llvm::opt::OptSpecifier Opt_EQ,
                           StringRef Device) const;
-  // Provides a vector of device library names that are associated with the
-  // given triple and AOT information.
-  SmallVector<ToolChain::BitCodeLibraryInfo, 8>
-  getDeviceLibNames(const Driver &D, const llvm::opt::ArgList &Args,
-                    const llvm::Triple &TargetTriple) const;
 
   bool useIntegratedAs() const override { return true; }
   bool isPICDefault() const override {