[Driver][SYCL][NewOffloadModel] Pull in device libraries at device compile

clang/lib/Driver/Driver.cpp

@@ -5528,7 +5528,7 @@ class OffloadingActionBuilder final {
         Action *NativeCPULib = nullptr;
         if (IsSPIR || IsNVPTX || IsAMDGCN || IsNativeCPU) {
           SYCLDeviceLibLinked = addSYCLDeviceLibs(
-              TC, SYCLDeviceLibs, IsSpirvAOT,
+              TC, SYCLDeviceLibs,
               C.getDefaultToolChain().getTriple().isWindowsMSVCEnvironment(),
               IsNativeCPU, NativeCPULib, BoundArch);
         }
@@ -5756,15 +5756,21 @@ class OffloadingActionBuilder final {
     }
 
     bool addSYCLDeviceLibs(const ToolChain *TC, ActionList &DeviceLinkObjects,
-                           bool isSpirvAOT, bool isMSVCEnv, bool isNativeCPU,
+                           bool isMSVCEnv, bool isNativeCPU,
                            Action *&NativeCPULib, const char *BoundArch) {
+
       int NumOfDeviceLibLinked = 0;
       SmallVector<SmallString<128>, 4> LibLocCandidates;
       SYCLInstallation.getSYCLDeviceLibPath(LibLocCandidates);
 
+      const toolchains::SYCLToolChain &SYCLTC =
+          static_cast<const toolchains::SYCLToolChain &>(*TC);
       SmallVector<std::string, 8> DeviceLibraries;
+      // TODO: Use the getDeviceLibs for each toolchain instead of using the
+      // specific libs and doing a separate directory search.  Each toolchain
+      // has their own getDeviceLibs that we can potentially use.
       DeviceLibraries =
-          tools::SYCL::getDeviceLibraries(C, TC->getTriple(), isSpirvAOT);
+          SYCLTC.getDeviceLibNames(C.getDriver(), Args, TC->getTriple());
 
       for (const auto &DeviceLib : DeviceLibraries) {
         for (const auto &LLCandidate : LibLocCandidates) {

clang/lib/Driver/ToolChains/Clang.cpp

@@ -11474,7 +11474,6 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
     // one bitcode library to link in for a specific triple. Additionally, the
     // path is *not* relative to the -sycl-device-library-location - the full
     // path must be provided.
-    SmallString<256> LibList;
     SmallVector<std::string, 4> BCLibList;
 
     auto appendToList = [](SmallString<256> &List, const Twine &Arg) {
@@ -11487,9 +11486,14 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
     for (const auto &[Kind, TC] :
          llvm::make_range(ToolChainRange.first, ToolChainRange.second)) {
       llvm::Triple TargetTriple = TC->getTriple();
-      bool IsSpirAOT = TargetTriple.isSPIRAOT();
+      // SPIR or SPIR-V device libraries are compiled into the device compile
+      // step.
+      if (TargetTriple.isSPIROrSPIRV())
+        continue;
+      const toolchains::SYCLToolChain &SYCLTC =
+          static_cast<const toolchains::SYCLToolChain &>(*TC);
       SmallVector<std::string, 8> SYCLDeviceLibs =
-          SYCL::getDeviceLibraries(C, TargetTriple, IsSpirAOT);
+          SYCLTC.getDeviceLibNames(D, Args, TargetTriple);
       for (const auto &AddLib : SYCLDeviceLibs) {
         if (llvm::sys::path::extension(AddLib) == ".bc") {
           SmallString<256> LibPath(DeviceLibDir);
@@ -11498,8 +11502,6 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
               (Twine(TC->getTriple().str()) + "=" + LibPath).str());
           continue;
         }
-
-        appendToList(LibList, AddLib);
       }
 
       if (TC->getTriple().isNVPTX())
@@ -11509,10 +11511,6 @@ void LinkerWrapper::ConstructJob(Compilation &C, const JobAction &JA,
               (Twine(TC->getTriple().str()) + "=" + LibSpirvFile).str());
     }
 
-    if (LibList.size())
-      CmdArgs.push_back(
-          Args.MakeArgString(Twine("-sycl-device-libraries=") + LibList));
-
     if (BCLibList.size())
       for (const std::string &Lib : BCLibList)
         CmdArgs.push_back(

clang/lib/Driver/ToolChains/SYCL.cpp

@@ -229,15 +229,14 @@ bool SYCL::shouldDoPerObjectFileLinking(const Compilation &C) {
 }
 
 // Return whether to use native bfloat16 library.
-static bool selectBfloatLibs(const llvm::Triple &Triple, const Compilation &C,
+static bool selectBfloatLibs(const llvm::opt::ArgList &Args,
+                             const llvm::Triple &Triple, const ToolChain &TC,
                              bool &UseNative) {
-
   static llvm::SmallSet<StringRef, 8> GPUArchsWithNBF16{
       "intel_gpu_pvc",     "intel_gpu_acm_g10", "intel_gpu_acm_g11",
       "intel_gpu_acm_g12", "intel_gpu_dg2_g10", "intel_gpu_dg2_g11",
       "intel_dg2_g12",     "intel_gpu_bmg_g21", "intel_gpu_lnl_m",
       "intel_gpu_ptl_h",   "intel_gpu_ptl_u",   "intel_gpu_wcl"};
-  const llvm::opt::ArgList &Args = C.getArgs();
   bool NeedLibs = false;
 
   // spir64 target is actually JIT compilation, so we defer selection of
@@ -246,21 +245,11 @@ static bool selectBfloatLibs(const llvm::Triple &Triple, const Compilation &C,
   NeedLibs = Triple.getSubArch() != llvm::Triple::NoSubArch &&
              !Triple.isNVPTX() && !Triple.isAMDGCN();
   UseNative = false;
-  if (NeedLibs && Triple.getSubArch() == llvm::Triple::SPIRSubArch_gen &&
-      C.hasOffloadToolChain<Action::OFK_SYCL>()) {
+  if (NeedLibs && Triple.getSubArch() == llvm::Triple::SPIRSubArch_gen) {
     ArgStringList TargArgs;
-    auto ToolChains = C.getOffloadToolChains<Action::OFK_SYCL>();
-    // Match up the toolchain with the incoming Triple so we are grabbing the
-    // expected arguments to scrutinize.
-    for (auto TI = ToolChains.first, TE = ToolChains.second; TI != TE; ++TI) {
-      llvm::Triple SYCLTriple = TI->second->getTriple();
-      if (SYCLTriple == Triple) {
-        const toolchains::SYCLToolChain *SYCLTC =
-            static_cast<const toolchains::SYCLToolChain *>(TI->second);
-        SYCLTC->TranslateBackendTargetArgs(Triple, Args, TargArgs);
-        break;
-      }
-    }
+    const toolchains::SYCLToolChain &SYCLTC =
+        static_cast<const toolchains::SYCLToolChain &>(TC);
+    SYCLTC.TranslateBackendTargetArgs(Triple, Args, TargArgs);
 
     // We need to select fallback/native bfloat16 devicelib in AOT compilation
     // targetting for Intel GPU devices. Users have 2 ways to apply AOT,
@@ -416,13 +405,11 @@ static bool checkPVCDevice(std::string SingleArg, std::string &DevArg) {
 
 #if !defined(_WIN32)
 static void
-addSYCLDeviceSanitizerLibs(const Compilation &C, bool IsSpirvAOT,
-                           StringRef LibSuffix,
+addSYCLDeviceSanitizerLibs(const llvm::opt::ArgList &Args,
                            SmallVector<std::string, 8> &LibraryList) {
-  const llvm::opt::ArgList &Args = C.getArgs();
   enum { JIT = 0, AOT_CPU, AOT_DG2, AOT_PVC };
   auto addSingleLibrary = [&](StringRef DeviceLibName) {
-    LibraryList.push_back(Args.MakeArgString(Twine(DeviceLibName) + LibSuffix));
+    LibraryList.push_back(Args.MakeArgString(Twine(DeviceLibName) + ".bc"));
   };
 
   // This function is used to check whether there is only one GPU device
@@ -445,9 +432,6 @@ addSYCLDeviceSanitizerLibs(const Compilation &C, bool IsSpirvAOT,
   };
 
   auto getSingleBuildTarget = [&]() -> size_t {
-    if (!IsSpirvAOT)
-      return JIT;
-
     llvm::opt::Arg *SYCLTarget =
         Args.getLastArg(options::OPT_offload_targets_EQ);
     if (!SYCLTarget || (SYCLTarget->getValues().size() != 1))
@@ -548,12 +532,12 @@ addSYCLDeviceSanitizerLibs(const Compilation &C, bool IsSpirvAOT,
 }
 #endif
 
-// Get the list of SYCL device libraries to link with user's device image.
+// Get the list of SYCL device library names to link with user's device image.
 SmallVector<std::string, 8>
-SYCL::getDeviceLibraries(const Compilation &C, const llvm::Triple &TargetTriple,
-                         bool IsSpirvAOT) {
+SYCLToolChain::getDeviceLibNames(const Driver &D,
+                                 const llvm::opt::ArgList &Args,
+                                 const llvm::Triple &TargetTriple) const {
   SmallVector<std::string, 8> LibraryList;
-  const llvm::opt::ArgList &Args = C.getArgs();
   bool NoOffloadLib =
       !Args.hasFlag(options::OPT_offloadlib, options::OPT_no_offloadlib, true);
   if (TargetTriple.isNVPTX()) {
@@ -597,17 +581,9 @@ SYCL::getDeviceLibraries(const Compilation &C, const llvm::Triple &TargetTriple,
                                              "libsycl-fallback-imf",
                                              "libsycl-fallback-imf-fp64",
                                              "libsycl-fallback-imf-bf16"};
-  bool IsWindowsMSVCEnv =
-      C.getDefaultToolChain().getTriple().isWindowsMSVCEnvironment();
-  bool IsNewOffload = C.getDriver().getUseNewOffloadingDriver();
-  StringRef LibSuffix = ".bc";
-  if (IsNewOffload)
-    // For new offload model, we use packaged .bc files.
-    LibSuffix = IsWindowsMSVCEnv ? ".new.obj" : ".new.o";
   auto addLibraries = [&](const SYCLDeviceLibsList &LibsList) {
-    for (const StringRef &Lib : LibsList) {
-      LibraryList.push_back(Args.MakeArgString(Twine(Lib) + LibSuffix));
-    }
+    for (const StringRef &Lib : LibsList)
+      LibraryList.push_back(Args.MakeArgString(Twine(Lib) + ".bc"));
   };
 
   if (!NoOffloadLib)
@@ -627,7 +603,8 @@ SYCL::getDeviceLibraries(const Compilation &C, const llvm::Triple &TargetTriple,
   const SYCLDeviceLibsList SYCLDeviceBfloat16NativeLib = {
       "libsycl-native-bfloat16"};
   bool NativeBfloatLibs;
-  bool NeedBfloatLibs = selectBfloatLibs(TargetTriple, C, NativeBfloatLibs);
+  bool NeedBfloatLibs =
+      selectBfloatLibs(Args, TargetTriple, *this, NativeBfloatLibs);
   if (NeedBfloatLibs && !NoOffloadLib) {
     // Add native or fallback bfloat16 library.
     if (NativeBfloatLibs)
@@ -640,7 +617,7 @@ SYCL::getDeviceLibraries(const Compilation &C, const llvm::Triple &TargetTriple,
   // Linux platform only, so compiler only provides device sanitizer libraries
   // on Linux platform.
 #if !defined(_WIN32)
-  addSYCLDeviceSanitizerLibs(C, IsSpirvAOT, LibSuffix, LibraryList);
+  addSYCLDeviceSanitizerLibs(Args, LibraryList);
 #endif
 
   return LibraryList;
@@ -1456,6 +1433,22 @@ void SYCLToolChain::addClangTargetOptions(
     SYCLInstallation.addLibspirvLinkArgs(getEffectiveTriple(), DriverArgs,
                                          HostTC.getTriple(), CC1Args);
   }
+  // mtoguchi
+  // Add device libraries.
+  // This is currently only done with the new offloading model, to better fit
+  // with community expectations
+  if (!getDriver().getUseNewOffloadingDriver() || !getTriple().isSPIROrSPIRV())
+    return;
+
+  llvm::SmallVector<BitCodeLibraryInfo, 12> BCLibs;
+  BCLibs.append(SYCLToolChain::getDeviceLibs(DriverArgs, DeviceOffloadingKind));
+  for (auto BCFile : BCLibs) {
+    CC1Args.push_back("-mlink-builtin-bitcode");
+    CC1Args.push_back(DriverArgs.MakeArgString(BCFile.Path));
+  }
+  // FIXME: Turn off potential linker warnings when linking in device library
+  // files that are built for spir64, but we are compiling for AOT.
+  CC1Args.push_back("-Wno-linker-warnings");
 }
 
 llvm::opt::DerivedArgList *
@@ -1892,6 +1885,34 @@ void SYCLToolChain::AddClangCXXStdlibIncludeArgs(const ArgList &Args,
   HostTC.AddClangCXXStdlibIncludeArgs(Args, CC1Args);
 }
 
+llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12>
+SYCLToolChain::getDeviceLibs(
+    const llvm::opt::ArgList &DriverArgs,
+    const Action::OffloadKind DeviceOffloadingKind) const {
+  llvm::SmallVector<ToolChain::BitCodeLibraryInfo, 12> BCLibs;
+
+  SmallVector<SmallString<128>, 4> LibraryPaths;
+  SYCLInstallation.getSYCLDeviceLibPath(LibraryPaths);
+
+  // Formulate all of the device libraries needed for this compilation.
+  SmallVector<std::string, 8> DeviceLibs =
+      getDeviceLibNames(getDriver(), DriverArgs, getTriple());
+
+  // Create full path names to each device library.  If found, add to the list
+  // of device libraries that will be linked against.
+  for (auto DeviceLib : DeviceLibs) {
+    for (auto LibraryPath : LibraryPaths) {
+      SmallString<1208> FullLibName(LibraryPath);
+      llvm::sys::path::append(FullLibName, DeviceLib);
+      if (llvm::sys::fs::exists(FullLibName)) {
+        BCLibs.emplace_back(FullLibName);
+        break;
+      }
+    }
+  }
+  return BCLibs;
+}
+
 SanitizerMask SYCLToolChain::getSupportedSanitizers() const {
   return SanitizerKind::Address | SanitizerKind::Memory | SanitizerKind::Thread;
 }

clang/lib/Driver/ToolChains/SYCL.h

@@ -33,12 +33,6 @@ void constructLLVMForeachCommand(Compilation &C, const JobAction &JA,
                                  StringRef Increment, StringRef Ext = "out",
                                  StringRef ParallelJobs = "");
 
-// Provides a vector of device library names that are associated with the
-// given triple and AOT information.
-SmallVector<std::string, 8> getDeviceLibraries(const Compilation &C,
-                                               const llvm::Triple &TargetTriple,
-                                               bool IsSpirvAOT);
-
 // Populates the SYCL device traits macros.
 void populateSYCLDeviceTraitsMacrosArgs(
     Compilation &C, const llvm::opt::ArgList &Args,
@@ -160,6 +154,11 @@ class LLVM_LIBRARY_VISIBILITY SYCLToolChain : public ToolChain {
                           llvm::opt::OptSpecifier Opt,
                           llvm::opt::OptSpecifier Opt_EQ,
                           StringRef Device) const;
+  // Provides a vector of device library names that are associated with the
+  // given triple and AOT information.
+  SmallVector<std::string, 8>
+  getDeviceLibNames(const Driver &D, const llvm::opt::ArgList &Args,
+                    const llvm::Triple &TargetTriple) const;
 
   bool useIntegratedAs() const override { return true; }
   bool isPICDefault() const override {
@@ -189,6 +188,10 @@ class LLVM_LIBRARY_VISIBILITY SYCLToolChain : public ToolChain {
       const llvm::opt::ArgList &Args,
       llvm::opt::ArgStringList &CC1Args) const override;
 
+  llvm::SmallVector<BitCodeLibraryInfo, 12>
+  getDeviceLibs(const llvm::opt::ArgList &Args,
+                const Action::OffloadKind DeviceOffloadingKind) const override;
+
   SanitizerMask getSupportedSanitizers() const override;
 
 protected: