[SYCLBIN] Draft for moving to new offload binary format

clang/include/clang/Options/Options.td

@@ -7682,7 +7682,7 @@ def fsycl_dump_device_code_EQ : Joined<["-"], "fsycl-dump-device-code=">,
   Flags<[NoXarchOption, Deprecated]>,
   HelpText<"Dump device code into the user provided directory. (deprecated)">;
 def fsyclbin_EQ : Joined<["-"], "fsyclbin=">, Values<"executable,object,input">,
-  HelpText<"Output in the SYCLBIN binary format in the state specified by <arg> (input, object or executable (default))">;
+  HelpText<"Output the SYCL binary in the state specified by <arg> (input, object or executable (default))">;
 def fsyclbin : Flag<["-"], "fsyclbin">, Alias<fsyclbin_EQ>,
   AliasArgs<["executable"]>;
 } // let Group = sycl_Group

clang/test/Driver/linker-wrapper-image.c

@@ -25,7 +25,7 @@
 // OPENMP-REL: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}", section ".llvm.offloading.relocatable", align 8
 
 //      OPENMP: @.omp_offloading.device_image = internal unnamed_addr constant [[[SIZE:[0-9]+]] x i8] c"\10\FF\10\AD{{.*}}", section ".llvm.offloading", align 8
-// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr ([[[BEGIN:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr getelementptr ([[[END:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 144), ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries }]
+// OPENMP-NEXT: @.omp_offloading.device_images = internal unnamed_addr constant [1 x %__tgt_device_image] [%__tgt_device_image { ptr getelementptr ([[[IMG_OFF:[0-9]+]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 [[IMG_OFF]]), ptr getelementptr ([[[IMG_OFF]] x i8], ptr @.omp_offloading.device_image, i64 0, i64 [[IMG_OFF]]), ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries }]
 // OPENMP-NEXT: @.omp_offloading.descriptor = internal constant %__tgt_bin_desc { i32 1, ptr @.omp_offloading.device_images, ptr @__start_llvm_offload_entries, ptr @__stop_llvm_offload_entries }
 // OPENMP-NEXT: @llvm.global_ctors = appending global [1 x { i32, ptr, ptr }] [{ i32, ptr, ptr } { i32 101, ptr @.omp_offloading.descriptor_reg, ptr null }]
 

clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp

@@ -260,7 +260,7 @@ Expected<OffloadFile> getInputBitcodeLibrary(StringRef Input) {
   auto NewBinaryOrErr = OffloadBinary::create(*Binary);
   if (!NewBinaryOrErr)
     return NewBinaryOrErr.takeError();
-  return OffloadFile(std::move(*NewBinaryOrErr), std::move(Binary));
+  return OffloadFile(std::move(NewBinaryOrErr.get()[0]), std::move(Binary));
 }
 
 std::string getMainExecutable(const char *Name) {
@@ -1228,36 +1228,22 @@ static Expected<StringRef>
 packageSYCLBIN(SYCLBIN::BundleState State,
                const ArrayRef<SYCLBIN::SYCLBINModuleDesc> Modules) {
   SYCLBIN::SYCLBINDesc SYCLBIND{State, Modules};
-  size_t SYCLBINByteSize = 0;
-  if (Error E = SYCLBIND.getSYCLBINByteSize().moveInto(SYCLBINByteSize))
+  SmallString<0> Binary;
+  raw_svector_ostream BinaryOS{Binary};
+  if (Error E = SYCLBIN::write(SYCLBIND, BinaryOS))
     return std::move(E);
 
-  SmallString<0> SYCLBINImage;
-  SYCLBINImage.reserve(SYCLBINByteSize);
-  raw_svector_ostream SYCLBINImageOS{SYCLBINImage};
-  if (Error E = SYCLBIN::write(SYCLBIND, SYCLBINImageOS))
-    return std::move(E);
-
-  OffloadingImage Image{};
-  Image.TheImageKind = IMG_SYCLBIN;
-  Image.TheOffloadKind = OFK_SYCL;
-  Image.Image = MemoryBuffer::getMemBuffer(SYCLBINImage, /*BufferName=*/"",
-                                           /*RequiresNullTerminator=*/false);
-
-  std::unique_ptr<MemoryBuffer> Binary = MemoryBuffer::getMemBufferCopy(
-      OffloadBinary::write(Image), Image.Image->getBufferIdentifier());
-
   auto OutFileOrErr =
       createOutputFile(sys::path::filename(ExecutableName), "syclbin");
   if (!OutFileOrErr)
     return OutFileOrErr.takeError();
 
   Expected<std::unique_ptr<FileOutputBuffer>> OutputOrErr =
-      FileOutputBuffer::create(*OutFileOrErr, Binary->getBufferSize());
+      FileOutputBuffer::create(*OutFileOrErr, Binary.size());
   if (!OutputOrErr)
     return OutputOrErr.takeError();
   std::unique_ptr<FileOutputBuffer> Output = std::move(*OutputOrErr);
-  llvm::copy(Binary->getBuffer(), Output->getBufferStart());
+  llvm::copy(Binary, Output->getBufferStart());
   if (Error E = Output->commit())
     return std::move(E);
 
@@ -1278,6 +1264,7 @@ Error copyFileToFinalExecutable(StringRef File, const ArgList &Args) {
   return Error::success();
 }
 
+// if we use OffloadBinary instead of SYCLBIN, do we get merging for free???
 Error mergeSYCLBIN(ArrayRef<StringRef> Files, const ArgList &Args) {
   // Fast path for the general case where there's only one file. In this case we
   // do not need to parse it and can instead simply copy it.

clang/tools/clang-offload-wrapper/ClangOffloadWrapper.cpp

@@ -85,7 +85,10 @@ static constexpr char COL_PROPS[] = "Properties";
 // Offload models supported by this tool. The support basically means mapping
 // a string representation given at the command line to a value from this
 // enum.
-enum OffloadKind {
+// OffloadBinary.h has different OffloadKind.
+// We keep COW_OffloadKind for old offloading model.
+// New offloading model should use OffloadKind from OffloadBinary.h.
+enum COW_OffloadKind {
   Unknown = 0,
   Host,
   OpenMP,
@@ -96,20 +99,21 @@ enum OffloadKind {
 };
 
 namespace llvm {
-template <> struct DenseMapInfo<OffloadKind> {
-  static inline OffloadKind getEmptyKey() {
-    return static_cast<OffloadKind>(DenseMapInfo<unsigned>::getEmptyKey());
+template <> struct DenseMapInfo<COW_OffloadKind> {
+  static inline COW_OffloadKind getEmptyKey() {
+    return static_cast<COW_OffloadKind>(DenseMapInfo<unsigned>::getEmptyKey());
   }
 
-  static inline OffloadKind getTombstoneKey() {
-    return static_cast<OffloadKind>(DenseMapInfo<unsigned>::getTombstoneKey());
+  static inline COW_OffloadKind getTombstoneKey() {
+    return static_cast<COW_OffloadKind>(
+        DenseMapInfo<unsigned>::getTombstoneKey());
   }
 
-  static unsigned getHashValue(const OffloadKind &Val) {
+  static unsigned getHashValue(const COW_OffloadKind &Val) {
     return DenseMapInfo<unsigned>::getHashValue(static_cast<unsigned>(Val));
   }
 
-  static bool isEqual(const OffloadKind &LHS, const OffloadKind &RHS) {
+  static bool isEqual(const COW_OffloadKind &LHS, const COW_OffloadKind &RHS) {
     return LHS == RHS;
   }
 };
@@ -178,7 +182,7 @@ enum BinaryImageFormat {
 };
 
 /// Sets offload kind.
-static cl::list<OffloadKind> Kinds(
+static cl::list<COW_OffloadKind> Kinds(
     "kind", cl::desc("offload kind:"), cl::OneOrMore,
     cl::values(clEnumValN(Unknown, "unknown", "unknown"),
                clEnumValN(Host, "host", "host"),
@@ -264,17 +268,17 @@ static cl::opt<bool> BatchMode(
     cl::cat(ClangOffloadWrapperCategory));
 // clang-format on
 
-static StringRef offloadKindToString(OffloadKind Kind) {
+static StringRef offloadKindToString(COW_OffloadKind Kind) {
   switch (Kind) {
-  case OffloadKind::Unknown:
+  case COW_OffloadKind::Unknown:
     return "unknown";
-  case OffloadKind::Host:
+  case COW_OffloadKind::Host:
     return "host";
-  case OffloadKind::OpenMP:
+  case COW_OffloadKind::OpenMP:
     return "openmp";
-  case OffloadKind::HIP:
+  case COW_OffloadKind::HIP:
     return "hip";
-  case OffloadKind::SYCL:
+  case COW_OffloadKind::SYCL:
     return "sycl";
   }
   llvm_unreachable("bad offload kind");
@@ -365,12 +369,12 @@ class BinaryWrapper {
   PointerType *PtrTy = nullptr;
 
   /// Records all added device binary images per offload kind.
-  llvm::DenseMap<OffloadKind, std::unique_ptr<SameKindPack>> Packs;
+  llvm::DenseMap<COW_OffloadKind, std::unique_ptr<SameKindPack>> Packs;
   /// Records all created memory buffers for safe auto-gc
   llvm::SmallVector<std::unique_ptr<MemoryBuffer>, 4> AutoGcBufs;
 
 public:
-  void addImage(const OffloadKind Kind, llvm::StringRef File,
+  void addImage(const COW_OffloadKind Kind, llvm::StringRef File,
                 llvm::StringRef Tgt, const BinaryImageFormat Fmt,
                 llvm::StringRef CompileOpts, llvm::StringRef LinkOpts,
                 llvm::StringRef EntriesFile, llvm::StringRef PropsFile) {
@@ -532,7 +536,7 @@ class BinaryWrapper {
   //    /// should increment the version.
   //    uint16_t Version;
   //    /// the kind of offload model the image employs.
-  //    uint8_t OffloadKind;
+  //    uint8_t COW_OffloadKind;
   //    /// format of the image data - SPIRV, LLVMIR bitcode,...
   //    uint8_t Format;
   //    /// null-terminated string representation of the device's target
@@ -561,7 +565,7 @@ class BinaryWrapper {
       SyclImageTy = StructType::create(
           {
               Type::getInt16Ty(C), // Version
-              Type::getInt8Ty(C),  // OffloadKind
+              Type::getInt8Ty(C),  // COW_OffloadKind
               Type::getInt8Ty(C),  // Format
               getPtrTy(),          // DeviceTargetSpec
               getPtrTy(),          // CompileOptions
@@ -730,7 +734,7 @@ class BinaryWrapper {
   // contains the image data.
   std::pair<Constant *, Constant *>
   addDeviceImageToModule(ArrayRef<char> Buf, const Twine &Name,
-                         OffloadKind Kind, StringRef TargetTriple) {
+                         COW_OffloadKind Kind, StringRef TargetTriple) {
     // Create global variable for the image data.
     return addArrayToModule(Buf, Name,
                             TargetTriple.empty()
@@ -969,14 +973,14 @@ class BinaryWrapper {
   /// };
   ///
   /// Global variable that represents BinDesc is returned.
-  Expected<GlobalVariable *> createBinDesc(OffloadKind Kind,
+  Expected<GlobalVariable *> createBinDesc(COW_OffloadKind Kind,
                                            SameKindPack &Pack) {
     const std::string OffloadKindTag =
         (Twine(".") + offloadKindToString(Kind) + Twine("_offloading.")).str();
 
     Constant *EntriesB = nullptr, *EntriesE = nullptr;
 
-    if (Kind != OffloadKind::SYCL) {
+    if (Kind != COW_OffloadKind::SYCL) {
       // Create external begin/end symbols for the offload entries table.
       auto *EntriesStart = new GlobalVariable(
           M, getEntryTy(), /*isConstant*/ true, GlobalValue::ExternalLinkage,
@@ -1054,7 +1058,7 @@ class BinaryWrapper {
       if (!BinOrErr)
         return BinOrErr.takeError();
       MemoryBuffer *Bin = *BinOrErr;
-      if (Img.File != "-" && Kind == OffloadKind::OpenMP &&
+      if (Img.File != "-" && Kind == COW_OffloadKind::OpenMP &&
           AddOpenMPOffloadNotes) {
         // Adding ELF notes for STDIN is not supported yet.
         Bin = addELFNotes(Bin, Img.File);
@@ -1084,7 +1088,7 @@ class BinaryWrapper {
         // Don't compress if the user explicitly specifies the binary image
         // format or if the image is smaller than OffloadCompressThreshold
         // bytes.
-        if (Kind != OffloadKind::SYCL || !OffloadCompressDevImgs ||
+        if (Kind != COW_OffloadKind::SYCL || !OffloadCompressDevImgs ||
             Img.Fmt != BinaryImageFormat::none ||
             !llvm::compression::zstd::isAvailable() ||
             static_cast<int>(Bin->getBufferSize()) < OffloadCompressThreshold) {
@@ -1132,7 +1136,7 @@ class BinaryWrapper {
         }
       }
 
-      if (Kind == OffloadKind::SYCL) {
+      if (Kind == COW_OffloadKind::SYCL) {
         // For SYCL image offload entries are defined here, by wrapper, so
         // those are created per image
         Expected<std::pair<Constant *, Constant *>> EntriesOrErr =
@@ -1178,7 +1182,7 @@ class BinaryWrapper {
 
     // Then create images array.
     auto *ImagesData =
-        Kind == OffloadKind::SYCL
+        Kind == COW_OffloadKind::SYCL
             ? ConstantArray::get(
                   ArrayType::get(getSyclDeviceImageTy(), ImagesInits.size()),
                   ImagesInits)
@@ -1199,7 +1203,7 @@ class BinaryWrapper {
 
     // And finally create the binary descriptor object.
     auto *DescInit =
-        Kind == OffloadKind::SYCL
+        Kind == COW_OffloadKind::SYCL
             ? ConstantStruct::get(
                   getSyclBinDescTy(),
                   ConstantInt::get(Type::getInt16Ty(C), BinDescStructVersion),
@@ -1221,7 +1225,7 @@ class BinaryWrapper {
     return Res;
   }
 
-  void createRegisterFunction(OffloadKind Kind, GlobalVariable *BinDesc) {
+  void createRegisterFunction(COW_OffloadKind Kind, GlobalVariable *BinDesc) {
     auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
     auto *Func =
         Function::Create(FuncTy, GlobalValue::InternalLinkage,
@@ -1231,10 +1235,10 @@ class BinaryWrapper {
     // Get RegFuncName function declaration.
     auto *RegFuncTy = FunctionType::get(Type::getVoidTy(C), getPtrTy(),
                                         /*isVarArg=*/false);
-    FunctionCallee RegFuncC =
-        M.getOrInsertFunction(Kind == OffloadKind::SYCL ? "__sycl_register_lib"
-                                                        : "__tgt_register_lib",
-                              RegFuncTy);
+    FunctionCallee RegFuncC = M.getOrInsertFunction(
+        Kind == COW_OffloadKind::SYCL ? "__sycl_register_lib"
+                                      : "__tgt_register_lib",
+        RegFuncTy);
 
     // Construct function body
     IRBuilder<> Builder(BasicBlock::Create(C, "entry", Func));
@@ -1250,7 +1254,7 @@ class BinaryWrapper {
     appendToGlobalCtors(M, Func, /*Priority*/ 1);
   }
 
-  void createUnregisterFunction(OffloadKind Kind, GlobalVariable *BinDesc) {
+  void createUnregisterFunction(COW_OffloadKind Kind, GlobalVariable *BinDesc) {
     auto *FuncTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg*/ false);
     auto *Func =
         Function::Create(FuncTy, GlobalValue::InternalLinkage,
@@ -1261,8 +1265,8 @@ class BinaryWrapper {
     auto *UnRegFuncTy = FunctionType::get(Type::getVoidTy(C), getPtrTy(),
                                           /*isVarArg=*/false);
     FunctionCallee UnRegFuncC = M.getOrInsertFunction(
-        Kind == OffloadKind::SYCL ? "__sycl_unregister_lib"
-                                  : "__tgt_unregister_lib",
+        Kind == COW_OffloadKind::SYCL ? "__sycl_unregister_lib"
+                                      : "__tgt_unregister_lib",
         UnRegFuncTy);
 
     // Construct function body
@@ -1386,15 +1390,15 @@ class BinaryWrapper {
 
   Expected<const Module *> wrap() {
     for (auto &X : Packs) {
-      OffloadKind Kind = X.first;
+      COW_OffloadKind Kind = X.first;
       SameKindPack *Pack = X.second.get();
       Expected<GlobalVariable *> DescOrErr = createBinDesc(Kind, *Pack);
       if (!DescOrErr)
         return DescOrErr.takeError();
 
       if (EmitRegFuncs) {
         GlobalVariable *Desc = *DescOrErr;
-        if (Kind == OffloadKind::SYCL &&
+        if (Kind == COW_OffloadKind::SYCL &&
             Triple(M.getTargetTriple()).isOSWindows()) {
           createSyclRegisterWithAtexitUnregister(Desc);
         } else {
@@ -1849,7 +1853,7 @@ int main(int argc, const char **argv) {
   // add them to the wrapper
 
   BinaryWrapper Wr(Target, argv[0], SymPropBCFiles);
-  OffloadKind Knd = OffloadKind::Unknown;
+  COW_OffloadKind Knd = COW_OffloadKind::Unknown;
   llvm::StringRef Tgt = "";
   BinaryImageFormat Fmt = BinaryImageFormat::none;
   llvm::StringRef CompileOpts = "";
@@ -1899,7 +1903,7 @@ int main(int argc, const char **argv) {
                         Row.getCell(COL_PROPS, ""));
           }
         } else {
-          if (Knd == OffloadKind::Unknown) {
+          if (Knd == COW_OffloadKind::Unknown) {
             reportError(createStringError(errc::invalid_argument,
                                           "offload model not set"));
             return 1;

clang/tools/clang-sycl-linker/ClangSYCLLinker.cpp

@@ -496,12 +496,12 @@ Error runSYCLLink(ArrayRef<std::string> Files, const ArgList &Args) {
     if (!ModOrErr)
       return ModOrErr.takeError();
 
-    SmallVector<StringRef> Symbols;
+    std::string SymbolData;
     for (Function &F : **ModOrErr) {
       if (isKernel(F))
-        Symbols.push_back(F.getName());
+        SymbolData.append(F.getName().data(), F.getName().size() + 1);
     }
-    SymbolTable.emplace_back(llvm::join(Symbols.begin(), Symbols.end(), "\n"));
+    SymbolTable.emplace_back(std::move(SymbolData));
   }
 
   bool IsAOTCompileNeeded = IsIntelOffloadArch(
@@ -541,13 +541,7 @@ Error runSYCLLink(ArrayRef<std::string> Files, const ArgList &Args) {
         return createFileError(File, EC);
     }
     OffloadingImage TheImage{};
-    // TODO: TheImageKind should be
-    // `IsAOTCompileNeeded ? IMG_Object : IMG_SPIRV;`
-    // For that we need to update SYCL Runtime to align with the ImageKind enum.
-    // Temporarily it is initalized to IMG_None, because in that case, SYCL
-    // Runtime has a heuristic to understand what the Image Kind is, so at least
-    // it works.
-    TheImage.TheImageKind = IMG_None;
+    TheImage.TheImageKind = IsAOTCompileNeeded ? IMG_Object : IMG_SPIRV;
     TheImage.TheOffloadKind = OFK_SYCL;
     TheImage.StringData["triple"] =
         Args.MakeArgString(Args.getLastArgValue(OPT_triple_EQ));