diff options
author | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-06-29 10:00:50 +0300 |
---|---|---|
committer | vitalyisaev <vitalyisaev@yandex-team.com> | 2023-06-29 10:00:50 +0300 |
commit | 6ffe9e53658409f212834330e13564e4952558f6 (patch) | |
tree | 85b1e00183517648b228aafa7c8fb07f5276f419 /contrib/libs/clang14/tools/clang-offload-bundler/ClangOffloadBundler.cpp | |
parent | 726057070f9c5a91fc10fde0d5024913d10f1ab9 (diff) | |
download | ydb-6ffe9e53658409f212834330e13564e4952558f6.tar.gz |
YQ Connector: support managed ClickHouse
Со стороны dqrun можно обратиться к инстансу коннектора, который работает на streaming стенде, и извлечь данные из облачного CH.
Diffstat (limited to 'contrib/libs/clang14/tools/clang-offload-bundler/ClangOffloadBundler.cpp')
-rw-r--r-- | contrib/libs/clang14/tools/clang-offload-bundler/ClangOffloadBundler.cpp | 1492 |
1 files changed, 1492 insertions, 0 deletions
diff --git a/contrib/libs/clang14/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/contrib/libs/clang14/tools/clang-offload-bundler/ClangOffloadBundler.cpp new file mode 100644 index 0000000000..ce67362886 --- /dev/null +++ b/contrib/libs/clang14/tools/clang-offload-bundler/ClangOffloadBundler.cpp @@ -0,0 +1,1492 @@ +//===-- clang-offload-bundler/ClangOffloadBundler.cpp ---------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements a clang-offload-bundler that bundles different +/// files that relate with the same source code but different targets into a +/// single one. Also the implements the opposite functionality, i.e. unbundle +/// files previous created by this tool. +/// +//===----------------------------------------------------------------------===// + +#include "clang/Basic/Cuda.h" +#include "clang/Basic/Version.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/ADT/Triple.h" +#include "llvm/Object/Archive.h" +#include "llvm/Object/ArchiveWriter.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorOr.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Program.h" +#include "llvm/Support/Signals.h" +#include "llvm/Support/StringSaver.h" +#include "llvm/Support/WithColor.h" +#include "llvm/Support/raw_ostream.h" +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <forward_list> +#include <memory> +#include <set> +#include <string> +#include <system_error> +#include <utility> + +using namespace llvm; +using namespace llvm::object; + +static cl::opt<bool> Help("h", cl::desc("Alias for -help"), cl::Hidden); + +// Mark all our options with this category, everything else (except for -version +// and -help) will be hidden. +static cl::OptionCategory + ClangOffloadBundlerCategory("clang-offload-bundler options"); + +static cl::list<std::string> + InputFileNames("inputs", cl::CommaSeparated, cl::OneOrMore, + cl::desc("[<input file>,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list<std::string> + OutputFileNames("outputs", cl::CommaSeparated, + cl::desc("[<output file>,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::list<std::string> + TargetNames("targets", cl::CommaSeparated, + cl::desc("[<offload kind>-<target triple>,...]"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt<std::string> + FilesType("type", cl::Required, + cl::desc("Type of the files to be bundled/unbundled.\n" + "Current supported types are:\n" + " i - cpp-output\n" + " ii - c++-cpp-output\n" + " cui - cuda/hip-output\n" + " d - dependency\n" + " ll - llvm\n" + " bc - llvm-bc\n" + " s - assembler\n" + " o - object\n" + " a - archive of objects\n" + " gch - precompiled-header\n" + " ast - clang AST file"), + cl::cat(ClangOffloadBundlerCategory)); +static cl::opt<bool> + Unbundle("unbundle", + cl::desc("Unbundle bundled file into several output files.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +static cl::opt<bool> + ListBundleIDs("list", cl::desc("List bundle IDs in the bundled file.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +static cl::opt<bool> PrintExternalCommands( + "###", + cl::desc("Print any external commands that are to be executed " + "instead of actually executing them - for testing purposes.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +static cl::opt<bool> + AllowMissingBundles("allow-missing-bundles", + cl::desc("Create empty files if bundles are missing " + "when unbundling.\n"), + cl::init(false), cl::cat(ClangOffloadBundlerCategory)); + +static cl::opt<unsigned> + BundleAlignment("bundle-align", + cl::desc("Alignment of bundle for binary files"), + cl::init(1), cl::cat(ClangOffloadBundlerCategory)); + +/// Magic string that marks the existence of offloading data. +#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__" + +/// The index of the host input in the list of inputs. +static unsigned HostInputIndex = ~0u; + +/// Whether not having host target is allowed. +static bool AllowNoHost = false; + +/// Path to the current binary. +static std::string BundlerExecutable; + +/// Obtain the offload kind, real machine triple, and an optional GPUArch +/// out of the target information specified by the user. +/// Bundle Entry ID (or, Offload Target String) has following components: +/// * Offload Kind - Host, OpenMP, or HIP +/// * Triple - Standard LLVM Triple +/// * GPUArch (Optional) - Processor name, like gfx906 or sm_30 + +struct OffloadTargetInfo { + StringRef OffloadKind; + llvm::Triple Triple; + StringRef GPUArch; + + OffloadTargetInfo(const StringRef Target) { + auto TargetFeatures = Target.split(':'); + auto TripleOrGPU = TargetFeatures.first.rsplit('-'); + + if (clang::StringToCudaArch(TripleOrGPU.second) != + clang::CudaArch::UNKNOWN) { + auto KindTriple = TripleOrGPU.first.split('-'); + this->OffloadKind = KindTriple.first; + this->Triple = llvm::Triple(KindTriple.second); + this->GPUArch = Target.substr(Target.find(TripleOrGPU.second)); + } else { + auto KindTriple = TargetFeatures.first.split('-'); + this->OffloadKind = KindTriple.first; + this->Triple = llvm::Triple(KindTriple.second); + this->GPUArch = ""; + } + } + + bool hasHostKind() const { return this->OffloadKind == "host"; } + + bool isOffloadKindValid() const { + return OffloadKind == "host" || OffloadKind == "openmp" || + OffloadKind == "hip" || OffloadKind == "hipv4"; + } + + bool isTripleValid() const { + return !Triple.str().empty() && Triple.getArch() != Triple::UnknownArch; + } + + bool operator==(const OffloadTargetInfo &Target) const { + return OffloadKind == Target.OffloadKind && + Triple.isCompatibleWith(Target.Triple) && GPUArch == Target.GPUArch; + } + + std::string str() { + return Twine(OffloadKind + "-" + Triple.str() + "-" + GPUArch).str(); + } +}; + +static StringRef getDeviceFileExtension(StringRef Device, + StringRef BundleFileName) { + if (Device.contains("gfx")) + return ".bc"; + if (Device.contains("sm_")) + return ".cubin"; + return sys::path::extension(BundleFileName); +} + +static std::string getDeviceLibraryFileName(StringRef BundleFileName, + StringRef Device) { + StringRef LibName = sys::path::stem(BundleFileName); + StringRef Extension = getDeviceFileExtension(Device, BundleFileName); + + std::string Result; + Result += LibName; + Result += Extension; + return Result; +} + +/// Generic file handler interface. +class FileHandler { +public: + struct BundleInfo { + StringRef BundleID; + }; + + FileHandler() {} + + virtual ~FileHandler() {} + + /// Update the file handler with information from the header of the bundled + /// file. + virtual Error ReadHeader(MemoryBuffer &Input) = 0; + + /// Read the marker of the next bundled to be read in the file. The bundle + /// name is returned if there is one in the file, or `None` if there are no + /// more bundles to be read. + virtual Expected<Optional<StringRef>> + ReadBundleStart(MemoryBuffer &Input) = 0; + + /// Read the marker that closes the current bundle. + virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0; + + /// Read the current bundle and write the result into the stream \a OS. + virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0; + + /// Write the header of the bundled file to \a OS based on the information + /// gathered from \a Inputs. + virtual Error WriteHeader(raw_fd_ostream &OS, + ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) = 0; + + /// Write the marker that initiates a bundle for the triple \a TargetTriple to + /// \a OS. + virtual Error WriteBundleStart(raw_fd_ostream &OS, + StringRef TargetTriple) = 0; + + /// Write the marker that closes a bundle for the triple \a TargetTriple to \a + /// OS. + virtual Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0; + + /// Write the bundle from \a Input into \a OS. + virtual Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0; + + /// List bundle IDs in \a Input. + virtual Error listBundleIDs(MemoryBuffer &Input) { + if (Error Err = ReadHeader(Input)) + return Err; + + return forEachBundle(Input, [&](const BundleInfo &Info) -> Error { + llvm::outs() << Info.BundleID << '\n'; + Error Err = listBundleIDsCallback(Input, Info); + if (Err) + return Err; + return Error::success(); + }); + } + + /// For each bundle in \a Input, do \a Func. + Error forEachBundle(MemoryBuffer &Input, + std::function<Error(const BundleInfo &)> Func) { + while (true) { + Expected<Optional<StringRef>> CurTripleOrErr = ReadBundleStart(Input); + if (!CurTripleOrErr) + return CurTripleOrErr.takeError(); + + // No more bundles. + if (!*CurTripleOrErr) + break; + + StringRef CurTriple = **CurTripleOrErr; + assert(!CurTriple.empty()); + + BundleInfo Info{CurTriple}; + if (Error Err = Func(Info)) + return Err; + } + return Error::success(); + } + +protected: + virtual Error listBundleIDsCallback(MemoryBuffer &Input, + const BundleInfo &Info) { + return Error::success(); + } +}; + +/// Handler for binary files. The bundled file will have the following format +/// (all integers are stored in little-endian format): +/// +/// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string) +/// +/// NumberOfOffloadBundles (8-byte integer) +/// +/// OffsetOfBundle1 (8-byte integer) +/// SizeOfBundle1 (8-byte integer) +/// NumberOfBytesInTripleOfBundle1 (8-byte integer) +/// TripleOfBundle1 (byte length defined before) +/// +/// ... +/// +/// OffsetOfBundleN (8-byte integer) +/// SizeOfBundleN (8-byte integer) +/// NumberOfBytesInTripleOfBundleN (8-byte integer) +/// TripleOfBundleN (byte length defined before) +/// +/// Bundle1 +/// ... +/// BundleN + +/// Read 8-byte integers from a buffer in little-endian format. +static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) { + uint64_t Res = 0; + const char *Data = Buffer.data(); + + for (unsigned i = 0; i < 8; ++i) { + Res <<= 8; + uint64_t Char = (uint64_t)Data[pos + 7 - i]; + Res |= 0xffu & Char; + } + return Res; +} + +/// Write 8-byte integers to a buffer in little-endian format. +static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) { + for (unsigned i = 0; i < 8; ++i) { + char Char = (char)(Val & 0xffu); + OS.write(&Char, 1); + Val >>= 8; + } +} + +class BinaryFileHandler final : public FileHandler { + /// Information about the bundles extracted from the header. + struct BinaryBundleInfo final : public BundleInfo { + /// Size of the bundle. + uint64_t Size = 0u; + /// Offset at which the bundle starts in the bundled file. + uint64_t Offset = 0u; + + BinaryBundleInfo() {} + BinaryBundleInfo(uint64_t Size, uint64_t Offset) + : Size(Size), Offset(Offset) {} + }; + + /// Map between a triple and the corresponding bundle information. + StringMap<BinaryBundleInfo> BundlesInfo; + + /// Iterator for the bundle information that is being read. + StringMap<BinaryBundleInfo>::iterator CurBundleInfo; + StringMap<BinaryBundleInfo>::iterator NextBundleInfo; + + /// Current bundle target to be written. + std::string CurWriteBundleTarget; + +public: + BinaryFileHandler() {} + + ~BinaryFileHandler() final {} + + Error ReadHeader(MemoryBuffer &Input) final { + StringRef FC = Input.getBuffer(); + + // Initialize the current bundle with the end of the container. + CurBundleInfo = BundlesInfo.end(); + + // Check if buffer is smaller than magic string. + size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1; + if (ReadChars > FC.size()) + return Error::success(); + + // Check if no magic was found. + StringRef Magic(FC.data(), sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1); + if (!Magic.equals(OFFLOAD_BUNDLER_MAGIC_STR)) + return Error::success(); + + // Read number of bundles. + if (ReadChars + 8 > FC.size()) + return Error::success(); + + uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read bundle offsets, sizes and triples. + for (uint64_t i = 0; i < NumberOfBundles; ++i) { + + // Read offset. + if (ReadChars + 8 > FC.size()) + return Error::success(); + + uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read size. + if (ReadChars + 8 > FC.size()) + return Error::success(); + + uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read triple size. + if (ReadChars + 8 > FC.size()) + return Error::success(); + + uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars); + ReadChars += 8; + + // Read triple. + if (ReadChars + TripleSize > FC.size()) + return Error::success(); + + StringRef Triple(&FC.data()[ReadChars], TripleSize); + ReadChars += TripleSize; + + // Check if the offset and size make sense. + if (!Offset || Offset + Size > FC.size()) + return Error::success(); + + assert(BundlesInfo.find(Triple) == BundlesInfo.end() && + "Triple is duplicated??"); + BundlesInfo[Triple] = BinaryBundleInfo(Size, Offset); + } + // Set the iterator to where we will start to read. + CurBundleInfo = BundlesInfo.end(); + NextBundleInfo = BundlesInfo.begin(); + return Error::success(); + } + + Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final { + if (NextBundleInfo == BundlesInfo.end()) + return None; + CurBundleInfo = NextBundleInfo++; + return CurBundleInfo->first(); + } + + Error ReadBundleEnd(MemoryBuffer &Input) final { + assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!"); + return Error::success(); + } + + Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final { + assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!"); + StringRef FC = Input.getBuffer(); + OS.write(FC.data() + CurBundleInfo->second.Offset, + CurBundleInfo->second.Size); + return Error::success(); + } + + Error WriteHeader(raw_fd_ostream &OS, + ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final { + // Compute size of the header. + uint64_t HeaderSize = 0; + + HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1; + HeaderSize += 8; // Number of Bundles + + for (auto &T : TargetNames) { + HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple. + HeaderSize += T.size(); // The triple. + } + + // Write to the buffer the header. + OS << OFFLOAD_BUNDLER_MAGIC_STR; + + Write8byteIntegerToBuffer(OS, TargetNames.size()); + + unsigned Idx = 0; + for (auto &T : TargetNames) { + MemoryBuffer &MB = *Inputs[Idx++]; + HeaderSize = alignTo(HeaderSize, BundleAlignment); + // Bundle offset. + Write8byteIntegerToBuffer(OS, HeaderSize); + // Size of the bundle (adds to the next bundle's offset) + Write8byteIntegerToBuffer(OS, MB.getBufferSize()); + BundlesInfo[T] = BinaryBundleInfo(MB.getBufferSize(), HeaderSize); + HeaderSize += MB.getBufferSize(); + // Size of the triple + Write8byteIntegerToBuffer(OS, T.size()); + // Triple + OS << T; + } + return Error::success(); + } + + Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final { + CurWriteBundleTarget = TargetTriple.str(); + return Error::success(); + } + + Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final { + return Error::success(); + } + + Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { + auto BI = BundlesInfo[CurWriteBundleTarget]; + OS.seek(BI.Offset); + OS.write(Input.getBufferStart(), Input.getBufferSize()); + return Error::success(); + } +}; + +namespace { + +// This class implements a list of temporary files that are removed upon +// object destruction. +class TempFileHandlerRAII { +public: + ~TempFileHandlerRAII() { + for (const auto &File : Files) + sys::fs::remove(File); + } + + // Creates temporary file with given contents. + Expected<StringRef> Create(Optional<ArrayRef<char>> Contents) { + SmallString<128u> File; + if (std::error_code EC = + sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File)) + return createFileError(File, EC); + Files.push_front(File); + + if (Contents) { + std::error_code EC; + raw_fd_ostream OS(File, EC); + if (EC) + return createFileError(File, EC); + OS.write(Contents->data(), Contents->size()); + } + return Files.front().str(); + } + +private: + std::forward_list<SmallString<128u>> Files; +}; + +} // end anonymous namespace + +/// Handler for object files. The bundles are organized by sections with a +/// designated name. +/// +/// To unbundle, we just copy the contents of the designated section. +class ObjectFileHandler final : public FileHandler { + + /// The object file we are currently dealing with. + std::unique_ptr<ObjectFile> Obj; + + /// Return the input file contents. + StringRef getInputFileContents() const { return Obj->getData(); } + + /// Return bundle name (<kind>-<triple>) if the provided section is an offload + /// section. + static Expected<Optional<StringRef>> IsOffloadSection(SectionRef CurSection) { + Expected<StringRef> NameOrErr = CurSection.getName(); + if (!NameOrErr) + return NameOrErr.takeError(); + + // If it does not start with the reserved suffix, just skip this section. + if (!NameOrErr->startswith(OFFLOAD_BUNDLER_MAGIC_STR)) + return None; + + // Return the triple that is right after the reserved prefix. + return NameOrErr->substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1); + } + + /// Total number of inputs. + unsigned NumberOfInputs = 0; + + /// Total number of processed inputs, i.e, inputs that were already + /// read from the buffers. + unsigned NumberOfProcessedInputs = 0; + + /// Iterator of the current and next section. + section_iterator CurrentSection; + section_iterator NextSection; + +public: + ObjectFileHandler(std::unique_ptr<ObjectFile> ObjIn) + : Obj(std::move(ObjIn)), CurrentSection(Obj->section_begin()), + NextSection(Obj->section_begin()) {} + + ~ObjectFileHandler() final {} + + Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); } + + Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final { + while (NextSection != Obj->section_end()) { + CurrentSection = NextSection; + ++NextSection; + + // Check if the current section name starts with the reserved prefix. If + // so, return the triple. + Expected<Optional<StringRef>> TripleOrErr = + IsOffloadSection(*CurrentSection); + if (!TripleOrErr) + return TripleOrErr.takeError(); + if (*TripleOrErr) + return **TripleOrErr; + } + return None; + } + + Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); } + + Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final { + Expected<StringRef> ContentOrErr = CurrentSection->getContents(); + if (!ContentOrErr) + return ContentOrErr.takeError(); + StringRef Content = *ContentOrErr; + + // Copy fat object contents to the output when extracting host bundle. + if (Content.size() == 1u && Content.front() == 0) + Content = StringRef(Input.getBufferStart(), Input.getBufferSize()); + + OS.write(Content.data(), Content.size()); + return Error::success(); + } + + Error WriteHeader(raw_fd_ostream &OS, + ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final { + assert(HostInputIndex != ~0u && "Host input index not defined."); + + // Record number of inputs. + NumberOfInputs = Inputs.size(); + return Error::success(); + } + + Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final { + ++NumberOfProcessedInputs; + return Error::success(); + } + + Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final { + assert(NumberOfProcessedInputs <= NumberOfInputs && + "Processing more inputs that actually exist!"); + assert(HostInputIndex != ~0u && "Host input index not defined."); + + // If this is not the last output, we don't have to do anything. + if (NumberOfProcessedInputs != NumberOfInputs) + return Error::success(); + + // We will use llvm-objcopy to add target objects sections to the output + // fat object. These sections should have 'exclude' flag set which tells + // link editor to remove them from linker inputs when linking executable or + // shared library. + + // Find llvm-objcopy in order to create the bundle binary. + ErrorOr<std::string> Objcopy = sys::findProgramByName( + "llvm-objcopy", sys::path::parent_path(BundlerExecutable)); + if (!Objcopy) + Objcopy = sys::findProgramByName("llvm-objcopy"); + if (!Objcopy) + return createStringError(Objcopy.getError(), + "unable to find 'llvm-objcopy' in path"); + + // We write to the output file directly. So, we close it and use the name + // to pass down to llvm-objcopy. + OS.close(); + + // Temporary files that need to be removed. + TempFileHandlerRAII TempFiles; + + // Compose llvm-objcopy command line for add target objects' sections with + // appropriate flags. + BumpPtrAllocator Alloc; + StringSaver SS{Alloc}; + SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"}; + for (unsigned I = 0; I < NumberOfInputs; ++I) { + StringRef InputFile = InputFileNames[I]; + if (I == HostInputIndex) { + // Special handling for the host bundle. We do not need to add a + // standard bundle for the host object since we are going to use fat + // object as a host object. Therefore use dummy contents (one zero byte) + // when creating section for the host bundle. + Expected<StringRef> TempFileOrErr = TempFiles.Create(ArrayRef<char>(0)); + if (!TempFileOrErr) + return TempFileOrErr.takeError(); + InputFile = *TempFileOrErr; + } + + ObjcopyArgs.push_back(SS.save(Twine("--add-section=") + + OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] + + "=" + InputFile)); + ObjcopyArgs.push_back(SS.save(Twine("--set-section-flags=") + + OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] + + "=readonly,exclude")); + } + ObjcopyArgs.push_back("--"); + ObjcopyArgs.push_back(InputFileNames[HostInputIndex]); + ObjcopyArgs.push_back(OutputFileNames.front()); + + if (Error Err = executeObjcopy(*Objcopy, ObjcopyArgs)) + return Err; + + return Error::success(); + } + + Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { + return Error::success(); + } + +private: + static Error executeObjcopy(StringRef Objcopy, ArrayRef<StringRef> Args) { + // If the user asked for the commands to be printed out, we do that + // instead of executing it. + if (PrintExternalCommands) { + errs() << "\"" << Objcopy << "\""; + for (StringRef Arg : drop_begin(Args, 1)) + errs() << " \"" << Arg << "\""; + errs() << "\n"; + } else { + if (sys::ExecuteAndWait(Objcopy, Args)) + return createStringError(inconvertibleErrorCode(), + "'llvm-objcopy' tool failed"); + } + return Error::success(); + } +}; + +/// Handler for text files. The bundled file will have the following format. +/// +/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple" +/// Bundle 1 +/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple" +/// ... +/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple" +/// Bundle N +/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple" +class TextFileHandler final : public FileHandler { + /// String that begins a line comment. + StringRef Comment; + + /// String that initiates a bundle. + std::string BundleStartString; + + /// String that closes a bundle. + std::string BundleEndString; + + /// Number of chars read from input. + size_t ReadChars = 0u; + +protected: + Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); } + + Expected<Optional<StringRef>> ReadBundleStart(MemoryBuffer &Input) final { + StringRef FC = Input.getBuffer(); + + // Find start of the bundle. + ReadChars = FC.find(BundleStartString, ReadChars); + if (ReadChars == FC.npos) + return None; + + // Get position of the triple. + size_t TripleStart = ReadChars = ReadChars + BundleStartString.size(); + + // Get position that closes the triple. + size_t TripleEnd = ReadChars = FC.find("\n", ReadChars); + if (TripleEnd == FC.npos) + return None; + + // Next time we read after the new line. + ++ReadChars; + + return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart); + } + + Error ReadBundleEnd(MemoryBuffer &Input) final { + StringRef FC = Input.getBuffer(); + + // Read up to the next new line. + assert(FC[ReadChars] == '\n' && "The bundle should end with a new line."); + + size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1); + if (TripleEnd != FC.npos) + // Next time we read after the new line. + ++ReadChars; + + return Error::success(); + } + + Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final { + StringRef FC = Input.getBuffer(); + size_t BundleStart = ReadChars; + + // Find end of the bundle. + size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars); + + StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart); + OS << Bundle; + + return Error::success(); + } + + Error WriteHeader(raw_fd_ostream &OS, + ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final { + return Error::success(); + } + + Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final { + OS << BundleStartString << TargetTriple << "\n"; + return Error::success(); + } + + Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final { + OS << BundleEndString << TargetTriple << "\n"; + return Error::success(); + } + + Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final { + OS << Input.getBuffer(); + return Error::success(); + } + +public: + TextFileHandler(StringRef Comment) : Comment(Comment), ReadChars(0) { + BundleStartString = + "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ "; + BundleEndString = + "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ "; + } + + Error listBundleIDsCallback(MemoryBuffer &Input, + const BundleInfo &Info) final { + // TODO: To list bundle IDs in a bundled text file we need to go through + // all bundles. The format of bundled text file may need to include a + // header if the performance of listing bundle IDs of bundled text file is + // important. + ReadChars = Input.getBuffer().find(BundleEndString, ReadChars); + if (Error Err = ReadBundleEnd(Input)) + return Err; + return Error::success(); + } +}; + +/// Return an appropriate object file handler. We use the specific object +/// handler if we know how to deal with that format, otherwise we use a default +/// binary file handler. +static std::unique_ptr<FileHandler> +CreateObjectFileHandler(MemoryBuffer &FirstInput) { + // Check if the input file format is one that we know how to deal with. + Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput); + + // We only support regular object files. If failed to open the input as a + // known binary or this is not an object file use the default binary handler. + if (errorToBool(BinaryOrErr.takeError()) || !isa<ObjectFile>(*BinaryOrErr)) + return std::make_unique<BinaryFileHandler>(); + + // Otherwise create an object file handler. The handler will be owned by the + // client of this function. + return std::make_unique<ObjectFileHandler>( + std::unique_ptr<ObjectFile>(cast<ObjectFile>(BinaryOrErr->release()))); +} + +/// Return an appropriate handler given the input files and options. +static Expected<std::unique_ptr<FileHandler>> +CreateFileHandler(MemoryBuffer &FirstInput) { + if (FilesType == "i") + return std::make_unique<TextFileHandler>(/*Comment=*/"//"); + if (FilesType == "ii") + return std::make_unique<TextFileHandler>(/*Comment=*/"//"); + if (FilesType == "cui") + return std::make_unique<TextFileHandler>(/*Comment=*/"//"); + // TODO: `.d` should be eventually removed once `-M` and its variants are + // handled properly in offload compilation. + if (FilesType == "d") + return std::make_unique<TextFileHandler>(/*Comment=*/"#"); + if (FilesType == "ll") + return std::make_unique<TextFileHandler>(/*Comment=*/";"); + if (FilesType == "bc") + return std::make_unique<BinaryFileHandler>(); + if (FilesType == "s") + return std::make_unique<TextFileHandler>(/*Comment=*/"#"); + if (FilesType == "o") + return CreateObjectFileHandler(FirstInput); + if (FilesType == "a") + return CreateObjectFileHandler(FirstInput); + if (FilesType == "gch") + return std::make_unique<BinaryFileHandler>(); + if (FilesType == "ast") + return std::make_unique<BinaryFileHandler>(); + + return createStringError(errc::invalid_argument, + "'" + FilesType + "': invalid file type specified"); +} + +/// Bundle the files. Return true if an error was found. +static Error BundleFiles() { + std::error_code EC; + + // Create output file. + raw_fd_ostream OutputFile(OutputFileNames.front(), EC, sys::fs::OF_None); + if (EC) + return createFileError(OutputFileNames.front(), EC); + + // Open input files. + SmallVector<std::unique_ptr<MemoryBuffer>, 8u> InputBuffers; + InputBuffers.reserve(InputFileNames.size()); + for (auto &I : InputFileNames) { + ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr = + MemoryBuffer::getFileOrSTDIN(I); + if (std::error_code EC = CodeOrErr.getError()) + return createFileError(I, EC); + InputBuffers.emplace_back(std::move(*CodeOrErr)); + } + + // Get the file handler. We use the host buffer as reference. + assert((HostInputIndex != ~0u || AllowNoHost) && + "Host input index undefined??"); + Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr = + CreateFileHandler(*InputBuffers[AllowNoHost ? 0 : HostInputIndex]); + if (!FileHandlerOrErr) + return FileHandlerOrErr.takeError(); + + std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr; + assert(FH); + + // Write header. + if (Error Err = FH->WriteHeader(OutputFile, InputBuffers)) + return Err; + + // Write all bundles along with the start/end markers. If an error was found + // writing the end of the bundle component, abort the bundle writing. + auto Input = InputBuffers.begin(); + for (auto &Triple : TargetNames) { + if (Error Err = FH->WriteBundleStart(OutputFile, Triple)) + return Err; + if (Error Err = FH->WriteBundle(OutputFile, **Input)) + return Err; + if (Error Err = FH->WriteBundleEnd(OutputFile, Triple)) + return Err; + ++Input; + } + return Error::success(); +} + +// List bundle IDs. Return true if an error was found. +static Error ListBundleIDsInFile(StringRef InputFileName) { + // Open Input file. + ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr = + MemoryBuffer::getFileOrSTDIN(InputFileName); + if (std::error_code EC = CodeOrErr.getError()) + return createFileError(InputFileName, EC); + + MemoryBuffer &Input = **CodeOrErr; + + // Select the right files handler. + Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr = + CreateFileHandler(Input); + if (!FileHandlerOrErr) + return FileHandlerOrErr.takeError(); + + std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr; + assert(FH); + return FH->listBundleIDs(Input); +} + +// Unbundle the files. Return true if an error was found. +static Error UnbundleFiles() { + // Open Input file. + ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr = + MemoryBuffer::getFileOrSTDIN(InputFileNames.front()); + if (std::error_code EC = CodeOrErr.getError()) + return createFileError(InputFileNames.front(), EC); + + MemoryBuffer &Input = **CodeOrErr; + + // Select the right files handler. + Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr = + CreateFileHandler(Input); + if (!FileHandlerOrErr) + return FileHandlerOrErr.takeError(); + + std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr; + assert(FH); + + // Read the header of the bundled file. + if (Error Err = FH->ReadHeader(Input)) + return Err; + + // Create a work list that consist of the map triple/output file. + StringMap<StringRef> Worklist; + auto Output = OutputFileNames.begin(); + for (auto &Triple : TargetNames) { + Worklist[Triple] = *Output; + ++Output; + } + + // Read all the bundles that are in the work list. If we find no bundles we + // assume the file is meant for the host target. + bool FoundHostBundle = false; + while (!Worklist.empty()) { + Expected<Optional<StringRef>> CurTripleOrErr = FH->ReadBundleStart(Input); + if (!CurTripleOrErr) + return CurTripleOrErr.takeError(); + + // We don't have more bundles. + if (!*CurTripleOrErr) + break; + + StringRef CurTriple = **CurTripleOrErr; + assert(!CurTriple.empty()); + + auto Output = Worklist.find(CurTriple); + // The file may have more bundles for other targets, that we don't care + // about. Therefore, move on to the next triple + if (Output == Worklist.end()) + continue; + + // Check if the output file can be opened and copy the bundle to it. + std::error_code EC; + raw_fd_ostream OutputFile(Output->second, EC, sys::fs::OF_None); + if (EC) + return createFileError(Output->second, EC); + if (Error Err = FH->ReadBundle(OutputFile, Input)) + return Err; + if (Error Err = FH->ReadBundleEnd(Input)) + return Err; + Worklist.erase(Output); + + // Record if we found the host bundle. + auto OffloadInfo = OffloadTargetInfo(CurTriple); + if (OffloadInfo.hasHostKind()) + FoundHostBundle = true; + } + + if (!AllowMissingBundles && !Worklist.empty()) { + std::string ErrMsg = "Can't find bundles for"; + std::set<StringRef> Sorted; + for (auto &E : Worklist) + Sorted.insert(E.first()); + unsigned I = 0; + unsigned Last = Sorted.size() - 1; + for (auto &E : Sorted) { + if (I != 0 && Last > 1) + ErrMsg += ","; + ErrMsg += " "; + if (I == Last && I != 0) + ErrMsg += "and "; + ErrMsg += E.str(); + ++I; + } + return createStringError(inconvertibleErrorCode(), ErrMsg); + } + + // If no bundles were found, assume the input file is the host bundle and + // create empty files for the remaining targets. + if (Worklist.size() == TargetNames.size()) { + for (auto &E : Worklist) { + std::error_code EC; + raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None); + if (EC) + return createFileError(E.second, EC); + + // If this entry has a host kind, copy the input file to the output file. + auto OffloadInfo = OffloadTargetInfo(E.getKey()); + if (OffloadInfo.hasHostKind()) + OutputFile.write(Input.getBufferStart(), Input.getBufferSize()); + } + return Error::success(); + } + + // If we found elements, we emit an error if none of those were for the host + // in case host bundle name was provided in command line. + if (!FoundHostBundle && HostInputIndex != ~0u) + return createStringError(inconvertibleErrorCode(), + "Can't find bundle for the host target"); + + // If we still have any elements in the worklist, create empty files for them. + for (auto &E : Worklist) { + std::error_code EC; + raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None); + if (EC) + return createFileError(E.second, EC); + } + + return Error::success(); +} + +static Archive::Kind getDefaultArchiveKindForHost() { + return Triple(sys::getDefaultTargetTriple()).isOSDarwin() ? Archive::K_DARWIN + : Archive::K_GNU; +} + +/// @brief Checks if a code object \p CodeObjectInfo is compatible with a given +/// target \p TargetInfo. +/// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id +bool isCodeObjectCompatible(OffloadTargetInfo &CodeObjectInfo, + OffloadTargetInfo &TargetInfo) { + + // Compatible in case of exact match. + if (CodeObjectInfo == TargetInfo) { + DEBUG_WITH_TYPE("CodeObjectCompatibility", + dbgs() << "Compatible: Exact match: \t[CodeObject: " + << CodeObjectInfo.str() + << "]\t:\t[Target: " << TargetInfo.str() << "]\n"); + return true; + } + + // Incompatible if Kinds or Triples mismatch. + if (CodeObjectInfo.OffloadKind != TargetInfo.OffloadKind || + !CodeObjectInfo.Triple.isCompatibleWith(TargetInfo.Triple)) { + DEBUG_WITH_TYPE( + "CodeObjectCompatibility", + dbgs() << "Incompatible: Kind/Triple mismatch \t[CodeObject: " + << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str() + << "]\n"); + return false; + } + + // Incompatible if GPUArch mismatch. + if (CodeObjectInfo.GPUArch != TargetInfo.GPUArch) { + DEBUG_WITH_TYPE("CodeObjectCompatibility", + dbgs() << "Incompatible: GPU Arch mismatch \t[CodeObject: " + << CodeObjectInfo.str() + << "]\t:\t[Target: " << TargetInfo.str() << "]\n"); + return false; + } + + DEBUG_WITH_TYPE( + "CodeObjectCompatibility", + dbgs() << "Compatible: Code Objects are compatible \t[CodeObject: " + << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str() + << "]\n"); + return true; +} + +/// @brief Computes a list of targets among all given targets which are +/// compatible with this code object +/// @param [in] CodeObjectInfo Code Object +/// @param [out] CompatibleTargets List of all compatible targets among all +/// given targets +/// @return false, if no compatible target is found. +static bool +getCompatibleOffloadTargets(OffloadTargetInfo &CodeObjectInfo, + SmallVectorImpl<StringRef> &CompatibleTargets) { + if (!CompatibleTargets.empty()) { + DEBUG_WITH_TYPE("CodeObjectCompatibility", + dbgs() << "CompatibleTargets list should be empty\n"); + return false; + } + for (auto &Target : TargetNames) { + auto TargetInfo = OffloadTargetInfo(Target); + if (isCodeObjectCompatible(CodeObjectInfo, TargetInfo)) + CompatibleTargets.push_back(Target); + } + return !CompatibleTargets.empty(); +} + +/// UnbundleArchive takes an archive file (".a") as input containing bundled +/// code object files, and a list of offload targets (not host), and extracts +/// the code objects into a new archive file for each offload target. Each +/// resulting archive file contains all code object files corresponding to that +/// particular offload target. The created archive file does not +/// contain an index of the symbols and code object files are named as +/// <<Parent Bundle Name>-<CodeObject's GPUArch>>, with ':' replaced with '_'. +static Error UnbundleArchive() { + std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers; + + /// Map of target names with list of object files that will form the device + /// specific archive for that target + StringMap<std::vector<NewArchiveMember>> OutputArchivesMap; + + // Map of target names and output archive filenames + StringMap<StringRef> TargetOutputFileNameMap; + + auto Output = OutputFileNames.begin(); + for (auto &Target : TargetNames) { + TargetOutputFileNameMap[Target] = *Output; + ++Output; + } + + StringRef IFName = InputFileNames.front(); + ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr = + MemoryBuffer::getFileOrSTDIN(IFName, true, false); + if (std::error_code EC = BufOrErr.getError()) + return createFileError(InputFileNames.front(), EC); + + ArchiveBuffers.push_back(std::move(*BufOrErr)); + Expected<std::unique_ptr<llvm::object::Archive>> LibOrErr = + Archive::create(ArchiveBuffers.back()->getMemBufferRef()); + if (!LibOrErr) + return LibOrErr.takeError(); + + auto Archive = std::move(*LibOrErr); + + Error ArchiveErr = Error::success(); + auto ChildEnd = Archive->child_end(); + + /// Iterate over all bundled code object files in the input archive. + for (auto ArchiveIter = Archive->child_begin(ArchiveErr); + ArchiveIter != ChildEnd; ++ArchiveIter) { + if (ArchiveErr) + return ArchiveErr; + auto ArchiveChildNameOrErr = (*ArchiveIter).getName(); + if (!ArchiveChildNameOrErr) + return ArchiveChildNameOrErr.takeError(); + + StringRef BundledObjectFile = sys::path::filename(*ArchiveChildNameOrErr); + + auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef(); + if (!CodeObjectBufferRefOrErr) + return CodeObjectBufferRefOrErr.takeError(); + + auto CodeObjectBuffer = + MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false); + + Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr = + CreateFileHandler(*CodeObjectBuffer); + if (!FileHandlerOrErr) + return FileHandlerOrErr.takeError(); + + std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr; + assert(FileHandler && + "FileHandle creation failed for file in the archive!"); + + if (Error ReadErr = FileHandler.get()->ReadHeader(*CodeObjectBuffer)) + return ReadErr; + + Expected<Optional<StringRef>> CurBundleIDOrErr = + FileHandler->ReadBundleStart(*CodeObjectBuffer); + if (!CurBundleIDOrErr) + return CurBundleIDOrErr.takeError(); + + Optional<StringRef> OptionalCurBundleID = *CurBundleIDOrErr; + // No device code in this child, skip. + if (!OptionalCurBundleID.hasValue()) + continue; + StringRef CodeObject = *OptionalCurBundleID; + + // Process all bundle entries (CodeObjects) found in this child of input + // archive. + while (!CodeObject.empty()) { + SmallVector<StringRef> CompatibleTargets; + auto CodeObjectInfo = OffloadTargetInfo(CodeObject); + if (CodeObjectInfo.hasHostKind()) { + // Do nothing, we don't extract host code yet. + } else if (getCompatibleOffloadTargets(CodeObjectInfo, + CompatibleTargets)) { + std::string BundleData; + raw_string_ostream DataStream(BundleData); + if (Error Err = + FileHandler.get()->ReadBundle(DataStream, *CodeObjectBuffer)) + return Err; + + for (auto &CompatibleTarget : CompatibleTargets) { + SmallString<128> BundledObjectFileName; + BundledObjectFileName.assign(BundledObjectFile); + auto OutputBundleName = + Twine(llvm::sys::path::stem(BundledObjectFileName) + "-" + + CodeObject + + getDeviceLibraryFileName(BundledObjectFileName, + CodeObjectInfo.GPUArch)) + .str(); + // Replace ':' in optional target feature list with '_' to ensure + // cross-platform validity. + std::replace(OutputBundleName.begin(), OutputBundleName.end(), ':', + '_'); + + std::unique_ptr<MemoryBuffer> MemBuf = MemoryBuffer::getMemBufferCopy( + DataStream.str(), OutputBundleName); + ArchiveBuffers.push_back(std::move(MemBuf)); + llvm::MemoryBufferRef MemBufRef = + MemoryBufferRef(*(ArchiveBuffers.back())); + + // For inserting <CompatibleTarget, list<CodeObject>> entry in + // OutputArchivesMap. + if (OutputArchivesMap.find(CompatibleTarget) == + OutputArchivesMap.end()) { + + std::vector<NewArchiveMember> ArchiveMembers; + ArchiveMembers.push_back(NewArchiveMember(MemBufRef)); + OutputArchivesMap.insert_or_assign(CompatibleTarget, + std::move(ArchiveMembers)); + } else { + OutputArchivesMap[CompatibleTarget].push_back( + NewArchiveMember(MemBufRef)); + } + } + } + + if (Error Err = FileHandler.get()->ReadBundleEnd(*CodeObjectBuffer)) + return Err; + + Expected<Optional<StringRef>> NextTripleOrErr = + FileHandler->ReadBundleStart(*CodeObjectBuffer); + if (!NextTripleOrErr) + return NextTripleOrErr.takeError(); + + CodeObject = ((*NextTripleOrErr).hasValue()) ? **NextTripleOrErr : ""; + } // End of processing of all bundle entries of this child of input archive. + } // End of while over children of input archive. + + assert(!ArchiveErr && "Error occured while reading archive!"); + + /// Write out an archive for each target + for (auto &Target : TargetNames) { + StringRef FileName = TargetOutputFileNameMap[Target]; + StringMapIterator<std::vector<llvm::NewArchiveMember>> CurArchiveMembers = + OutputArchivesMap.find(Target); + if (CurArchiveMembers != OutputArchivesMap.end()) { + if (Error WriteErr = writeArchive(FileName, CurArchiveMembers->getValue(), + true, getDefaultArchiveKindForHost(), + true, false, nullptr)) + return WriteErr; + } else if (!AllowMissingBundles) { + std::string ErrMsg = + Twine("no compatible code object found for the target '" + Target + + "' in heterogeneous archive library: " + IFName) + .str(); + return createStringError(inconvertibleErrorCode(), ErrMsg); + } else { // Create an empty archive file if no compatible code object is + // found and "allow-missing-bundles" is enabled. It ensures that + // the linker using output of this step doesn't complain about + // the missing input file. + std::vector<llvm::NewArchiveMember> EmptyArchive; + EmptyArchive.clear(); + if (Error WriteErr = writeArchive(FileName, EmptyArchive, true, + getDefaultArchiveKindForHost(), true, + false, nullptr)) + return WriteErr; + } + } + + return Error::success(); +} + +static void PrintVersion(raw_ostream &OS) { + OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n'; +} + +int main(int argc, const char **argv) { + sys::PrintStackTraceOnErrorSignal(argv[0]); + + cl::HideUnrelatedOptions(ClangOffloadBundlerCategory); + cl::SetVersionPrinter(PrintVersion); + cl::ParseCommandLineOptions( + argc, argv, + "A tool to bundle several input files of the specified type <type> \n" + "referring to the same source file but different targets into a single \n" + "one. The resulting file can also be unbundled into different files by \n" + "this tool if -unbundle is provided.\n"); + + if (Help) { + cl::PrintHelpMessage(); + return 0; + } + + auto reportError = [argv](Error E) { + logAllUnhandledErrors(std::move(E), WithColor::error(errs(), argv[0])); + exit(1); + }; + + auto doWork = [&](std::function<llvm::Error()> Work) { + // Save the current executable directory as it will be useful to find other + // tools. + BundlerExecutable = argv[0]; + if (!llvm::sys::fs::exists(BundlerExecutable)) + BundlerExecutable = + sys::fs::getMainExecutable(argv[0], &BundlerExecutable); + + if (llvm::Error Err = Work()) { + reportError(std::move(Err)); + } + }; + + if (ListBundleIDs) { + if (Unbundle) { + reportError( + createStringError(errc::invalid_argument, + "-unbundle and -list cannot be used together")); + } + if (InputFileNames.size() != 1) { + reportError(createStringError(errc::invalid_argument, + "only one input file supported for -list")); + } + if (OutputFileNames.size()) { + reportError(createStringError(errc::invalid_argument, + "-outputs option is invalid for -list")); + } + if (TargetNames.size()) { + reportError(createStringError(errc::invalid_argument, + "-targets option is invalid for -list")); + } + + doWork([]() { return ListBundleIDsInFile(InputFileNames.front()); }); + return 0; + } + + if (OutputFileNames.getNumOccurrences() == 0) { + reportError(createStringError( + errc::invalid_argument, + "for the --outputs option: must be specified at least once!")); + } + if (TargetNames.getNumOccurrences() == 0) { + reportError(createStringError( + errc::invalid_argument, + "for the --targets option: must be specified at least once!")); + } + if (Unbundle) { + if (InputFileNames.size() != 1) { + reportError(createStringError( + errc::invalid_argument, + "only one input file supported in unbundling mode")); + } + if (OutputFileNames.size() != TargetNames.size()) { + reportError(createStringError(errc::invalid_argument, + "number of output files and targets should " + "match in unbundling mode")); + } + } else { + if (FilesType == "a") { + reportError(createStringError(errc::invalid_argument, + "Archive files are only supported " + "for unbundling")); + } + if (OutputFileNames.size() != 1) { + reportError(createStringError( + errc::invalid_argument, + "only one output file supported in bundling mode")); + } + if (InputFileNames.size() != TargetNames.size()) { + reportError(createStringError( + errc::invalid_argument, + "number of input files and targets should match in bundling mode")); + } + } + + // Verify that the offload kinds and triples are known. We also check that we + // have exactly one host target. + unsigned Index = 0u; + unsigned HostTargetNum = 0u; + bool HIPOnly = true; + llvm::DenseSet<StringRef> ParsedTargets; + for (StringRef Target : TargetNames) { + if (ParsedTargets.contains(Target)) { + reportError(createStringError(errc::invalid_argument, + "Duplicate targets are not allowed")); + } + ParsedTargets.insert(Target); + + auto OffloadInfo = OffloadTargetInfo(Target); + bool KindIsValid = OffloadInfo.isOffloadKindValid(); + bool TripleIsValid = OffloadInfo.isTripleValid(); + + if (!KindIsValid || !TripleIsValid) { + SmallVector<char, 128u> Buf; + raw_svector_ostream Msg(Buf); + Msg << "invalid target '" << Target << "'"; + if (!KindIsValid) + Msg << ", unknown offloading kind '" << OffloadInfo.OffloadKind << "'"; + if (!TripleIsValid) + Msg << ", unknown target triple '" << OffloadInfo.Triple.str() << "'"; + reportError(createStringError(errc::invalid_argument, Msg.str())); + } + + if (KindIsValid && OffloadInfo.hasHostKind()) { + ++HostTargetNum; + // Save the index of the input that refers to the host. + HostInputIndex = Index; + } + + if (OffloadInfo.OffloadKind != "hip" && OffloadInfo.OffloadKind != "hipv4") + HIPOnly = false; + + ++Index; + } + + // HIP uses clang-offload-bundler to bundle device-only compilation results + // for multiple GPU archs, therefore allow no host target if all entries + // are for HIP. + AllowNoHost = HIPOnly; + + // Host triple is not really needed for unbundling operation, so do not + // treat missing host triple as error if we do unbundling. + if ((Unbundle && HostTargetNum > 1) || + (!Unbundle && HostTargetNum != 1 && !AllowNoHost)) { + reportError(createStringError(errc::invalid_argument, + "expecting exactly one host target but got " + + Twine(HostTargetNum))); + } + + doWork([]() { + if (Unbundle) { + if (FilesType == "a") + return UnbundleArchive(); + else + return UnbundleFiles(); + } else + return BundleFiles(); + }); + return 0; +} |