diff --git a/llvm/docs/CommandGuide/llvm-objdump.rst b/llvm/docs/CommandGuide/llvm-objdump.rst index ab9f583e96ec6..5e5eaccecd2b7 100644 --- a/llvm/docs/CommandGuide/llvm-objdump.rst +++ b/llvm/docs/CommandGuide/llvm-objdump.rst @@ -217,7 +217,7 @@ OPTIONS .. option:: --offloading - Display the content of the LLVM offloading section. + Display the content of the LLVM offloading sections and HIP offload bundles. .. option:: --prefix= diff --git a/llvm/include/llvm/Object/OffloadBundle.h b/llvm/include/llvm/Object/OffloadBundle.h new file mode 100644 index 0000000000000..7fc0ab141966a --- /dev/null +++ b/llvm/include/llvm/Object/OffloadBundle.h @@ -0,0 +1,211 @@ +//===- OffloadBundle.h - Utilities for offload bundles---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===-------------------------------------------------------------------------===// +// +// This file contains the binary format used for budingling device metadata with +// an associated device image. The data can then be stored inside a host object +// file to create a fat binary and read by the linker. This is intended to be a +// thin wrapper around the image itself. If this format becomes sufficiently +// complex it should be moved to a standard binary format like msgpack or ELF. +// +//===-------------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_OFFLOADBUNDLE_H +#define LLVM_OBJECT_OFFLOADBUNDLE_H + +#include "llvm/ADT/MapVector.h" +#include "llvm/ADT/SmallString.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Compression.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include + +namespace llvm { + +namespace object { + +class CompressedOffloadBundle { +private: + static inline const size_t MagicSize = 4; + static inline const size_t VersionFieldSize = sizeof(uint16_t); + static inline const size_t MethodFieldSize = sizeof(uint16_t); + static inline const size_t FileSizeFieldSize = sizeof(uint32_t); + static inline const size_t UncompressedSizeFieldSize = sizeof(uint32_t); + static inline const size_t HashFieldSize = sizeof(uint64_t); + static inline const size_t V1HeaderSize = + MagicSize + VersionFieldSize + MethodFieldSize + + UncompressedSizeFieldSize + HashFieldSize; + static inline const size_t V2HeaderSize = + MagicSize + VersionFieldSize + FileSizeFieldSize + MethodFieldSize + + UncompressedSizeFieldSize + HashFieldSize; + static inline const llvm::StringRef MagicNumber = "CCOB"; + static inline const uint16_t Version = 2; + +public: + static llvm::Expected> + compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input, + bool Verbose = false); + static llvm::Expected> + decompress(llvm::MemoryBufferRef &Input, bool Verbose = false); +}; + +/// Bundle entry in binary clang-offload-bundler format. +struct OffloadBundleEntry { + uint64_t Offset = 0u; + uint64_t Size = 0u; + uint64_t IDLength = 0u; + StringRef ID; + OffloadBundleEntry(uint64_t O, uint64_t S, uint64_t I, StringRef T) + : Offset(O), Size(S), IDLength(I), ID(T) {} + void dumpInfo(raw_ostream &OS) { + OS << "Offset = " << Offset << ", Size = " << Size + << ", ID Length = " << IDLength << ", ID = " << ID; + } + void dumpURI(raw_ostream &OS, StringRef FilePath) { + OS << ID.data() << "\tfile://" << FilePath << "#offset=" << Offset + << "&size=" << Size << "\n"; + } +}; + +/// Fat binary embedded in object files in clang-offload-bundler format +class OffloadBundleFatBin { + + uint64_t Size = 0u; + StringRef FileName; + uint64_t NumberOfEntries; + SmallVector Entries; + +public: + SmallVector getEntries() { return Entries; } + uint64_t getSize() const { return Size; } + StringRef getFileName() const { return FileName; } + uint64_t getNumEntries() const { return NumberOfEntries; } + + static Expected> + create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName); + Error extractBundle(const ObjectFile &Source); + + Error dumpEntryToCodeObject(); + + Error readEntries(StringRef Section, uint64_t SectionOffset); + void dumpEntries() { + for (OffloadBundleEntry &Entry : Entries) + Entry.dumpInfo(outs()); + } + + void printEntriesAsURI() { + for (OffloadBundleEntry &Entry : Entries) + Entry.dumpURI(outs(), FileName); + } + + OffloadBundleFatBin(MemoryBufferRef Source, StringRef File) + : FileName(File), NumberOfEntries(0), + Entries(SmallVector()) {} + + SmallVector entryIDContains(StringRef Str) { + + SmallVector Found = SmallVector(); + llvm::transform(Entries, std::back_inserter(Found), [Str](auto &X) { + if (X.ID.contains(Str)) + return X; + }); + return Found; + } +}; + +enum UriTypeT { FILE_URI, MEMORY_URI }; + +struct OffloadBundleURI { + int64_t Offset = 0; + int64_t Size = 0; + uint64_t ProcessID = 0; + StringRef FileName; + UriTypeT URIType; + + // Constructors + // TODO: add a Copy ctor ? + OffloadBundleURI(StringRef File, int64_t Off, int64_t Size) + : Offset(Off), Size(Size), ProcessID(0), FileName(File), + URIType(FILE_URI) {} + +public: + static Expected> + createOffloadBundleURI(StringRef Str, UriTypeT Type) { + switch (Type) { + case FILE_URI: + return createFileURI(Str); + break; + case MEMORY_URI: + return createMemoryURI(Str); + break; + default: + return createStringError(object_error::parse_failed, + "Unrecognized URI type"); + } + } + + static Expected> + createFileURI(StringRef Str) { + int64_t O = 0; + int64_t S = 0; + + if (!Str.consume_front("file://")) + return createStringError(object_error::parse_failed, + "Reading type of URI"); + + StringRef FilePathname = + Str.take_until([](char C) { return (C == '#') || (C == '?'); }); + Str = Str.drop_front(FilePathname.size()); + + if (!Str.consume_front("#offset=")) + return createStringError(object_error::parse_failed, + "Reading 'offset' in URI"); + + StringRef OffsetStr = Str.take_until([](char C) { return C == '&'; }); + OffsetStr.getAsInteger(10, O); + Str = Str.drop_front(OffsetStr.size()); + + if (Str.consume_front("&size=")) + return createStringError(object_error::parse_failed, + "Reading 'size' in URI"); + + Str.getAsInteger(10, S); + std::unique_ptr OffloadingURI( + new OffloadBundleURI(FilePathname, O, S)); + return OffloadingURI; + } + + static Expected> + createMemoryURI(StringRef Str) { + // TODO: add parseMemoryURI type + return createStringError(object_error::parse_failed, + "Memory Type URI is not currently supported."); + } + + StringRef getFileName() const { return FileName; } +}; + +/// Extracts fat binary in binary clang-offload-bundler format from object \p +/// Obj and return it in \p Bundles +Error extractOffloadBundleFatBinary( + const ObjectFile &Obj, SmallVectorImpl &Bundles); + +/// Extract code object memory from the given \p Source object file at \p Offset +/// and of \p Size, and copy into \p OutputFileName. +Error extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_t Size, + StringRef OutputFileName); + +/// Extracts an Offload Bundle Entry given by URI +Error extractOffloadBundleByURI(StringRef URIstr); + +} // namespace object + +} // namespace llvm +#endif diff --git a/llvm/lib/Object/CMakeLists.txt b/llvm/lib/Object/CMakeLists.txt index bfb420e57a7f4..870169a83174f 100644 --- a/llvm/lib/Object/CMakeLists.txt +++ b/llvm/lib/Object/CMakeLists.txt @@ -22,6 +22,7 @@ add_llvm_component_library(LLVMObject Object.cpp ObjectFile.cpp OffloadBinary.cpp + OffloadBundle.cpp RecordStreamer.cpp RelocationResolver.cpp SymbolicFile.cpp diff --git a/llvm/lib/Object/OffloadBundle.cpp b/llvm/lib/Object/OffloadBundle.cpp new file mode 100644 index 0000000000000..b3d85be811a5a --- /dev/null +++ b/llvm/lib/Object/OffloadBundle.cpp @@ -0,0 +1,478 @@ +//===- OffloadBundle.cpp - Utilities for offload bundles---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------===// + +#include "llvm/Object/OffloadBundle.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Module.h" +#include "llvm/IRReader/IRReader.h" +#include "llvm/MC/StringTableBuilder.h" +#include "llvm/Object/Archive.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/Error.h" +#include "llvm/Object/IRObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/SourceMgr.h" +#include "llvm/Support/Timer.h" + +using namespace llvm; +using namespace llvm::object; + +static llvm::TimerGroup + OffloadBundlerTimerGroup("Offload Bundler Timer Group", + "Timer group for offload bundler"); + +// Extract an Offload bundle (usually a Offload Bundle) from a fat_bin +// section +Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset, + StringRef FileName, + SmallVectorImpl &Bundles) { + + uint64_t Offset = 0; + int64_t NextbundleStart = 0; + + // There could be multiple offloading bundles stored at this section. + while (NextbundleStart >= 0) { + + std::unique_ptr Buffer = + MemoryBuffer::getMemBuffer(Contents.getBuffer().drop_front(Offset), "", + /*RequiresNullTerminator=*/false); + + // Create the FatBinBindle object. This will also create the Bundle Entry + // list info. + auto FatBundleOrErr = + OffloadBundleFatBin::create(*Buffer, SectionOffset + Offset, FileName); + if (!FatBundleOrErr) + return FatBundleOrErr.takeError(); + + // Add current Bundle to list. + Bundles.emplace_back(std::move(**FatBundleOrErr)); + + // Find the next bundle by searching for the magic string + StringRef Str = Buffer->getBuffer(); + NextbundleStart = + (int64_t)Str.find(StringRef("__CLANG_OFFLOAD_BUNDLE__"), 24); + + if (NextbundleStart >= 0) + Offset += NextbundleStart; + } + + return Error::success(); +} + +Error OffloadBundleFatBin::readEntries(StringRef Buffer, + uint64_t SectionOffset) { + uint64_t NumOfEntries = 0; + + BinaryStreamReader Reader(Buffer, llvm::endianness::little); + + // Read the Magic String first. + StringRef Magic; + if (auto EC = Reader.readFixedString(Magic, 24)) + return errorCodeToError(object_error::parse_failed); + + // Read the number of Code Objects (Entries) in the current Bundle. + if (auto EC = Reader.readInteger(NumOfEntries)) + return errorCodeToError(object_error::parse_failed); + + NumberOfEntries = NumOfEntries; + + // For each Bundle Entry (code object) + for (uint64_t I = 0; I < NumOfEntries; I++) { + uint64_t EntrySize; + uint64_t EntryOffset; + uint64_t EntryIDSize; + StringRef EntryID; + + if (auto EC = Reader.readInteger(EntryOffset)) + return errorCodeToError(object_error::parse_failed); + + if (auto EC = Reader.readInteger(EntrySize)) + return errorCodeToError(object_error::parse_failed); + + if (auto EC = Reader.readInteger(EntryIDSize)) + return errorCodeToError(object_error::parse_failed); + + if (auto EC = Reader.readFixedString(EntryID, EntryIDSize)) + return errorCodeToError(object_error::parse_failed); + + // Create a Bundle Entry object: + // auto Entry = new OffloadBundleEntry(EntryOffset + SectionOffset, + // EntrySize, + // EntryIDSize, EntryID); + auto Entry = std::make_unique( + EntryOffset + SectionOffset, EntrySize, EntryIDSize, EntryID); + + Entries.push_back(*Entry); + } + + return Error::success(); +} + +Expected> +OffloadBundleFatBin::create(MemoryBufferRef Buf, uint64_t SectionOffset, + StringRef FileName) { + if (Buf.getBufferSize() < 24) + return errorCodeToError(object_error::parse_failed); + + // Check for magic bytes. + if (identify_magic(Buf.getBuffer()) != file_magic::offload_bundle) + return errorCodeToError(object_error::parse_failed); + + OffloadBundleFatBin *TheBundle = new OffloadBundleFatBin(Buf, FileName); + + // Read the Bundle Entries + Error Err = TheBundle->readEntries(Buf.getBuffer(), SectionOffset); + if (Err) + return errorCodeToError(object_error::parse_failed); + + return std::unique_ptr(TheBundle); +} + +Error OffloadBundleFatBin::extractBundle(const ObjectFile &Source) { + // This will extract all entries in the Bundle + for (OffloadBundleEntry &Entry : Entries) { + + if (Entry.Size == 0) + continue; + + // create output file name. Which should be + // -offset-size.co" + std::string Str = getFileName().str() + "-offset" + itostr(Entry.Offset) + + "-size" + itostr(Entry.Size) + ".co"; + if (Error Err = object::extractCodeObject(Source, Entry.Offset, Entry.Size, + StringRef(Str))) + return Err; + } + + return Error::success(); +} + +Error object::extractOffloadBundleFatBinary( + const ObjectFile &Obj, SmallVectorImpl &Bundles) { + assert((Obj.isELF() || Obj.isCOFF()) && "Invalid file type"); + + // Iterate through Sections until we find an offload_bundle section. + for (SectionRef Sec : Obj.sections()) { + Expected Buffer = Sec.getContents(); + if (!Buffer) + return Buffer.takeError(); + + // If it does not start with the reserved suffix, just skip this section. + if ((llvm::identify_magic(*Buffer) == llvm::file_magic::offload_bundle) || + (llvm::identify_magic(*Buffer) == + llvm::file_magic::offload_bundle_compressed)) { + + uint64_t SectionOffset = 0; + if (Obj.isELF()) { + SectionOffset = ELFSectionRef(Sec).getOffset(); + } else if (Obj.isCOFF()) { + if (const COFFObjectFile *COFFObj = dyn_cast(&Obj)) + const coff_section *CoffSection = COFFObj->getCOFFSection(Sec); + } + + MemoryBufferRef Contents(*Buffer, Obj.getFileName()); + + if (llvm::identify_magic(*Buffer) == + llvm::file_magic::offload_bundle_compressed) { + // Decompress the input if necessary. + Expected> DecompressedBufferOrErr = + CompressedOffloadBundle::decompress(Contents, false); + + if (!DecompressedBufferOrErr) + return createStringError( + inconvertibleErrorCode(), + "Failed to decompress input: " + + llvm::toString(DecompressedBufferOrErr.takeError())); + + MemoryBuffer &DecompressedInput = **DecompressedBufferOrErr; + if (Error Err = extractOffloadBundle(DecompressedInput, SectionOffset, + Obj.getFileName(), Bundles)) + return Err; + } else { + if (Error Err = extractOffloadBundle(Contents, SectionOffset, + Obj.getFileName(), Bundles)) + return Err; + } + } + } + return Error::success(); +} + +Error object::extractCodeObject(const ObjectFile &Source, int64_t Offset, + int64_t Size, StringRef OutputFileName) { + Expected> BufferOrErr = + FileOutputBuffer::create(OutputFileName, Size); + + if (!BufferOrErr) + return BufferOrErr.takeError(); + + Expected InputBuffOrErr = Source.getMemoryBufferRef(); + if (Error Err = InputBuffOrErr.takeError()) + return Err; + + std::unique_ptr Buf = std::move(*BufferOrErr); + std::copy(InputBuffOrErr->getBufferStart() + Offset, + InputBuffOrErr->getBufferStart() + Offset + Size, + Buf->getBufferStart()); + if (Error E = Buf->commit()) + return E; + + return Error::success(); +} + +// given a file name, offset, and size, extract data into a code object file, +// into file -offset-size.co +Error object::extractOffloadBundleByURI(StringRef URIstr) { + // create a URI object + Expected> UriOrErr( + OffloadBundleURI::createOffloadBundleURI(URIstr, FILE_URI)); + if (!UriOrErr) + return UriOrErr.takeError(); + + OffloadBundleURI &Uri = **UriOrErr; + std::string OutputFile = Uri.FileName.str(); + OutputFile += + "-offset" + itostr(Uri.Offset) + "-size" + itostr(Uri.Size) + ".co"; + + // Create an ObjectFile object from uri.file_uri + auto ObjOrErr = ObjectFile::createObjectFile(Uri.FileName); + if (!ObjOrErr) + return ObjOrErr.takeError(); + + auto Obj = ObjOrErr->getBinary(); + if (Error Err = + object::extractCodeObject(*Obj, Uri.Offset, Uri.Size, OutputFile)) + return Err; + + return Error::success(); +} + +// Utility function to format numbers with commas +static std::string formatWithCommas(unsigned long long Value) { + std::string Num = std::to_string(Value); + int InsertPosition = Num.length() - 3; + while (InsertPosition > 0) { + Num.insert(InsertPosition, ","); + InsertPosition -= 3; + } + return Num; +} + +llvm::Expected> +CompressedOffloadBundle::decompress(llvm::MemoryBufferRef &Input, + bool Verbose) { + StringRef Blob = Input.getBuffer(); + + if (Blob.size() < V1HeaderSize) + return llvm::MemoryBuffer::getMemBufferCopy(Blob); + + if (llvm::identify_magic(Blob) != + llvm::file_magic::offload_bundle_compressed) { + if (Verbose) + llvm::errs() << "Uncompressed bundle.\n"; + return llvm::MemoryBuffer::getMemBufferCopy(Blob); + } + + size_t CurrentOffset = MagicSize; + + uint16_t ThisVersion; + memcpy(&ThisVersion, Blob.data() + CurrentOffset, sizeof(uint16_t)); + CurrentOffset += VersionFieldSize; + + uint16_t CompressionMethod; + memcpy(&CompressionMethod, Blob.data() + CurrentOffset, sizeof(uint16_t)); + CurrentOffset += MethodFieldSize; + + uint32_t TotalFileSize; + if (ThisVersion >= 2) { + if (Blob.size() < V2HeaderSize) + return createStringError(inconvertibleErrorCode(), + "Compressed bundle header size too small"); + memcpy(&TotalFileSize, Blob.data() + CurrentOffset, sizeof(uint32_t)); + CurrentOffset += FileSizeFieldSize; + } + + uint32_t UncompressedSize; + memcpy(&UncompressedSize, Blob.data() + CurrentOffset, sizeof(uint32_t)); + CurrentOffset += UncompressedSizeFieldSize; + + uint64_t StoredHash; + memcpy(&StoredHash, Blob.data() + CurrentOffset, sizeof(uint64_t)); + CurrentOffset += HashFieldSize; + + llvm::compression::Format CompressionFormat; + if (CompressionMethod == + static_cast(llvm::compression::Format::Zlib)) + CompressionFormat = llvm::compression::Format::Zlib; + else if (CompressionMethod == + static_cast(llvm::compression::Format::Zstd)) + CompressionFormat = llvm::compression::Format::Zstd; + else + return createStringError(inconvertibleErrorCode(), + "Unknown compressing method"); + + llvm::Timer DecompressTimer("Decompression Timer", "Decompression time", + OffloadBundlerTimerGroup); + if (Verbose) + DecompressTimer.startTimer(); + + SmallVector DecompressedData; + StringRef CompressedData = Blob.substr(CurrentOffset); + if (llvm::Error DecompressionError = llvm::compression::decompress( + CompressionFormat, llvm::arrayRefFromStringRef(CompressedData), + DecompressedData, UncompressedSize)) + return createStringError(inconvertibleErrorCode(), + "Could not decompress embedded file contents: " + + llvm::toString(std::move(DecompressionError))); + + if (Verbose) { + DecompressTimer.stopTimer(); + + double DecompressionTimeSeconds = + DecompressTimer.getTotalTime().getWallTime(); + + // Recalculate MD5 hash for integrity check. + llvm::Timer HashRecalcTimer("Hash Recalculation Timer", + "Hash recalculation time", + OffloadBundlerTimerGroup); + HashRecalcTimer.startTimer(); + llvm::MD5 Hash; + llvm::MD5::MD5Result Result; + Hash.update(llvm::ArrayRef(DecompressedData.data(), + DecompressedData.size())); + Hash.final(Result); + uint64_t RecalculatedHash = Result.low(); + HashRecalcTimer.stopTimer(); + bool HashMatch = (StoredHash == RecalculatedHash); + + double CompressionRate = + static_cast(UncompressedSize) / CompressedData.size(); + double DecompressionSpeedMBs = + (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds; + + llvm::errs() << "Compressed bundle format version: " << ThisVersion << "\n"; + if (ThisVersion >= 2) + llvm::errs() << "Total file size (from header): " + << formatWithCommas(TotalFileSize) << " bytes\n"; + llvm::errs() << "Decompression method: " + << (CompressionFormat == llvm::compression::Format::Zlib + ? "zlib" + : "zstd") + << "\n" + << "Size before decompression: " + << formatWithCommas(CompressedData.size()) << " bytes\n" + << "Size after decompression: " + << formatWithCommas(UncompressedSize) << " bytes\n" + << "Compression rate: " + << llvm::format("%.2lf", CompressionRate) << "\n" + << "Compression ratio: " + << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n" + << "Decompression speed: " + << llvm::format("%.2lf MB/s", DecompressionSpeedMBs) << "\n" + << "Stored hash: " << llvm::format_hex(StoredHash, 16) << "\n" + << "Recalculated hash: " + << llvm::format_hex(RecalculatedHash, 16) << "\n" + << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n"; + } + + return llvm::MemoryBuffer::getMemBufferCopy( + llvm::toStringRef(DecompressedData)); +} + +llvm::Expected> +CompressedOffloadBundle::compress(llvm::compression::Params P, + const llvm::MemoryBuffer &Input, + bool Verbose) { + if (!llvm::compression::zstd::isAvailable() && + !llvm::compression::zlib::isAvailable()) + return createStringError(llvm::inconvertibleErrorCode(), + "Compression not supported"); + + llvm::Timer HashTimer("Hash Calculation Timer", "Hash calculation time", + OffloadBundlerTimerGroup); + if (Verbose) + HashTimer.startTimer(); + llvm::MD5 Hash; + llvm::MD5::MD5Result Result; + Hash.update(Input.getBuffer()); + Hash.final(Result); + uint64_t TruncatedHash = Result.low(); + if (Verbose) + HashTimer.stopTimer(); + + SmallVector CompressedBuffer; + auto BufferUint8 = llvm::ArrayRef( + reinterpret_cast(Input.getBuffer().data()), + Input.getBuffer().size()); + + llvm::Timer CompressTimer("Compression Timer", "Compression time", + OffloadBundlerTimerGroup); + if (Verbose) + CompressTimer.startTimer(); + llvm::compression::compress(P, BufferUint8, CompressedBuffer); + if (Verbose) + CompressTimer.stopTimer(); + + uint16_t CompressionMethod = static_cast(P.format); + uint32_t UncompressedSize = Input.getBuffer().size(); + uint32_t TotalFileSize = MagicNumber.size() + sizeof(TotalFileSize) + + sizeof(Version) + sizeof(CompressionMethod) + + sizeof(UncompressedSize) + sizeof(TruncatedHash) + + CompressedBuffer.size(); + + SmallVector FinalBuffer; + llvm::raw_svector_ostream OS(FinalBuffer); + OS << MagicNumber; + OS.write(reinterpret_cast(&Version), sizeof(Version)); + OS.write(reinterpret_cast(&CompressionMethod), + sizeof(CompressionMethod)); + OS.write(reinterpret_cast(&TotalFileSize), + sizeof(TotalFileSize)); + OS.write(reinterpret_cast(&UncompressedSize), + sizeof(UncompressedSize)); + OS.write(reinterpret_cast(&TruncatedHash), + sizeof(TruncatedHash)); + OS.write(reinterpret_cast(CompressedBuffer.data()), + CompressedBuffer.size()); + + if (Verbose) { + auto MethodUsed = + P.format == llvm::compression::Format::Zstd ? "zstd" : "zlib"; + double CompressionRate = + static_cast(UncompressedSize) / CompressedBuffer.size(); + double CompressionTimeSeconds = CompressTimer.getTotalTime().getWallTime(); + double CompressionSpeedMBs = + (UncompressedSize / (1024.0 * 1024.0)) / CompressionTimeSeconds; + + llvm::errs() << "Compressed bundle format version: " << Version << "\n" + << "Total file size (including headers): " + << formatWithCommas(TotalFileSize) << " bytes\n" + << "Compression method used: " << MethodUsed << "\n" + << "Compression level: " << P.level << "\n" + << "Binary size before compression: " + << formatWithCommas(UncompressedSize) << " bytes\n" + << "Binary size after compression: " + << formatWithCommas(CompressedBuffer.size()) << " bytes\n" + << "Compression rate: " + << llvm::format("%.2lf", CompressionRate) << "\n" + << "Compression ratio: " + << llvm::format("%.2lf%%", 100.0 / CompressionRate) << "\n" + << "Compression speed: " + << llvm::format("%.2lf MB/s", CompressionSpeedMBs) << "\n" + << "Truncated MD5 hash: " + << llvm::format_hex(TruncatedHash, 16) << "\n"; + } + return llvm::MemoryBuffer::getMemBufferCopy( + llvm::StringRef(FinalBuffer.data(), FinalBuffer.size())); +} diff --git a/llvm/test/tools/llvm-objdump/Offloading/fatbin.test b/llvm/test/tools/llvm-objdump/Offloading/fatbin.test new file mode 100644 index 0000000000000..c93e5692f583f --- /dev/null +++ b/llvm/test/tools/llvm-objdump/Offloading/fatbin.test @@ -0,0 +1,60 @@ +## Test that --offloading with a fatbin works correctly + +# REQUIRES: target={{.*-linux.*}} +# RUN: yaml2obj %s -o %t.elf +# RUN: llvm-objdump --offloading %t.elf +# RUN: llvm-objdump -d %t.elf:0.hipv4-amdgcn-amd-amdhsa--gfx908 | FileCheck %s + + +# CHECK: s_load_dword s7, s[4:5], 0x24 +# CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +# CHECK-NEXT: v_mov_b32_e32 v1, 0 +# CHECK-NEXT: s_waitcnt lgkmcnt(0) +# CHECK-NEXT: s_and_b32 s4, s7, 0xffff +# CHECK-NEXT: s_mul_i32 s6, s6, s4 +# CHECK-NEXT: v_add_u32_e32 v0, s6, v0 +# CHECK-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1] +# CHECK-NEXT: v_mov_b32_e32 v3, s3 +# CHECK-NEXT: v_add_co_u32_e32 v2, vcc, s2, v0 +# CHECK-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v1, vcc +# CHECK-NEXT: global_load_dword v2, v[2:3], off +# CHECK-NEXT: v_mov_b32_e32 v3, s1 +# CHECK-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0 +# CHECK-NEXT: v_addc_co_u32_e32 v1, vcc, v3, v1, vcc +# CHECK-NEXT: global_load_dword v3, v[0:1], off +# CHECK-NEXT: s_waitcnt vmcnt(0) +# CHECK-NEXT: v_add_u32_e32 v2, v3, v2 +# CHECK-NEXT: global_store_dword v[0:1], v2, off +# CHECK-NEXT: s_endpgm + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_X86_64 + Entry: 0x2041B0 +ProgramHeaders: + - Type: PT_PHDR + Flags: [ PF_R ] + VAddr: 0x200040 + Align: 0x8 + Offset: 0x40 + - Type: PT_GNU_STACK + Flags: [ PF_W, PF_R ] + Align: 0x0 + Offset: 0x0 +Sections: + - Name: .hip_fatbin + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x201000 + AddressAlign: 0x1000 + Content: 5F5F434C414E475F4F46464C4F41445F42554E444C455F5F0200000000000000001000000000000000000000000000001B00000000000000686F73742D7838365F36342D756E6B6E6F776E2D6C696E75782D2D0010000000000000D00F0000000000001F0000000000000068697076342D616D6467636E2D616D642D616D646873612D2D676678393038000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007F454C460201014003000000000000000300E0000100000000000000000000004000000000000000100C0000000000003005000040003800090040000F000D000600000004000000400000000000000040000000000000004000000000000000F801000000000000F80100000000000008000000000000000100000004000000000000000000000000000000000000000000000000000000C008000000000000C008000000000000001000000000000001000000050000000009000000000000001900000000000000190000000000006C000000000000006C00000000000000001000000000000001000000060000007009000000000000702900000000000070290000000000007000000000000000900600000000000000100000000000000100000006000000E009000000000000E039000000000000E039000000000000000000000000000001000000000000000010000000000000020000000600000070090000000000007029000000000000702900000000000070000000000000007000000000000000080000000000000052E574640400000070090000000000007029000000000000702900000000000070000000000000009006000000000000010000000000000051E57464060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000004000000380200000000000038020000000000003802000000000000340500000000000034050000000000000400000000000000070000001D05000020000000414D44475055000083AE616D646873612E6B65726E656C7391DE0012AB2E616770725F636F756E7400A52E61726773DC001085AE2E616464726573735F7370616365A6676C6F62616CA52E6E616D65AA415F642E636F65726365A72E6F666673657400A52E73697A6508AB2E76616C75655F6B696E64AD676C6F62616C5F62756666657285AE2E616464726573735F7370616365A6676C6F62616CA52E6E616D65AA425F642E636F65726365A72E6F666673657408A52E73697A6508AB2E76616C75655F6B696E64AD676C6F62616C5F62756666657284A52E6E616D65A14EA72E6F666673657410A52E73697A6508AB2E76616C75655F6B696E64A862795F76616C756583A72E6F666673657418A52E73697A6504AB2E76616C75655F6B696E64B468696464656E5F626C6F636B5F636F756E745F7883A72E6F66667365741CA52E73697A6504AB2E76616C75655F6B696E64B468696464656E5F626C6F636B5F636F756E745F7983A72E6F666673657420A52E73697A6504AB2E76616C75655F6B696E64B468696464656E5F626C6F636B5F636F756E745F7A83A72E6F666673657424A52E73697A6502AB2E76616C75655F6B696E64B368696464656E5F67726F75705F73697A655F7883A72E6F666673657426A52E73697A6502AB2E76616C75655F6B696E64B368696464656E5F67726F75705F73697A655F7983A72E6F666673657428A52E73697A6502AB2E76616C75655F6B696E64B368696464656E5F67726F75705F73697A655F7A83A72E6F66667365742AA52E73697A6502AB2E76616C75655F6B696E64B268696464656E5F72656D61696E6465725F7883A72E6F66667365742CA52E73697A6502AB2E76616C75655F6B696E64B268696464656E5F72656D61696E6465725F7983A72E6F66667365742EA52E73697A6502AB2E76616C75655F6B696E64B268696464656E5F72656D61696E6465725F7A83A72E6F666673657440A52E73697A6508AB2E76616C75655F6B696E64B668696464656E5F676C6F62616C5F6F66667365745F7883A72E6F666673657448A52E73697A6508AB2E76616C75655F6B696E64B668696464656E5F676C6F62616C5F6F66667365745F7983A72E6F666673657450A52E73697A6508AB2E76616C75655F6B696E64B668696464656E5F676C6F62616C5F6F66667365745F7A83A72E6F666673657458A52E73697A6502AB2E76616C75655F6B696E64B068696464656E5F677269645F64696D73B92E67726F75705F7365676D656E745F66697865645F73697A6500B62E6B65726E6172675F7365676D656E745F616C69676E08B52E6B65726E6172675F7365676D656E745F73697A65CD0118A92E6C616E6775616765A84F70656E434C2043B12E6C616E67756167655F76657273696F6E920200B82E6D61785F666C61745F776F726B67726F75705F73697A65CD0400A52E6E616D65B25F5A3973696D706C65416464506A504B6A6DBB2E707269766174655F7365676D656E745F66697865645F73697A6500AB2E736770725F636F756E740CB12E736770725F7370696C6C5F636F756E7400A72E73796D626F6CB55F5A3973696D706C65416464506A504B6A6D2E6B64B82E756E69666F726D5F776F726B5F67726F75705F73697A6501B32E757365735F64796E616D69635F737461636BC2AB2E766770725F636F756E7404B12E766770725F7370696C6C5F636F756E7400AF2E7761766566726F6E745F73697A6540AD616D646873612E746172676574B9616D6467636E2D616D642D616D646873612D2D676678393038AE616D646873612E76657273696F6E92010200000000000000000000000000000000000000000000000000000000000000010000001203070000190000000000006C000000000000001400000011030600800800000000000040000000000000002A00000011000A00E03900000000000001000000000000000100000001000000010000001A000000000008400000D20001000000360A4A7A5238A4D3F113F4DD04000000040000000200000001000000000000000300000000000000000000000000000000000000005F5A3973696D706C65416464506A504B6A6D005F5A3973696D706C65416464506A504B6A6D2E6B64005F5F6869705F637569645F623730363264386333326134613933330000000000000000000000000000000000000000000000000000000000000000000000180100000000000080100000000000000000000000000000000000000000000000000000000000004000AF008C000000090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000C20102C02400000002000AC0000000008002027E7FC08CBF07FF0486FFFF0000060406920600006800008FD2820002000302067E0200043203030638008050DC02007F020102067E0000003203030238008050DC00007F03700F8CBF03050468008070DC00027F00000081BF00000000060000000000000070070000000000000B000000000000001800000000000000050000000000000020080000000000000A000000000000004600000000000000F5FEFF6F00000000D0070000000000000400000000000000F807000000000000000000000000000000000000000000004C696E6B65723A20414D44204C4C442031392E302E3000414D4420636C616E672076657273696F6E2031392E302E306769742028202032343231322063393630313665636534313337356462646438663037356266333762643666633333323230376233290000414D4420636C616E672076657273696F6E2031382E302E3067697420287373683A2F2F6765727269746769742F6C696768746E696E672F65632F6C6C766D2D70726F6A65637420616D642D6D61696E6C696E652D6F70656E20323431373620663935303039613166393032313232343865313036333964653837653635636163616338643961372900000000000000000000000000000000000000000000000000460000000002080070290000000000000000000000000000010000001203070000190000000000006C000000000000001400000011030600800800000000000040000000000000002A00000011000A00E0390000000000000100000000000000002E6E6F7465002E64796E73796D002E676E752E68617368002E68617368002E64796E737472002E726F64617461002E74657874002E64796E616D6963002E72656C726F5F70616464696E67002E627373002E636F6D6D656E74002E73796D746162002E7368737472746162002E73747274616200005F5A3973696D706C65416464506A504B6A6D005F5A3973696D706C65416464506A504B6A6D2E6B64005F5F6869705F637569645F62373036326438633332613461393333005F44594E414D494300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000070000000200000000000000380200000000000038020000000000003405000000000000000000000000000004000000000000000000000000000000070000000B00000002000000000000007007000000000000700700000000000060000000000000000500000001000000080000000000000018000000000000000F000000F6FFFF6F0200000000000000D007000000000000D007000000000000280000000000000002000000000000000800000000000000000000000000000019000000050000000200000000000000F807000000000000F80700000000000028000000000000000200000000000000040000000000000004000000000000001F000000030000000200000000000000200800000000000020080000000000004600000000000000000000000000000001000000000000000000000000000000270000000100000002000000000000008008000000000000800800000000000040000000000000000000000000000000400000000000000000000000000000002F000000010000000600000000000000001900000000000000090000000000006C00000000000000000000000000000000010000000000000000000000000000350000000600000003000000000000007029000000000000700900000000000070000000000000000500000000000000080000000000000010000000000000003E000000080000000300000000000000E029000000000000E00900000000000020060000000000000000000000000000010000000000000000000000000000004D000000080000000300000000000000E039000000000000E0090000000000000100000000000000000000000000000001000000000000000000000000000000520000000100000030000000000000000000000000000000E009000000000000F0000000000000000000000000000000010000000000000001000000000000005B0000000200000000000000000000000000000000000000D00A00000000000078000000000000000E0000000200000008000000000000001800000000000000630000000300000000000000000000000000000000000000480B00000000000075000000000000000000000000000000010000000000000000000000000000006D0000000300000000000000000000000000000000000000BD0B0000000000004F00000000000000000000000000000001000000000000000000000000000000 + - Name: .hipFatBinSegment + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x202FD0 + AddressAlign: 0x8 + Content: '465049480100000000102000000000000000000000000000' +... diff --git a/llvm/tools/llvm-objdump/OffloadDump.cpp b/llvm/tools/llvm-objdump/OffloadDump.cpp index 4ac6b99e79bbb..654709a4ddff5 100644 --- a/llvm/tools/llvm-objdump/OffloadDump.cpp +++ b/llvm/tools/llvm-objdump/OffloadDump.cpp @@ -14,12 +14,16 @@ #include "OffloadDump.h" #include "llvm-objdump.h" #include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/OffloadBinary.h" +#include "llvm/Object/OffloadBundle.h" #include "llvm/Support/Alignment.h" using namespace llvm; using namespace llvm::object; using namespace llvm::objdump; +void disassembleObject(llvm::object::ObjectFile *, bool InlineRelocs); + /// Get the printable name of the image kind. static StringRef getImageName(const OffloadBinary &OB) { switch (OB.getImageKind()) { @@ -48,7 +52,7 @@ static void printBinary(const OffloadBinary &OB, uint64_t Index) { } /// Print the embedded offloading contents of an ObjectFile \p O. -void llvm::dumpOffloadBinary(const ObjectFile &O) { +void llvm::dumpOffloadBinary(const ObjectFile &O, StringRef ArchName) { if (!O.isELF() && !O.isCOFF()) { reportWarning( "--offloading is currently only supported for COFF and ELF targets", @@ -64,6 +68,46 @@ void llvm::dumpOffloadBinary(const ObjectFile &O) { // Print out all the binaries that are contained in this buffer. for (uint64_t I = 0, E = Binaries.size(); I != E; ++I) printBinary(*Binaries[I].getBinary(), I); + + dumpOffloadBundleFatBinary(O, ArchName); +} + +// Given an Object file, collect all Bundles of FatBin Binaries +// and dump them into Code Object files +// if -arch=-name is specified, only dump the Entries that match the target arch +void llvm::dumpOffloadBundleFatBinary(const ObjectFile &O, StringRef ArchName) { + if (!O.isELF() && !O.isCOFF()) { + reportWarning( + "--offloading is currently only supported for COFF and ELF targets", + O.getFileName()); + return; + } + + SmallVector FoundBundles; + SmallVector FoundEntries; + + if (Error Err = llvm::object::extractOffloadBundleFatBinary(O, FoundBundles)) + reportError(O.getFileName(), "while extracting offload FatBin bundles: " + + toString(std::move(Err))); + + for (const auto &[BundleNum, Bundle] : llvm::enumerate(FoundBundles)) { + if (!ArchName.empty()) + FoundEntries = Bundle.entryIDContains(ArchName); + else + FoundEntries = Bundle.getEntries(); + + for (OffloadBundleEntry &Entry : FoundEntries) { + // create file name for this object file: :. + std::string str = Bundle.getFileName().str() + ":" + itostr(BundleNum) + + "." + Entry.ID.str(); + if (Error Err = object::extractCodeObject(O, Entry.Offset, Entry.Size, + StringRef(str))) + reportError(O.getFileName(), + "while extracting offload Bundle Entries: " + + toString(std::move(Err))); + } + } } /// Print the contents of an offload binary file \p OB. This may contain diff --git a/llvm/tools/llvm-objdump/OffloadDump.h b/llvm/tools/llvm-objdump/OffloadDump.h index 75f188e9d5065..229d479ae357b 100644 --- a/llvm/tools/llvm-objdump/OffloadDump.h +++ b/llvm/tools/llvm-objdump/OffloadDump.h @@ -11,12 +11,16 @@ #include "llvm/Object/ObjectFile.h" #include "llvm/Object/OffloadBinary.h" +#include "llvm/Object/OffloadBundle.h" namespace llvm { void dumpOffloadSections(const object::OffloadBinary &OB); -void dumpOffloadBinary(const object::ObjectFile &O); +void dumpOffloadBinary(const object::ObjectFile &O, StringRef ArchName); +/// Dump fat binary in binary clang-offload-bundler format +void dumpOffloadBundleFatBinary(const object::ObjectFile &O, + StringRef ArchName); } // namespace llvm #endif diff --git a/llvm/tools/llvm-objdump/llvm-objdump.cpp b/llvm/tools/llvm-objdump/llvm-objdump.cpp index 93fed8ee8e6f4..05ca8ca91d47d 100644 --- a/llvm/tools/llvm-objdump/llvm-objdump.cpp +++ b/llvm/tools/llvm-objdump/llvm-objdump.cpp @@ -56,6 +56,7 @@ #include "llvm/Object/MachO.h" #include "llvm/Object/MachOUniversal.h" #include "llvm/Object/OffloadBinary.h" +#include "llvm/Object/OffloadBundle.h" #include "llvm/Object/Wasm.h" #include "llvm/Option/Arg.h" #include "llvm/Option/ArgList.h" @@ -3338,7 +3339,7 @@ static void dumpObject(ObjectFile *O, const Archive *A = nullptr, if (FaultMapSection) printFaultMaps(O); if (Offloading) - dumpOffloadBinary(*O); + dumpOffloadBinary(*O, StringRef(ArchName)); } static void dumpObject(const COFFImportFile *I, const Archive *A, diff --git a/llvm/unittests/Object/CMakeLists.txt b/llvm/unittests/Object/CMakeLists.txt index 81bc4a5577e68..1343352d1dc69 100644 --- a/llvm/unittests/Object/CMakeLists.txt +++ b/llvm/unittests/Object/CMakeLists.txt @@ -16,6 +16,7 @@ add_llvm_unittest(ObjectTests MinidumpTest.cpp ObjectFileTest.cpp OffloadingTest.cpp + OffloadingBundleTest.cpp SymbolSizeTest.cpp SymbolicFileTest.cpp XCOFFObjectFileTest.cpp diff --git a/llvm/unittests/Object/OffloadingBundleTest.cpp b/llvm/unittests/Object/OffloadingBundleTest.cpp new file mode 100644 index 0000000000000..923e7fbf48da4 --- /dev/null +++ b/llvm/unittests/Object/OffloadingBundleTest.cpp @@ -0,0 +1,89 @@ +// Skip running on Windows. +#if !defined(_WIN32) + +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/OffloadBinary.h" +#include "llvm/Object/OffloadBundle.h" +#include "llvm/ObjectYAML/yaml2obj.h" +#include "llvm/Support/FileOutputBuffer.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/YAMLTraits.h" +#include "llvm/Testing/Support/Error.h" +#include "gtest/gtest.h" +#include + +using namespace llvm; +using namespace llvm::object; + +StringRef simpleAdd = R"( +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_X86_64 + Entry: 0x2041B0 +Sections: + - Name: .hip_fatbin + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x201000 + AddressAlign: 0x1000 + Content: 5F5F434C414E475F4F46464C4F41445F42554E444C455F5F0200000000000000001000000000000000000000000000001B00000000000000686F73742D7838365F36342D756E6B6E6F776E2D6C696E75782D2D0010000000000000D00F0000000000001F0000000000000068697076342D616D6467636E2D616D642D616D646873612D2D676678393038000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007F454C460201014003000000000000000300E0000100000000000000000000004000000000000000100C0000000000003005000040003800090040000F000D000600000004000000400000000000000040000000000000004000000000000000F801000000000000F80100000000000008000000000000000100000004000000000000000000000000000000000000000000000000000000C008000000000000C008000000000000001000000000000001000000050000000009000000000000001900000000000000190000000000006C000000000000006C00000000000000001000000000000001000000060000007009000000000000702900000000000070290000000000007000000000000000900600000000000000100000000000000100000006000000E009000000000000E039000000000000E039000000000000000000000000000001000000000000000010000000000000020000000600000070090000000000007029000000000000702900000000000070000000000000007000000000000000080000000000000052E574640400000070090000000000007029000000000000702900000000000070000000000000009006000000000000010000000000000051E57464060000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000004000000380200000000000038020000000000003802000000000000340500000000000034050000000000000400000000000000070000001D05000020000000414D44475055000083AE616D646873612E6B65726E656C7391DE0012AB2E616770725F636F756E7400A52E61726773DC001085AE2E616464726573735F7370616365A6676C6F62616CA52E6E616D65AA415F642E636F65726365A72E6F666673657400A52E73697A6508AB2E76616C75655F6B696E64AD676C6F62616C5F62756666657285AE2E616464726573735F7370616365A6676C6F62616CA52E6E616D65AA425F642E636F65726365A72E6F666673657408A52E73697A6508AB2E76616C75655F6B696E64AD676C6F62616C5F62756666657284A52E6E616D65A14EA72E6F666673657410A52E73697A6508AB2E76616C75655F6B696E64A862795F76616C756583A72E6F666673657418A52E73697A6504AB2E76616C75655F6B696E64B468696464656E5F626C6F636B5F636F756E745F7883A72E6F66667365741CA52E73697A6504AB2E76616C75655F6B696E64B468696464656E5F626C6F636B5F636F756E745F7983A72E6F666673657420A52E73697A6504AB2E76616C75655F6B696E64B468696464656E5F626C6F636B5F636F756E745F7A83A72E6F666673657424A52E73697A6502AB2E76616C75655F6B696E64B368696464656E5F67726F75705F73697A655F7883A72E6F666673657426A52E73697A6502AB2E76616C75655F6B696E64B368696464656E5F67726F75705F73697A655F7983A72E6F666673657428A52E73697A6502AB2E76616C75655F6B696E64B368696464656E5F67726F75705F73697A655F7A83A72E6F66667365742AA52E73697A6502AB2E76616C75655F6B696E64B268696464656E5F72656D61696E6465725F7883A72E6F66667365742CA52E73697A6502AB2E76616C75655F6B696E64B268696464656E5F72656D61696E6465725F7983A72E6F66667365742EA52E73697A6502AB2E76616C75655F6B696E64B268696464656E5F72656D61696E6465725F7A83A72E6F666673657440A52E73697A6508AB2E76616C75655F6B696E64B668696464656E5F676C6F62616C5F6F66667365745F7883A72E6F666673657448A52E73697A6508AB2E76616C75655F6B696E64B668696464656E5F676C6F62616C5F6F66667365745F7983A72E6F666673657450A52E73697A6508AB2E76616C75655F6B696E64B668696464656E5F676C6F62616C5F6F66667365745F7A83A72E6F666673657458A52E73697A6502AB2E76616C75655F6B696E64B068696464656E5F677269645F64696D73B92E67726F75705F7365676D656E745F66697865645F73697A6500B62E6B65726E6172675F7365676D656E745F616C69676E08B52E6B65726E6172675F7365676D656E745F73697A65CD0118A92E6C616E6775616765A84F70656E434C2043B12E6C616E67756167655F76657273696F6E920200B82E6D61785F666C61745F776F726B67726F75705F73697A65CD0400A52E6E616D65B25F5A3973696D706C65416464506A504B6A6DBB2E707269766174655F7365676D656E745F66697865645F73697A6500AB2E736770725F636F756E740CB12E736770725F7370696C6C5F636F756E7400A72E73796D626F6CB55F5A3973696D706C65416464506A504B6A6D2E6B64B82E756E69666F726D5F776F726B5F67726F75705F73697A6501B32E757365735F64796E616D69635F737461636BC2AB2E766770725F636F756E7404B12E766770725F7370696C6C5F636F756E7400AF2E7761766566726F6E745F73697A6540AD616D646873612E746172676574B9616D6467636E2D616D642D616D646873612D2D676678393038AE616D646873612E76657273696F6E92010200000000000000000000000000000000000000000000000000000000000000010000001203070000190000000000006C000000000000001400000011030600800800000000000040000000000000002A00000011000A00E03900000000000001000000000000000100000001000000010000001A000000000008400000D20001000000360A4A7A5238A4D3F113F4DD04000000040000000200000001000000000000000300000000000000000000000000000000000000005F5A3973696D706C65416464506A504B6A6D005F5A3973696D706C65416464506A504B6A6D2E6B64005F5F6869705F637569645F623730363264386333326134613933330000000000000000000000000000000000000000000000000000000000000000000000180100000000000080100000000000000000000000000000000000000000000000000000000000004000AF008C000000090000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000C20102C02400000002000AC0000000008002027E7FC08CBF07FF0486FFFF0000060406920600006800008FD2820002000302067E0200043203030638008050DC02007F020102067E0000003203030238008050DC00007F03700F8CBF03050468008070DC00027F00000081BF00000000060000000000000070070000000000000B000000000000001800000000000000050000000000000020080000000000000A000000000000004600000000000000F5FEFF6F00000000D0070000000000000400000000000000F807000000000000000000000000000000000000000000004C696E6B65723A20414D44204C4C442031392E302E3000414D4420636C616E672076657273696F6E2031392E302E306769742028202032343231322063393630313665636534313337356462646438663037356266333762643666633333323230376233290000414D4420636C616E672076657273696F6E2031382E302E3067697420287373683A2F2F6765727269746769742F6C696768746E696E672F65632F6C6C766D2D70726F6A65637420616D642D6D61696E6C696E652D6F70656E20323431373620663935303039613166393032313232343865313036333964653837653635636163616338643961372900000000000000000000000000000000000000000000000000460000000002080070290000000000000000000000000000010000001203070000190000000000006C000000000000001400000011030600800800000000000040000000000000002A00000011000A00E0390000000000000100000000000000002E6E6F7465002E64796E73796D002E676E752E68617368002E68617368002E64796E737472002E726F64617461002E74657874002E64796E616D6963002E72656C726F5F70616464696E67002E627373002E636F6D6D656E74002E73796D746162002E7368737472746162002E73747274616200005F5A3973696D706C65416464506A504B6A6D005F5A3973696D706C65416464506A504B6A6D2E6B64005F5F6869705F637569645F62373036326438633332613461393333005F44594E414D494300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000070000000200000000000000380200000000000038020000000000003405000000000000000000000000000004000000000000000000000000000000070000000B00000002000000000000007007000000000000700700000000000060000000000000000500000001000000080000000000000018000000000000000F000000F6FFFF6F0200000000000000D007000000000000D007000000000000280000000000000002000000000000000800000000000000000000000000000019000000050000000200000000000000F807000000000000F80700000000000028000000000000000200000000000000040000000000000004000000000000001F000000030000000200000000000000200800000000000020080000000000004600000000000000000000000000000001000000000000000000000000000000270000000100000002000000000000008008000000000000800800000000000040000000000000000000000000000000400000000000000000000000000000002F000000010000000600000000000000001900000000000000090000000000006C00000000000000000000000000000000010000000000000000000000000000350000000600000003000000000000007029000000000000700900000000000070000000000000000500000000000000080000000000000010000000000000003E000000080000000300000000000000E029000000000000E00900000000000020060000000000000000000000000000010000000000000000000000000000004D000000080000000300000000000000E039000000000000E0090000000000000100000000000000000000000000000001000000000000000000000000000000520000000100000030000000000000000000000000000000E009000000000000F0000000000000000000000000000000010000000000000001000000000000005B0000000200000000000000000000000000000000000000D00A00000000000078000000000000000E0000000200000008000000000000001800000000000000630000000300000000000000000000000000000000000000480B00000000000075000000000000000000000000000000010000000000000000000000000000006D0000000300000000000000000000000000000000000000BD0B0000000000004F00000000000000000000000000000001000000000000000000000000000000 + - Name: .hipFatBinSegment + Type: SHT_PROGBITS + Flags: [ SHF_ALLOC ] + Address: 0x202FD0 + AddressAlign: 0x8 + Content: '465049480100000000102000000000000000000000000000' +)"; + +// ELF Object creation +static Expected> +toBinary(SmallVectorImpl &Storage, StringRef Yaml) { + raw_svector_ostream OS(Storage); + yaml::Input YIn(Yaml); + if (!yaml::convertYAML(YIn, OS, [](const Twine &Msg) {})) + return createStringError(std::errc::invalid_argument, + "unable to convert YAML"); + return object::ObjectFile::createELFObjectFile( + MemoryBufferRef(OS.str(), "dummyELF")); +} + +TEST(OffloadingBundleTest, checkExtractOffloadBundleFatBinary) { + + // create a Memory Buffer with a fatbin offloading section + MemoryBufferRef mbuf; + StringRef FileName; + SmallVector(); + SmallString<0> Storage; + // Expected> ObjOrErr = toBinary(Storage, R"( + Expected> ObjOrErr = toBinary(Storage, simpleAdd); + + ASSERT_THAT_EXPECTED(ObjOrErr, Succeeded()); + + SmallVector Bundles; + Error Err = extractOffloadBundleFatBinary(**ObjOrErr, Bundles); + EXPECT_FALSE(errorToBool(std::move(Err))); +} + +TEST(OffloadingBundleTest, checkExtractCodeObject) { + // create a Memory Buffer with a fatbin offloading section + MemoryBufferRef mbuf; + StringRef FileName; + SmallVector(); + SmallString<0> Storage; + // Expected> ObjOrErr = toBinary(Storage, R"( + Expected> ObjOrErr = toBinary(Storage, simpleAdd); + + ASSERT_THAT_EXPECTED(ObjOrErr, Succeeded()); + + int64_t Offset = 8192; + int64_t Size = 4048; + + Error Err = extractCodeObject(**ObjOrErr, Offset, Size, + StringRef("checkExtractCodeObject.co")); + EXPECT_FALSE(errorToBool(std::move(Err))); +} + +#endif