Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ca35a19

Browse files
hctimhctim
committed
[lld] Synthesize metadata for MTE globals
As per the ABI at https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst, this patch interprets the SHT_AARCH64_MEMTAG_GLOBALS_STATIC section, which contains R_NONE relocations to tagged globals, and emits a SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC section, with the correct DT_AARCH64_MEMTAG_GLOBALS and DT_AARCH64_MEMTAG_GLOBALSSZ dynamic entries. This section describes, in a uleb-encoded stream, global memory ranges that should be tagged with MTE. We are also out of bits to spare in the LLD Symbol class. As a result, I've reused the 'needsTocRestore' bit, which is a PPC64 only feature. Now, it's also used for 'isTagged' on AArch64. An entry in SHT_AARCH64_MEMTAG_GLOBALS_STATIC is practically a guarantee from an objfile that all references to the linked symbol are through the GOT, and meet correct alignment requirements. As a result, we go through all symbols and make sure that, for all symbols $SYM, all object files that reference $SYM also have a SHT_AARCH64_MEMTAG_GLOBALS_STATIC entry for $SYM. If this isn't the case, we demote the symbol to being untagged. Symbols that are imported from other DSOs should always be fine, as they're GOT-referenced (and thus the GOT entry either has the correct tag or not, depending on whether it's tagged in the defining DSO or not). Additionally hand-tested by building {libc, libm, libc++, libm, and libnetd} on Android with some experimental MTE globals support in the linker/libc. Reviewed By: MaskRay, peter.smith Differential Revision: https://reviews.llvm.org/D152921
1 parent 8a677a7 commit ca35a19

15 files changed

Lines changed: 870 additions & 33 deletions

‎lld/ELF/Arch/AArch64.cpp‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
//
77
//===----------------------------------------------------------------------===//
88

9+
#include "InputFiles.h"
910
#include "OutputSections.h"
1011
#include "Symbols.h"
1112
#include "SyntheticSections.h"
@@ -377,6 +378,20 @@ void AArch64::relocate(uint8_t *loc, const Relocation &rel,
377378
write32(loc, val);
378379
break;
379380
case R_AARCH64_ABS64:
381+
// AArch64 relocations to tagged symbols have extended semantics, as
382+
// described here:
383+
// https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#841extended-semantics-of-r_aarch64_relative.
384+
// tl;dr: encode the symbol's special addend in the place, which is an
385+
// offset to the point where the logical tag is derived from. Quick hack, if
386+
// the addend is within the symbol's bounds, no need to encode the tag
387+
// derivation offset.
388+
if (rel.sym && rel.sym->isTagged() &&
389+
(rel.addend < 0 ||
390+
rel.addend >= static_cast<int64_t>(rel.sym->getSize())))
391+
write64(loc, -rel.addend);
392+
else
393+
write64(loc, val);
394+
break;
380395
case R_AARCH64_PREL64:
381396
write64(loc, val);
382397
break;
@@ -745,6 +760,12 @@ bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel,
745760
return true;
746761
}
747762

763+
// Tagged symbols have upper address bits that are added by the dynamic loader,
764+
// and thus need the full 64-bit GOT entry. Do not relax such symbols.
765+
static bool needsGotForMemtag(const Relocation &rel) {
766+
return rel.sym->isTagged() && needsGot(rel.expr);
767+
}
768+
748769
void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
749770
uint64_t secAddr = sec.getOutputSection()->addr;
750771
if (auto *s = dyn_cast<InputSection>(&sec))
@@ -756,6 +777,12 @@ void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
756777
const uint64_t val =
757778
sec.getRelocTargetVA(sec.file, rel.type, rel.addend,
758779
secAddr + rel.offset, *rel.sym, rel.expr);
780+
781+
if (needsGotForMemtag(rel)) {
782+
relocate(loc, rel, val);
783+
continue;
784+
}
785+
759786
switch (rel.expr) {
760787
case R_AARCH64_GOT_PAGE_PC:
761788
if (i + 1 < size &&
@@ -950,3 +977,107 @@ static TargetInfo *getTargetInfo() {
950977
}
951978

952979
TargetInfo *elf::getAArch64TargetInfo() { return getTargetInfo(); }
980+
981+
template <class ELFT>
982+
static void
983+
addTaggedSymbolReferences(InputSectionBase &sec,
984+
DenseMap<Symbol *, unsigned> &referenceCount) {
985+
assert(sec.type == SHT_AARCH64_MEMTAG_GLOBALS_STATIC);
986+
987+
const RelsOrRelas<ELFT> rels = sec.relsOrRelas<ELFT>();
988+
if (rels.areRelocsRel())
989+
error("non-RELA relocations are not allowed with memtag globals");
990+
991+
for (const typename ELFT::Rela &rel : rels.relas) {
992+
Symbol &sym = sec.getFile<ELFT>()->getRelocTargetSym(rel);
993+
// Linker-synthesized symbols such as __executable_start may be referenced
994+
// as tagged in input objfiles, and we don't want them to be tagged. A
995+
// cheap way to exclude them is the type check, but their type is
996+
// STT_NOTYPE. In addition, this save us from checking untaggable symbols,
997+
// like functions or TLS symbols.
998+
if (sym.type != STT_OBJECT)
999+
continue;
1000+
// STB_LOCAL symbols can't be referenced from outside the object file, and
1001+
// thus don't need to be checked for references from other object files.
1002+
if (sym.binding == STB_LOCAL) {
1003+
sym.setIsTagged(true);
1004+
continue;
1005+
}
1006+
++referenceCount[&sym];
1007+
}
1008+
sec.markDead();
1009+
}
1010+
1011+
// A tagged symbol must be denoted as being tagged by all references and the
1012+
// chosen definition. For simplicity, here, it must also be denoted as tagged
1013+
// for all definitions. Otherwise:
1014+
//
1015+
// 1. A tagged definition can be used by an untagged declaration, in which case
1016+
// the untagged access may be PC-relative, causing a tag mismatch at
1017+
// runtime.
1018+
// 2. An untagged definition can be used by a tagged declaration, where the
1019+
// compiler has taken advantage of the increased alignment of the tagged
1020+
// declaration, but the alignment at runtime is wrong, causing a fault.
1021+
//
1022+
// Ideally, this isn't a problem, as any TU that imports or exports tagged
1023+
// symbols should also be built with tagging. But, to handle these cases, we
1024+
// demote the symbol to be untagged.
1025+
void lld::elf::createTaggedSymbols(const SmallVector<ELFFileBase *, 0> &files) {
1026+
assert(config->emachine == EM_AARCH64 &&
1027+
config->androidMemtagMode != ELF::NT_MEMTAG_LEVEL_NONE);
1028+
1029+
// First, collect all symbols that are marked as tagged, and count how many
1030+
// times they're marked as tagged.
1031+
DenseMap<Symbol *, unsigned> taggedSymbolReferenceCount;
1032+
for (InputFile* file : files) {
1033+
if (file->kind() != InputFile::ObjKind)
1034+
continue;
1035+
for (InputSectionBase *section : file->getSections()) {
1036+
if (!section || section->type != SHT_AARCH64_MEMTAG_GLOBALS_STATIC ||
1037+
section == &InputSection::discarded)
1038+
continue;
1039+
invokeELFT(addTaggedSymbolReferences, *section,
1040+
taggedSymbolReferenceCount);
1041+
}
1042+
}
1043+
1044+
// Now, go through all the symbols. If the number of declarations +
1045+
// definitions to a symbol exceeds the amount of times they're marked as
1046+
// tagged, it means we have an objfile that uses the untagged variant of the
1047+
// symbol.
1048+
for (InputFile *file : files) {
1049+
if (file->kind() != InputFile::BinaryKind &&
1050+
file->kind() != InputFile::ObjKind)
1051+
continue;
1052+
1053+
for (Symbol *symbol : file->getSymbols()) {
1054+
// See `addTaggedSymbolReferences` for more details.
1055+
if (symbol->type != STT_OBJECT ||
1056+
symbol->binding == STB_LOCAL)
1057+
continue;
1058+
auto it = taggedSymbolReferenceCount.find(symbol);
1059+
if (it == taggedSymbolReferenceCount.end()) continue;
1060+
unsigned &remainingAllowedTaggedRefs = it->second;
1061+
if (remainingAllowedTaggedRefs == 0) {
1062+
taggedSymbolReferenceCount.erase(it);
1063+
continue;
1064+
}
1065+
--remainingAllowedTaggedRefs;
1066+
}
1067+
}
1068+
1069+
// `addTaggedSymbolReferences` has already checked that we have RELA
1070+
// relocations, the only other way to get written addends is with
1071+
// --apply-dynamic-relocs.
1072+
if (!taggedSymbolReferenceCount.empty() && config->writeAddends)
1073+
error("--apply-dynamic-relocs cannot be used with MTE globals");
1074+
1075+
// Now, `taggedSymbolReferenceCount` should only contain symbols that are
1076+
// defined as tagged exactly the same amount as it's referenced, meaning all
1077+
// uses are tagged.
1078+
for (auto &[symbol, remainingTaggedRefs] : taggedSymbolReferenceCount) {
1079+
assert(remainingTaggedRefs == 0 &&
1080+
"Symbol is defined as tagged more times than it's used");
1081+
symbol->setIsTagged(true);
1082+
}
1083+
}

‎lld/ELF/Arch/PPC64.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1556,7 +1556,7 @@ void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const {
15561556
break;
15571557

15581558
// Patch a nop (0x60000000) to a ld.
1559-
if (rel.sym->needsTocRestore) {
1559+
if (rel.sym->needsTocRestore()) {
15601560
// gcc/gfortran 5.4, 6.3 and earlier versions do not add nop for
15611561
// recursive calls even if the function is preemptible. This is not
15621562
// wrong in the common case where the function is not preempted at

‎lld/ELF/Driver.cpp‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -786,13 +786,6 @@ static int getMemtagMode(opt::InputArgList &args) {
786786
return ELF::NT_MEMTAG_LEVEL_NONE;
787787
}
788788

789-
if (!config->androidMemtagHeap && !config->androidMemtagStack) {
790-
error("when using --android-memtag-mode, at least one of "
791-
"--android-memtag-heap or "
792-
"--android-memtag-stack is required");
793-
return ELF::NT_MEMTAG_LEVEL_NONE;
794-
}
795-
796789
if (memtagModeArg == "sync")
797790
return ELF::NT_MEMTAG_LEVEL_SYNC;
798791
if (memtagModeArg == "async")
@@ -2940,6 +2933,12 @@ void LinkerDriver::link(opt::InputArgList &args) {
29402933
// partition.
29412934
copySectionsIntoPartitions();
29422935

2936+
if (config->emachine == EM_AARCH64 &&
2937+
config->androidMemtagMode != ELF::NT_MEMTAG_LEVEL_NONE) {
2938+
llvm::TimeTraceScope timeScope("Process memory tagged symbols");
2939+
createTaggedSymbols(ctx.objectFiles);
2940+
}
2941+
29432942
// Create synthesized sections such as .got and .plt. This is called before
29442943
// processSectionCommands() so that they can be placed by SECTIONS commands.
29452944
invokeELFT(createSyntheticSections,);

‎lld/ELF/Relocations.cpp‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
#include "lld/Common/ErrorHandler.h"
5454
#include "lld/Common/Memory.h"
5555
#include "llvm/ADT/SmallSet.h"
56+
#include "llvm/BinaryFormat/ELF.h"
5657
#include "llvm/Demangle/Demangle.h"
5758
#include "llvm/Support/Endian.h"
5859
#include <algorithm>
@@ -199,10 +200,7 @@ static bool needsPlt(RelExpr expr) {
199200
R_PPC32_PLTREL, R_PPC64_CALL_PLT>(expr);
200201
}
201202

202-
// Returns true if Expr refers a GOT entry. Note that this function
203-
// returns false for TLS variables even though they need GOT, because
204-
// TLS variables uses GOT differently than the regular variables.
205-
static bool needsGot(RelExpr expr) {
203+
bool lld::elf::needsGot(RelExpr expr) {
206204
return oneof<R_GOT, R_GOT_OFF, R_MIPS_GOT_LOCAL_PAGE, R_MIPS_GOT_OFF,
207205
R_MIPS_GOT_OFF32, R_AARCH64_GOT_PAGE_PC, R_GOT_PC, R_GOTPLT,
208206
R_AARCH64_GOT_PAGE, R_LOONGARCH_GOT, R_LOONGARCH_GOT_PAGE_PC>(
@@ -859,6 +857,23 @@ static void addRelativeReloc(InputSectionBase &isec, uint64_t offsetInSec,
859857
RelType type) {
860858
Partition &part = isec.getPartition();
861859

860+
if (sym.isTagged()) {
861+
std::lock_guard<std::mutex> lock(relocMutex);
862+
part.relaDyn->addRelativeReloc(target->relativeRel, isec, offsetInSec, sym,
863+
addend, type, expr);
864+
// With MTE globals, we always want to derive the address tag by `ldg`-ing
865+
// the symbol. When we have a RELATIVE relocation though, we no longer have
866+
// a reference to the symbol. Because of this, when we have an addend that
867+
// puts the result of the RELATIVE relocation out-of-bounds of the symbol
868+
// (e.g. the addend is outside of [0, sym.getSize()]), the AArch64 MemtagABI
869+
// says we should store the offset to the start of the symbol in the target
870+
// field. This is described in further detail in:
871+
// https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#841extended-semantics-of-r_aarch64_relative
872+
if (addend < 0 || static_cast<uint64_t>(addend) >= sym.getSize())
873+
isec.relocations.push_back({expr, type, offsetInSec, addend, &sym});
874+
return;
875+
}
876+
862877
// Add a relative relocation. If relrDyn section is enabled, and the
863878
// relocation offset is guaranteed to be even, add the relocation to
864879
// the relrDyn section, otherwise add it to the relaDyn section.
@@ -1645,6 +1660,10 @@ void elf::postScanRelocations() {
16451660
auto flags = sym.flags.load(std::memory_order_relaxed);
16461661
if (handleNonPreemptibleIfunc(sym, flags))
16471662
return;
1663+
1664+
if (sym.isTagged() && sym.isDefined())
1665+
mainPart->memtagDescriptors->addSymbol(sym);
1666+
16481667
if (!sym.needsDynReloc())
16491668
return;
16501669
sym.allocateAux();

‎lld/ELF/Relocations.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,11 @@ ArrayRef<RelTy> sortRels(ArrayRef<RelTy> rels, SmallVector<RelTy, 0> &storage) {
220220
}
221221
return rels;
222222
}
223+
224+
// Returns true if Expr refers a GOT entry. Note that this function returns
225+
// false for TLS variables even though they need GOT, because TLS variables uses
226+
// GOT differently than the regular variables.
227+
bool needsGot(RelExpr expr);
223228
} // namespace lld::elf
224229

225230
#endif

‎lld/ELF/Symbols.h‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -254,8 +254,8 @@ class Symbol {
254254
Symbol(Kind k, InputFile *file, StringRef name, uint8_t binding,
255255
uint8_t stOther, uint8_t type)
256256
: file(file), nameData(name.data()), nameSize(name.size()), type(type),
257-
binding(binding), stOther(stOther), symbolKind(k),
258-
exportDynamic(false) {}
257+
binding(binding), stOther(stOther), symbolKind(k), exportDynamic(false),
258+
archSpecificBit(false) {}
259259

260260
void overwrite(Symbol &sym, Kind k) const {
261261
if (sym.traced)
@@ -279,9 +279,18 @@ class Symbol {
279279
// True if defined relative to a section discarded by ICF.
280280
uint8_t folded : 1;
281281

282-
// True if a call to this symbol needs to be followed by a restore of the
283-
// PPC64 toc pointer.
284-
uint8_t needsTocRestore : 1;
282+
// Allow reuse of a bit between architecture-exclusive symbol flags.
283+
// - needsTocRestore(): On PPC64, true if a call to this symbol needs to be
284+
// followed by a restore of the toc pointer.
285+
// - isTagged(): On AArch64, true if the symbol needs special relocation and
286+
// metadata semantics because it's tagged, under the AArch64 MemtagABI.
287+
uint8_t archSpecificBit : 1;
288+
bool needsTocRestore() const { return archSpecificBit; }
289+
bool isTagged() const { return archSpecificBit; }
290+
void setNeedsTocRestore(bool v) { archSpecificBit = v; }
291+
void setIsTagged(bool v) {
292+
archSpecificBit = v;
293+
}
285294

286295
// True if this symbol is defined by a symbol assignment or wrapped by --wrap.
287296
//

‎lld/ELF/SyntheticSections.cpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1453,6 +1453,10 @@ DynamicSection<ELFT>::computeContents() {
14531453
addInt(DT_AARCH64_MEMTAG_MODE, config->androidMemtagMode == NT_MEMTAG_LEVEL_ASYNC);
14541454
addInt(DT_AARCH64_MEMTAG_HEAP, config->androidMemtagHeap);
14551455
addInt(DT_AARCH64_MEMTAG_STACK, config->androidMemtagStack);
1456+
if (mainPart->memtagDescriptors->isNeeded()) {
1457+
addInSec(DT_AARCH64_MEMTAG_GLOBALS, *mainPart->memtagDescriptors);
1458+
addInt(DT_AARCH64_MEMTAG_GLOBALSSZ, mainPart->memtagDescriptors->getSize());
1459+
}
14561460
}
14571461
}
14581462

@@ -3900,6 +3904,76 @@ size_t PackageMetadataNote::getSize() const {
39003904
alignTo(config->packageMetadata.size() + 1, 4);
39013905
}
39023906

3907+
// Helper function, return the size of the ULEB128 for 'v', optionally writing
3908+
// it to `*(buf + offset)` if `buf` is non-null.
3909+
static size_t computeOrWriteULEB128(uint64_t v, uint8_t *buf, size_t offset) {
3910+
if (buf)
3911+
return encodeULEB128(v, buf + offset);
3912+
return getULEB128Size(v);
3913+
}
3914+
3915+
// https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#83encoding-of-sht_aarch64_memtag_globals_dynamic
3916+
constexpr uint64_t kMemtagStepSizeBits = 3;
3917+
constexpr uint64_t kMemtagGranuleSize = 16;
3918+
static size_t createMemtagDescriptors(const SmallVector<const Symbol *, 0> &symbols,
3919+
uint8_t *buf = nullptr) {
3920+
size_t sectionSize = 0;
3921+
uint64_t lastGlobalEnd = 0;
3922+
3923+
for (const Symbol *sym : symbols) {
3924+
if (!includeInSymtab(*sym))
3925+
continue;
3926+
const uint64_t addr = sym->getVA();
3927+
const uint64_t size = sym->getSize();
3928+
3929+
if (addr <= kMemtagGranuleSize && buf != nullptr)
3930+
errorOrWarn("address of the tagged symbol \"" + sym->getName() +
3931+
"\" falls in the ELF header. This is indicative of a "
3932+
"compiler/linker bug");
3933+
if (addr % kMemtagGranuleSize != 0)
3934+
errorOrWarn("address of the tagged symbol \"" + sym->getName() +
3935+
"\" at 0x" + Twine::utohexstr(addr) +
3936+
"\" is not granule (16-byte) aligned");
3937+
if (size == 0)
3938+
errorOrWarn("size of the tagged symbol \"" + sym->getName() +
3939+
"\" is not allowed to be zero");
3940+
if (size % kMemtagGranuleSize != 0)
3941+
errorOrWarn("size of the tagged symbol \"" + sym->getName() +
3942+
"\" (size 0x" + Twine::utohexstr(size) +
3943+
") is not granule (16-byte) aligned");
3944+
3945+
const uint64_t sizeToEncode = size / kMemtagGranuleSize;
3946+
const uint64_t stepToEncode = ((addr - lastGlobalEnd) / kMemtagGranuleSize)
3947+
<< kMemtagStepSizeBits;
3948+
if (sizeToEncode < (1 << kMemtagStepSizeBits)) {
3949+
sectionSize += computeOrWriteULEB128(stepToEncode | sizeToEncode, buf, sectionSize);
3950+
} else {
3951+
sectionSize += computeOrWriteULEB128(stepToEncode, buf, sectionSize);
3952+
sectionSize += computeOrWriteULEB128(sizeToEncode - 1, buf, sectionSize);
3953+
}
3954+
lastGlobalEnd = addr + size;
3955+
}
3956+
3957+
return sectionSize;
3958+
}
3959+
3960+
bool MemtagDescriptors::updateAllocSize() {
3961+
size_t oldSize = getSize();
3962+
std::stable_sort(symbols.begin(), symbols.end(),
3963+
[](const Symbol *s1, const Symbol *s2) {
3964+
return s1->getVA() < s2->getVA();
3965+
});
3966+
return oldSize != getSize();
3967+
}
3968+
3969+
void MemtagDescriptors::writeTo(uint8_t *buf) {
3970+
createMemtagDescriptors(symbols, buf);
3971+
}
3972+
3973+
size_t MemtagDescriptors::getSize() const {
3974+
return createMemtagDescriptors(symbols);
3975+
}
3976+
39033977
InStruct elf::in;
39043978

39053979
std::vector<Partition> elf::partitions;

0 commit comments

Comments
 (0)