Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 38faf32

Browse files
alekuz01alekuz01
authored
[BOLT] Enable hugify for AArch64 (#117158)
Add required hugify instrumentation and runtime libraries support for AArch64. Fixes #58226 Unblocks #62695
1 parent 7eae1a4 commit 38faf32

File tree

7 files changed

+67
-21
lines changed

7 files changed

+67
-21
lines changed

‎bolt/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ if (LLVM_INCLUDE_TESTS)
136136
endif()
137137

138138
if (BOLT_ENABLE_RUNTIME)
139-
message(STATUS "Building BOLT runtime libraries for X86")
139+
message(STATUS "Building BOLT runtime libraries for ${CMAKE_SYSTEM_PROCESSOR}")
140140
set(extra_args "")
141141
if(CMAKE_SYSROOT)
142142
list(APPEND extra_args -DCMAKE_SYSROOT=${CMAKE_SYSROOT})

‎bolt/lib/Rewrite/RewriteInstance.cpp

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -597,8 +597,9 @@ Error RewriteInstance::discoverStorage() {
597597

598598
// Hugify: Additional huge page from left side due to
599599
// weird ASLR mapping addresses (4KB aligned)
600-
if (opts::Hugify && !BC->HasFixedLoadAddress)
600+
if (opts::Hugify && !BC->HasFixedLoadAddress) {
601601
NextAvailableAddress += BC->PageAlign;
602+
}
602603

603604
if (!opts::UseGnuStack && !BC->IsLinuxKernel) {
604605
// This is where the black magic happens. Creating PHDR table in a segment
@@ -5885,17 +5886,28 @@ void RewriteInstance::rewriteFile() {
58855886

58865887
// Write all allocatable sections - reloc-mode text is written here as well
58875888
for (BinarySection &Section : BC->allocatableSections()) {
5888-
if (!Section.isFinalized() || !Section.getOutputData())
5889+
if (!Section.isFinalized() || !Section.getOutputData()) {
5890+
LLVM_DEBUG(if (opts::Verbosity > 1) {
5891+
dbgs() << "BOLT-INFO: new section is finalized or !getOutputData, skip "
5892+
<< Section.getName() << '\n';
5893+
});
58895894
continue;
5890-
if (Section.isLinkOnly())
5895+
}
5896+
if (Section.isLinkOnly()) {
5897+
LLVM_DEBUG(if (opts::Verbosity > 1) {
5898+
dbgs() << "BOLT-INFO: new section is link only, skip "
5899+
<< Section.getName() << '\n';
5900+
});
58915901
continue;
5902+
}
58925903

58935904
if (opts::Verbosity >= 1)
58945905
BC->outs() << "BOLT: writing new section " << Section.getName()
58955906
<< "\n data at 0x"
58965907
<< Twine::utohexstr(Section.getAllocAddress()) << "\n of size "
58975908
<< Section.getOutputSize() << "\n at offset "
5898-
<< Section.getOutputFileOffset() << '\n';
5909+
<< Section.getOutputFileOffset() << " with content size "
5910+
<< Section.getOutputContents().size() << '\n';
58995911
OS.seek(Section.getOutputFileOffset());
59005912
Section.write(OS);
59015913
}

‎bolt/runtime/common.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,10 +151,12 @@ struct timespec {
151151
uint64_t tv_nsec; /* nanoseconds */
152152
};
153153

154-
#if defined(__aarch64__)
154+
#if defined(__aarch64__) || defined(__arm64__)
155155
#include "sys_aarch64.h"
156-
#else
156+
#elif defined(__x86_64__)
157157
#include "sys_x86_64.h"
158+
#else
159+
#error "For AArch64/ARM64 and X86_64 only."
158160
#endif
159161

160162
constexpr uint32_t BufSize = 10240;

‎bolt/runtime/hugify.cpp

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
//
77
//===---------------------------------------------------------------------===//
88

9-
#if defined (__x86_64__) && !defined(__APPLE__)
9+
#if defined(__x86_64__) || \
10+
(defined(__aarch64__) || defined(__arm64__)) && !defined(__APPLE__)
1011

1112
#include "common.h"
1213

@@ -73,8 +74,10 @@ static bool hasPagecacheTHPSupport() {
7374
if (Res < 0)
7475
return false;
7576

76-
if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]"))
77+
if (!strStr(Buf, "[always]") && !strStr(Buf, "[madvise]")) {
78+
DEBUG(report("[hugify] THP support is not enabled.\n");)
7779
return false;
80+
}
7881

7982
struct KernelVersionTy {
8083
uint32_t major;
@@ -167,12 +170,20 @@ extern "C" void __bolt_hugify_self_impl() {
167170

168171
/// This is hooking ELF's entry, it needs to save all machine state.
169172
extern "C" __attribute((naked)) void __bolt_hugify_self() {
173+
// clang-format off
170174
#if defined(__x86_64__)
171175
__asm__ __volatile__(SAVE_ALL "call __bolt_hugify_self_impl\n" RESTORE_ALL
172-
"jmp __bolt_hugify_start_program\n" ::
173-
:);
176+
"jmp __bolt_hugify_start_program\n"
177+
:::);
178+
#elif defined(__aarch64__) || defined(__arm64__)
179+
__asm__ __volatile__(SAVE_ALL "bl __bolt_hugify_self_impl\n" RESTORE_ALL
180+
"adrp x16, __bolt_hugify_start_program\n"
181+
"add x16, x16, #:lo12:__bolt_hugify_start_program\n"
182+
"br x16\n"
183+
:::);
174184
#else
175-
exit(1);
185+
__exit(1);
176186
#endif
187+
// clang-format on
177188
}
178189
#endif

‎bolt/test/runtime/X86/Inputs/user_func_order.txt renamed to ‎bolt/test/runtime/Inputs/user_func_order.txt

File renamed without changes.

‎bolt/test/runtime/X86/hugify.c renamed to ‎bolt/test/runtime/hugify.c

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,28 @@ int main(int argc, char **argv) {
1111
REQUIRES: system-linux,bolt-runtime
1212
1313
RUN: %clang %cflags -no-pie %s -o %t.nopie.exe -Wl,-q
14-
RUN: %clang %cflags -fpic -pie %s -o %t.pie.exe -Wl,-q
14+
RUN: %clang %cflags -fpic %s -o %t.pie.exe -Wl,-q
1515
1616
RUN: llvm-bolt %t.nopie.exe --lite=0 -o %t.nopie --hugify
1717
RUN: llvm-bolt %t.pie.exe --lite=0 -o %t.pie --hugify
1818
19+
RUN: llvm-nm --numeric-sort --print-armap %t.nopie | \
20+
RUN: FileCheck %s -check-prefix=CHECK-NM
1921
RUN: %t.nopie | FileCheck %s -check-prefix=CHECK-NOPIE
2022
21-
CHECK-NOPIE: Hello world
22-
23+
RUN: llvm-nm --numeric-sort --print-armap %t.pie | \
24+
RUN: FileCheck %s -check-prefix=CHECK-NM
2325
RUN: %t.pie | FileCheck %s -check-prefix=CHECK-PIE
2426
27+
CHECK-NM: W __hot_start
28+
CHECK-NM-NEXT: T _start
29+
CHECK-NM: T main
30+
CHECK-NM: W __hot_end
31+
CHECK-NM: t __bolt_hugify_start_program
32+
CHECK-NM-NEXT: W __bolt_runtime_start
33+
34+
CHECK-NOPIE: Hello world
35+
2536
CHECK-PIE: Hello world
2637
2738
*/

‎bolt/test/runtime/X86/user-func-reorder.c renamed to ‎bolt/test/runtime/user-func-reorder.c

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,15 @@
55
*/
66
#include <stdio.h>
77

8-
int foo(int x) {
9-
return x + 1;
10-
}
8+
int foo(int x) { return x + 1; }
119

1210
int fib(int x) {
1311
if (x < 2)
1412
return x;
1513
return fib(x - 1) + fib(x - 2);
1614
}
1715

18-
int bar(int x) {
19-
return x - 1;
20-
}
16+
int bar(int x) { return x - 1; }
2117

2218
int main(int argc, char **argv) {
2319
printf("fib(%d) = %d\n", argc, fib(argc));
@@ -31,14 +27,28 @@ RUN: %clang %cflags -no-pie %s -o %t.exe -Wl,-q
3127
3228
RUN: llvm-bolt %t.exe --relocs=1 --lite --reorder-functions=user \
3329
RUN: --hugify --function-order=%p/Inputs/user_func_order.txt -o %t
30+
RUN: llvm-bolt %t.exe --relocs=1 --lite --reorder-functions=user \
31+
RUN: --function-order=%p/Inputs/user_func_order.txt -o %t.nohugify
3432
RUN: llvm-nm --numeric-sort --print-armap %t | \
3533
RUN: FileCheck %s -check-prefix=CHECK-NM
3634
RUN: %t 1 2 3 | FileCheck %s -check-prefix=CHECK-OUTPUT
35+
RUN: llvm-nm --numeric-sort --print-armap %t.nohugify | \
36+
RUN: FileCheck %s -check-prefix=CHECK-NM-NOHUGIFY
37+
RUN: %t.nohugify 1 2 3 | FileCheck %s -check-prefix=CHECK-OUTPUT-NOHUGIFY
38+
3739
3840
CHECK-NM: W __hot_start
3941
CHECK-NM: T main
4042
CHECK-NM-NEXT: T fib
4143
CHECK-NM-NEXT: W __hot_end
44+
CHECK-NM: t __bolt_hugify_start_program
45+
CHECK-NM-NEXT: W __bolt_runtime_start
46+
47+
CHECK-NM-NOHUGIFY: W __hot_start
48+
CHECK-NM-NOHUGIFY: T main
49+
CHECK-NM-NOHUGIFY-NEXT: T fib
50+
CHECK-NM-NOHUGIFY-NEXT: W __hot_end
4251
4352
CHECK-OUTPUT: fib(4) = 3
53+
CHECK-OUTPUT-NOHUGIFY: fib(4) = 3
4454
*/

0 commit comments

Comments
 (0)