Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e45ea95

Browse files
rosefromthedeadrosefromthedead
authored
[DTLTO] support distributing bitcode from FatLTO objects (#176928)
We already have code to extract bitcode files from archives so they can be distributed. Extend this code to extract bitcode from FatLTO objects too, which otherwise cannot be used with DTLTO.
1 parent 3db365d commit e45ea95

5 files changed

Lines changed: 93 additions & 23 deletions

File tree

‎cross-project-tests/dtlto/fat-lto-objects.test‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
REQUIRES: ld.lld,llvm-ar
2+
3+
# Test that a DTLTO link succeeds and outputs the expected set of files
4+
# correctly when FatLTO objects are present.
5+
RUN: rm -rf %t && split-file %s %t && cd %t
6+
7+
# Compile bitcode. -O2 is required for cross-module importing.
8+
RUN: %clang -O2 --target=x86_64-linux-gnu -flto=thin -ffat-lto-objects -c \
9+
RUN: foo.c boo.c start.c
10+
11+
# We want to test FatLTO objects when included in archives.
12+
RUN: llvm-ar rcs foo.a foo.o
13+
RUN: llvm-ar rcsT boo.a boo.o
14+
15+
# Build with DTLTO.
16+
RUN: %clang --target=x86_64-linux-gnu -flto=thin -ffat-lto-objects \
17+
RUN: -fuse-ld=lld -nostdlib foo.a boo.a start.o -Wl,--save-temps \
18+
RUN: -fthinlto-distributor=%python \
19+
RUN: -Xthinlto-distributor=%llvm_src_root/utils/dtlto/local.py
20+
21+
# Check that the required output files have been created.
22+
RUN: ls | FileCheck %s
23+
24+
# thin archive member: <archive>(<member> at <offset>).<task>.<pid>.<task>.<pid>.native.o
25+
CHECK-DAG: {{^}}boo.a(boo.o at [[#BOO_OFFSET:]]).3.[[#%X,HEXPID:]].3.[[#PID:]].native.o{{$}}
26+
# archive member: <archive>(<member> at <offset>).<task>.<pid>.<task>.<pid>.native.o
27+
CHECK-DAG: {{^}}foo.a(foo.o at [[#FOO_OFFSET:]]).2.[[#%X,HEXPID]].2.[[#PID]].native.o{{$}}
28+
# FatLTO object: <file>.<task>.<pid>.<task>.<pid>.native.o.
29+
CHECK-DAG: {{^}}start.o.1.[[#%X,HEXPID]].1.[[#PID]].native.o{{$}}
30+
31+
# Check that all objects are named in all of the index files.
32+
# We expect this to happen because each object references symbols from the
33+
# others.
34+
RUN: llvm-dis *.1.*.thinlto.bc -o - | \
35+
RUN: FileCheck %s --check-prefixes=OBJECTS
36+
RUN: llvm-dis *.2.*.thinlto.bc -o - | \
37+
RUN: FileCheck %s --check-prefixes=OBJECTS
38+
RUN: llvm-dis *.3.*.thinlto.bc -o - | \
39+
RUN: FileCheck %s --check-prefixes=OBJECTS
40+
41+
OBJECTS-DAG: foo.o
42+
OBJECTS-DAG: boo.o
43+
OBJECTS-DAG: start.o
44+
45+
#--- foo.c
46+
extern int boo(int), _start(int);
47+
__attribute__((retain)) int foo(int x) { return x + boo(x) + _start(x); }
48+
49+
#--- boo.c
50+
extern int foo(int), _start(int);
51+
__attribute__((retain)) int boo(int x) { return x + foo(x) + _start(x); }
52+
53+
#--- start.c
54+
extern int foo(int), boo(int);
55+
__attribute__((retain)) int _start(int x) { return x + foo(x) + boo(x); }

‎lld/ELF/Driver.cpp‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -236,8 +236,10 @@ bool LinkerDriver::tryAddFatLTOFile(MemoryBufferRef mb, StringRef archiveName,
236236
IRObjectFile::findBitcodeInMemBuffer(mb);
237237
if (errorToBool(fatLTOData.takeError()))
238238
return false;
239-
files.push_back(std::make_unique<BitcodeFile>(ctx, *fatLTOData, archiveName,
240-
offsetInArchive, lazy));
239+
auto file = std::make_unique<BitcodeFile>(ctx, *fatLTOData, archiveName,
240+
offsetInArchive, lazy);
241+
file->obj->fatLTOObject(true);
242+
files.push_back(std::move(file));
241243
return true;
242244
}
243245

‎lld/test/ELF/dtlto/timetrace.test‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ RUN: %python filter_order_and_pprint.py %t.json | FileCheck %s
3333
CHECK: "name": "Add input for DTLTO"
3434
CHECK: "name": "Add input for DTLTO"
3535
CHECK: "name": "Remove temporary inputs for DTLTO"
36-
CHECK: "name": "Save input archive member for DTLTO"
36+
CHECK: "name": "Serialize bitcode input for DTLTO"
3737
CHECK-SAME: "detail": "t1.a(t1.bc at [[#ARCHIVE_OFFSET:]]).1.[[PID:[A-F0-9]+]].o"
3838
CHECK: "name": "Total Add input for DTLTO"
3939
CHECK-SAME: "count": 2,
4040
CHECK: "name": "Total Remove temporary inputs for DTLTO"
4141
CHECK-SAME: "count": 1,
42-
CHECK: "name": "Total Save input archive member for DTLTO"
42+
CHECK: "name": "Total Serialize bitcode input for DTLTO"
4343
CHECK-SAME: "count": 1,
4444

4545
#--- t1.ll

‎llvm/include/llvm/LTO/LTO.h‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,12 @@ class InputFile {
132132
std::vector<std::pair<StringRef, Comdat::SelectionKind>> ComdatTable;
133133

134134
MemoryBufferRef MbRef;
135-
bool IsMemberOfArchive = false;
135+
bool IsFatLTOObject = false;
136+
// For distributed compilation, each input must exist as an individual bitcode
137+
// file on disk and be identified by its ModuleID. Archive members and FatLTO
138+
// objects violate this. So, in these cases we flag that the bitcode must be
139+
// written out to a new standalone file.
140+
bool SerializeForDistribution = false;
136141
bool IsThinLTO = false;
137142
StringRef ArchivePath;
138143
StringRef MemberName;
@@ -205,10 +210,16 @@ class InputFile {
205210
LLVM_ABI BitcodeModule &getPrimaryBitcodeModule();
206211
// Returns the memory buffer reference for this input file.
207212
MemoryBufferRef getFileBuffer() const { return MbRef; }
208-
// Returns true if this input file is a member of an archive.
209-
bool isMemberOfArchive() const { return IsMemberOfArchive; }
210-
// Mark this input file as a member of archive.
211-
void memberOfArchive(bool MA) { IsMemberOfArchive = MA; }
213+
// Returns true if this input should be serialized to disk for distribution.
214+
// See the comment on SerializeForDistribution for details.
215+
bool getSerializeForDistribution() const { return SerializeForDistribution; }
216+
// Mark whether this input should be serialized to disk for distribution.
217+
// See the comment on SerializeForDistribution for details.
218+
void setSerializeForDistribution(bool SFD) { SerializeForDistribution = SFD; }
219+
// Returns true if this bitcode came from a FatLTO object.
220+
bool isFatLTOObject() const { return IsFatLTOObject; }
221+
// Mark this bitcode as coming from a FatLTO object.
222+
void fatLTOObject(bool FO) { IsFatLTOObject = FO; }
212223

213224
// Returns true if bitcode is ThinLTO.
214225
bool isThinLTO() const { return IsThinLTO; }

‎llvm/lib/DTLTO/DTLTO.cpp‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,13 @@
2121
#include "llvm/LTO/LTO.h"
2222
#include "llvm/Object/Archive.h"
2323
#include "llvm/Support/FileSystem.h"
24-
#include "llvm/Support/ManagedStatic.h"
2524
#include "llvm/Support/MemoryBufferRef.h"
2625
#include "llvm/Support/Path.h"
2726
#include "llvm/Support/Process.h"
2827
#include "llvm/Support/Signals.h"
2928
#include "llvm/Support/TimeProfiler.h"
3029
#include "llvm/Support/raw_ostream.h"
3130

32-
#include <iostream>
3331
#include <string>
3432

3533
using namespace llvm;
@@ -135,25 +133,29 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
135133
StringRef ModuleId = Input->getName();
136134
StringRef ArchivePath = Input->getArchivePath();
137135

138-
// Only process archive members.
139-
if (ArchivePath.empty())
136+
// In most cases, the module ID already points to an individual bitcode file
137+
// on disk, so no further preparation for distribution is required.
138+
if (ArchivePath.empty() && !Input->isFatLTOObject())
140139
return Input;
141140

142141
SmallString<64> NewModuleId;
143142
BitcodeModule &BM = Input->getPrimaryBitcodeModule();
144143

145-
// Check if the archive is a thin archive.
146-
Expected<bool> IsThin = isThinArchive(ArchivePath);
147-
if (!IsThin)
148-
return IsThin.takeError();
144+
// For a member of a thin archive that is not a FatLTO object, there is an
145+
// existing file on disk that can be used, so we can avoid having to
146+
// materialize.
147+
Expected<bool> UseThinMember =
148+
Input->isFatLTOObject() ? false : isThinArchive(ArchivePath);
149+
if (!UseThinMember)
150+
return UseThinMember.takeError();
149151

150-
if (*IsThin) {
152+
if (*UseThinMember) {
151153
// For thin archives, use the path to the actual file.
152154
NewModuleId =
153155
computeThinArchiveMemberPath(ArchivePath, Input->getMemberName());
154156
} else {
155-
// For regular archives, generate a unique name.
156-
Input->memberOfArchive(true);
157+
// For regular archives and FatLTO objects, generate a unique name.
158+
Input->setSerializeForDistribution(true);
157159

158160
// Create unique identifier using process ID and sequence number.
159161
std::string PID = utohexstr(sys::Process::getProcessId());
@@ -175,8 +177,8 @@ lto::DTLTO::addInput(std::unique_ptr<lto::InputFile> InputPtr) {
175177
// previously terminated linker process and can be safely overwritten.
176178
Error lto::DTLTO::saveInputArchiveMember(lto::InputFile *Input) {
177179
StringRef ModuleId = Input->getName();
178-
if (Input->isMemberOfArchive()) {
179-
TimeTraceScope TimeScope("Save input archive member for DTLTO", ModuleId);
180+
if (Input->getSerializeForDistribution()) {
181+
TimeTraceScope TimeScope("Serialize bitcode input for DTLTO", ModuleId);
180182
// Cleanup this file on abnormal process exit.
181183
if (!SaveTemps)
182184
llvm::sys::RemoveFileOnSignal(ModuleId);
@@ -216,7 +218,7 @@ void lto::DTLTO::cleanup() {
216218
if (!SaveTemps) {
217219
TimeTraceScope TimeScope("Remove temporary inputs for DTLTO");
218220
for (auto &Input : InputFiles) {
219-
if (!Input->isMemberOfArchive())
221+
if (!Input->getSerializeForDistribution())
220222
continue;
221223
std::error_code EC =
222224
sys::fs::remove(Input->getName(), /*IgnoreNonExisting=*/true);

0 commit comments

Comments
 (0)