Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 36df60f

Browse files
brettcannonbrettcannon
committed
Merge for issue #24915 
2 parents f409c7c + 7188a3e commit 36df60f

8 files changed

Lines changed: 185 additions & 7 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
*.rej
1212
*.swp
1313
*~
14+
*.gc??
15+
*.profclang?
16+
*.profraw
1417
.gdb_history
1518
Doc/build/
1619
Doc/venv/

‎.hgignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,9 @@ libpython*.so*
5050
*.pyd
5151
*.cover
5252
*~
53+
*.gc??
54+
*.profclang?
55+
*.profraw
5356
Lib/distutils/command/*.pdb
5457
Lib/lib2to3/*.pickle
5558
Lib/test/data/*

‎Makefile.pre.in‎

Lines changed: 24 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ LDVERSION= @LDVERSION@
4343
HGVERSION= @HGVERSION@
4444
HGTAG= @HGTAG@
4545
HGBRANCH= @HGBRANCH@
46+
PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@
47+
PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@
48+
LLVM_PROF_MERGER=@LLVM_PROF_MERGER@
49+
LLVM_PROF_FILE=@LLVM_PROF_FILE@
50+
LLVM_PROF_ERR=@LLVM_PROF_ERR@
4651

4752
GNULD= @GNULD@
4853

@@ -226,8 +231,7 @@ TCLTK_INCLUDES= @TCLTK_INCLUDES@
226231
TCLTK_LIBS= @TCLTK_LIBS@
227232

228233
# The task to run while instrument when building the profile-opt target
229-
PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
230-
#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py
234+
PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
231235

232236
# report files for gcov / lcov coverage report
233237
COVERAGE_INFO= $(abs_builddir)/coverage.info
@@ -477,27 +481,38 @@ LIBRARY_OBJS= \
477481
all: build_all
478482
build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config
479483

480-
# Compile a binary with gcc profile guided optimization.
484+
# Compile a binary with profile guided optimization.
481485
profile-opt:
486+
@if [ $(LLVM_PROF_ERR) == yes ]; then \
487+
echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
488+
echo "Please add it to PATH and run ./configure again" ;\
489+
exit 1;\
490+
fi
482491
@echo "Building with support for profile generation:"
483492
$(MAKE) clean
493+
$(MAKE) profile-removal
484494
$(MAKE) build_all_generate_profile
485-
@echo "Running benchmark to generate profile data:"
486495
$(MAKE) profile-removal
496+
@echo "Running code to generate profile data (this can take a while):"
487497
$(MAKE) run_profile_task
498+
$(MAKE) build_all_merge_profile
488499
@echo "Rebuilding with profile guided optimizations:"
489500
$(MAKE) clean
490501
$(MAKE) build_all_use_profile
502+
$(MAKE) profile-removal
491503

492504
build_all_generate_profile:
493-
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-generate" LDFLAGS="-fprofile-generate" LIBS="$(LIBS) -lgcov"
505+
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
494506

495507
run_profile_task:
496508
: # FIXME: can't run for a cross build
497-
$(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK)
509+
$(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
510+
511+
build_all_merge_profile:
512+
$(LLVM_PROF_MERGER)
498513

499514
build_all_use_profile:
500-
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-use -fprofile-correction"
515+
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)"
501516

502517
# Compile and run with gcov
503518
.PHONY=coverage coverage-lcov coverage-report
@@ -1568,9 +1583,11 @@ clean: pycremoval
15681583
-rm -f pybuilddir.txt
15691584
-rm -f Lib/lib2to3/*Grammar*.pickle
15701585
-rm -f Programs/_testembed Programs/_freeze_importlib
1586+
-rm -rf build
15711587

15721588
profile-removal:
15731589
find . -name '*.gc??' -exec rm -f {} ';'
1590+
find . -name '*.profclang?' -exec rm -f {} ';'
15741591
rm -f $(COVERAGE_INFO)
15751592
rm -rf $(COVERAGE_REPORT)
15761593

‎Misc/ACKS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1080,6 +1080,7 @@ Heikki Partanen
10801080
Harri Pasanen
10811081
Gaël Pasgrimaud
10821082
Ashish Nitin Patil
1083+
Alecsandru Patrascu
10831084
Randy Pausch
10841085
Samuele Pedroni
10851086
Justin Peel

‎Misc/NEWS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,9 @@ Tests
201201
Build
202202
-----
203203

204+
- Issue #24915: Add LLVM support for PGO builds and use the test suite to
205+
generate the profile data. Initiial patch by Alecsandru Patrascu of Intel.
206+
204207
- Issue #24910: Windows MSIs now have unique display names.
205208

206209
- Issue #24986: It is now possible to build Python on Windows without errors

‎README‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,34 @@ For example:
4646
(This will fail if you *also* built at the top-level directory.
4747
You should do a "make clean" at the toplevel first.)
4848

49+
If you need an optimized version of Python, you type "make profile-opt" in the
50+
top level directory. This will rebuild the interpreter executable using Profile
51+
Guided Optimization (PGO). For more details, see the section bellow.
52+
53+
54+
Profile Guided Optimization
55+
---------------------------
56+
57+
PGO takes advantage of recent versions of the GCC or Clang compilers.
58+
If ran, the "profile-opt" rule will do several steps.
59+
60+
First, the entire Python directory is cleaned of temporary files that
61+
may resulted in a previous compilation.
62+
63+
Then, an instrumented version of the interpreter is built, using suitable
64+
compiler flags for each flavour. Note that this is just an intermediary
65+
step and the binary resulted after this step is not good for real life
66+
workloads, as it has profiling instructions embedded inside.
67+
68+
After this instrumented version of the interpreter is built, the Makefile
69+
will automatically run a training workload. This is necessary in order to
70+
profile the interpreter execution. Note also that any output, both stdout
71+
and stderr, that may appear at this step is supressed.
72+
73+
Finally, the last step is to rebuild the interpreter, using the information
74+
collected in the previous one. The end result will be a the Python binary
75+
that is optimized and suitable for distribution or production installation.
76+
4977

5078
What's New
5179
----------

‎configure‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -667,6 +667,12 @@ UNIVERSAL_ARCH_FLAGS
667667
CFLAGS_NODIST
668668
BASECFLAGS
669669
OPT
670+
LLVM_PROF_FOUND
671+
LLVM_PROF_ERR
672+
LLVM_PROF_FILE
673+
LLVM_PROF_MERGER
674+
PGO_PROF_USE_FLAG
675+
PGO_PROF_GEN_FLAG
670676
ABIFLAGS
671677
LN
672678
MKDIR_P
@@ -6431,6 +6437,80 @@ $as_echo "no" >&6; }
64316437
fi
64326438

64336439

6440+
# Enable PGO flags.
6441+
# Extract the first word of "llvm-profdata", so it can be a program name with args.
6442+
set dummy llvm-profdata; ac_word=$2
6443+
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
6444+
$as_echo_n "checking for $ac_word... " >&6; }
6445+
if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
6446+
$as_echo_n "(cached) " >&6
6447+
else
6448+
if test -n "$LLVM_PROF_FOUND"; then
6449+
ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
6450+
else
6451+
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
6452+
for as_dir in $PATH
6453+
do
6454+
IFS=$as_save_IFS
6455+
test -z "$as_dir" && as_dir=.
6456+
for ac_exec_ext in '' $ac_executable_extensions; do
6457+
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
6458+
ac_cv_prog_LLVM_PROF_FOUND="found"
6459+
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
6460+
break 2
6461+
fi
6462+
done
6463+
done
6464+
IFS=$as_save_IFS
6465+
6466+
test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
6467+
fi
6468+
fi
6469+
LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
6470+
if test -n "$LLVM_PROF_FOUND"; then
6471+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
6472+
$as_echo "$LLVM_PROF_FOUND" >&6; }
6473+
else
6474+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
6475+
$as_echo "no" >&6; }
6476+
fi
6477+
6478+
6479+
LLVM_PROF_ERR=no
6480+
case $CC in
6481+
*clang*)
6482+
# Any changes made here should be reflected in the GCC+Darwin case below
6483+
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
6484+
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
6485+
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
6486+
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
6487+
if test $LLVM_PROF_FOUND = not-found
6488+
then
6489+
LLVM_PROF_ERR=yes
6490+
fi
6491+
;;
6492+
*gcc*)
6493+
case $ac_sys_system in
6494+
Darwin*)
6495+
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
6496+
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
6497+
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
6498+
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
6499+
if test $LLVM_PROF_FOUND = not-found
6500+
then
6501+
LLVM_PROF_ERR=yes
6502+
fi
6503+
;;
6504+
*)
6505+
PGO_PROF_GEN_FLAG="-fprofile-generate"
6506+
PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
6507+
LLVM_PROF_MERGER="true"
6508+
LLVM_PROF_FILE=""
6509+
;;
6510+
esac
6511+
;;
6512+
esac
6513+
64346514
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
64356515
# merged with this chunk of code?
64366516

‎configure.ac‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1218,6 +1218,49 @@ else AC_MSG_RESULT(no); Py_DEBUG='false'
12181218
fi],
12191219
[AC_MSG_RESULT(no)])
12201220

1221+
# Enable PGO flags.
1222+
AC_SUBST(PGO_PROF_GEN_FLAG)
1223+
AC_SUBST(PGO_PROF_USE_FLAG)
1224+
AC_SUBST(LLVM_PROF_MERGER)
1225+
AC_SUBST(LLVM_PROF_FILE)
1226+
AC_SUBST(LLVM_PROF_ERR)
1227+
AC_SUBST(LLVM_PROF_FOUND)
1228+
AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
1229+
LLVM_PROF_ERR=no
1230+
case $CC in
1231+
*clang*)
1232+
# Any changes made here should be reflected in the GCC+Darwin case below
1233+
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
1234+
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
1235+
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
1236+
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
1237+
if test $LLVM_PROF_FOUND = not-found
1238+
then
1239+
LLVM_PROF_ERR=yes
1240+
fi
1241+
;;
1242+
*gcc*)
1243+
case $ac_sys_system in
1244+
Darwin*)
1245+
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
1246+
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
1247+
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
1248+
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
1249+
if test $LLVM_PROF_FOUND = not-found
1250+
then
1251+
LLVM_PROF_ERR=yes
1252+
fi
1253+
;;
1254+
*)
1255+
PGO_PROF_GEN_FLAG="-fprofile-generate"
1256+
PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
1257+
LLVM_PROF_MERGER="true"
1258+
LLVM_PROF_FILE=""
1259+
;;
1260+
esac
1261+
;;
1262+
esac
1263+
12211264
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
12221265
# merged with this chunk of code?
12231266

0 commit comments

Comments
 (0)