Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 9e651e4

Browse files
ilya-biryukovtensorflower-gardener
ilya-biryukov
authored and
tensorflower-gardener
committed
Allow to download clang and use clang for CPU builds.
Previously we only allowed to download clang when doing GPU builds. The added skylark files use bazel's autoconf scripts, which were only added in 0.10.0. To provide nice error message for older versions of bazel (i.e. 'version is less than 0.10' vs 'can't load @bazel_tools/cpp/...'), we move the bazel version check into WORKSPACE file from workspace.bzl. PiperOrigin-RevId: 190050798
1 parent f9ccb89 commit 9e651e4

10 files changed

Lines changed: 104 additions & 65 deletions

File tree

‎WORKSPACE‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,12 @@ load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")
1414

1515
closure_repositories()
1616

17+
# We must check the bazel version before trying to parse any other BUILD
18+
# files, in case the parsing of those build files depends on the bazel
19+
# version we require here.
20+
load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
21+
check_bazel_version_at_least("0.10.0")
22+
1723
load("//tensorflow:workspace.bzl", "tf_workspace")
1824

1925
# Uncomment and update the paths in these entries to build the Android demo.

‎configure.py‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ def set_tf_cuda_clang(environ_cp):
524524

525525
def set_tf_download_clang(environ_cp):
526526
"""Set TF_DOWNLOAD_CLANG action_env."""
527-
question = 'Do you want to download a fresh release of clang? (Experimental)'
527+
question = 'Do you wish to download a fresh release of clang? (Experimental)'
528528
yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
529529
no_reply = 'Clang will not be downloaded.'
530530
set_action_env_var(
@@ -1380,7 +1380,7 @@ def main():
13801380
# environment variables.
13811381
environ_cp = dict(os.environ)
13821382

1383-
check_bazel_version('0.5.4')
1383+
check_bazel_version('0.10.0')
13841384

13851385
reset_tf_configure_bazelrc(args.workspace)
13861386
cleanup_makefile()
@@ -1397,6 +1397,9 @@ def main():
13971397
environ_cp['TF_NEED_OPENCL'] = '0'
13981398
environ_cp['TF_CUDA_CLANG'] = '0'
13991399
environ_cp['TF_NEED_TENSORRT'] = '0'
1400+
# TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
1401+
# Windows.
1402+
environ_cp['TF_DOWNLOAD_CLANG'] = '0'
14001403

14011404
if is_macos():
14021405
environ_cp['TF_NEED_JEMALLOC'] = '0'
@@ -1444,16 +1447,8 @@ def main():
14441447

14451448
set_tf_cuda_clang(environ_cp)
14461449
if environ_cp.get('TF_CUDA_CLANG') == '1':
1447-
if not is_windows():
1448-
# Ask if we want to download clang release while building.
1449-
set_tf_download_clang(environ_cp)
1450-
else:
1451-
# We use bazel's generated crosstool on Windows and there is no
1452-
# way to provide downloaded toolchain for that yet.
1453-
# TODO(ibiryukov): Investigate using clang as a cuda compiler on
1454-
# Windows.
1455-
environ_cp['TF_DOWNLOAD_CLANG'] = '0'
1456-
1450+
# Ask whether we should download the clang toolchain.
1451+
set_tf_download_clang(environ_cp)
14571452
if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
14581453
# Set up which clang we should use as the cuda / host compiler.
14591454
set_clang_cuda_compiler_path(environ_cp)
@@ -1463,6 +1458,13 @@ def main():
14631458
if not is_windows():
14641459
set_gcc_host_compiler_path(environ_cp)
14651460
set_other_cuda_vars(environ_cp)
1461+
else:
1462+
# CUDA not required. Ask whether we should download the clang toolchain and
1463+
# use it for the CPU build.
1464+
set_tf_download_clang(environ_cp)
1465+
if environ_cp.get('TF_DOWNLOAD_CLANG') == '1':
1466+
write_to_bazelrc('build --config=download_clang')
1467+
write_to_bazelrc('test --config=download_clang')
14661468

14671469
set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False)
14681470
if environ_cp.get('TF_NEED_MPI') == '1':

‎tensorflow/version_check.bzl‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
""" Helpers to check minimum version of bazel."""
2+
3+
def _extract_version_number(bazel_version):
4+
"""Extracts the semantic version number from a version string
5+
6+
Args:
7+
bazel_version: the version string that begins with the semantic version
8+
e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
9+
10+
Returns:
11+
The semantic version string, like "1.2.3".
12+
"""
13+
for i in range(len(bazel_version)):
14+
c = bazel_version[i]
15+
if not (c.isdigit() or c == "."):
16+
return bazel_version[:i]
17+
return bazel_version
18+
19+
# Parse the bazel version string from `native.bazel_version`.
20+
# e.g.
21+
# "0.10.0rc1 abc123d" => (0, 10, 0)
22+
# "0.3.0" => (0, 3, 0)
23+
def _parse_bazel_version(bazel_version):
24+
"""Parses a version string into a 3-tuple of ints
25+
26+
int tuples can be compared directly using binary operators (<, >).
27+
28+
Args:
29+
bazel_version: the Bazel version string
30+
31+
Returns:
32+
An int 3-tuple of a (major, minor, patch) version.
33+
"""
34+
35+
version = _extract_version_number(bazel_version)
36+
return tuple([int(n) for n in version.split(".")])
37+
38+
def check_bazel_version_at_least(minimum_bazel_version):
39+
if "bazel_version" not in dir(native):
40+
fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
41+
elif not native.bazel_version:
42+
print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
43+
print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
44+
return
45+
46+
if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
47+
fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
48+
native.bazel_version, minimum_bazel_version))

‎tensorflow/workspace.bzl‎

Lines changed: 3 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -10,65 +10,18 @@ load("//third_party/sycl:sycl_configure.bzl", "sycl_configure")
1010
load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure")
1111
load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure")
1212
load("//third_party:repo.bzl", "tf_http_archive")
13+
load("//third_party/clang_toolchain:cc_configure_clang.bzl", "cc_download_clang_toolchain")
1314
load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
1415
load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
1516

16-
def _extract_version_number(bazel_version):
17-
"""Extracts the semantic version number from a version string
18-
19-
Args:
20-
bazel_version: the version string that begins with the semantic version
21-
e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
22-
23-
Returns:
24-
The semantic version string, like "1.2.3".
25-
"""
26-
for i in range(len(bazel_version)):
27-
c = bazel_version[i]
28-
if not (c.isdigit() or c == "."):
29-
return bazel_version[:i]
30-
return bazel_version
31-
32-
# Parse the bazel version string from `native.bazel_version`.
33-
# e.g.
34-
# "0.10.0rc1 abc123d" => (0, 10, 0)
35-
# "0.3.0" => (0, 3, 0)
36-
def _parse_bazel_version(bazel_version):
37-
"""Parses a version string into a 3-tuple of ints
38-
39-
int tuples can be compared directly using binary operators (<, >).
40-
41-
Args:
42-
bazel_version: the Bazel version string
43-
44-
Returns:
45-
An int 3-tuple of a (major, minor, patch) version.
46-
"""
47-
48-
version = _extract_version_number(bazel_version)
49-
return tuple([int(n) for n in version.split(".")])
50-
51-
def check_bazel_version_at_least(minimum_bazel_version):
52-
if "bazel_version" not in dir(native):
53-
fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
54-
elif not native.bazel_version:
55-
print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
56-
print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
57-
return
58-
59-
if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
60-
fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
61-
native.bazel_version, minimum_bazel_version))
6217

6318
# If TensorFlow is linked as a submodule.
6419
# path_prefix is no longer used.
6520
# tf_repo_name is thought to be under consideration.
6621
def tf_workspace(path_prefix="", tf_repo_name=""):
67-
# We must check the bazel version before trying to parse any other BUILD
68-
# files, in case the parsing of those build files depends on the bazel
69-
# version we require here.
70-
check_bazel_version_at_least("0.5.4")
22+
# Note that we check the minimum bazel version in WORKSPACE.
7123
clang6_configure(name="local_config_clang6")
24+
cc_download_clang_toolchain(name="local_config_download_clang")
7225
cuda_configure(name="local_config_cuda")
7326
tensorrt_configure(name="local_config_tensorrt")
7427
git_configure(name="local_config_git")

‎third_party/clang_toolchain/BUILD‎

Whitespace-only changes.

‎third_party/clang_toolchain/cc_configure_clang.bzl‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
""" Downloads clang and configures the crosstool using bazel's autoconf."""
2+
3+
load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl")
4+
load(":download_clang.bzl", "download_clang")
5+
6+
_TF_DOWNLOAD_CLANG = "TF_DOWNLOAD_CLANG"
7+
_TF_NEED_CUDA = "TF_NEED_CUDA"
8+
9+
def _cc_clang_autoconf(repo_ctx):
10+
if repo_ctx.os.environ.get(_TF_DOWNLOAD_CLANG) != "1":
11+
return
12+
if repo_ctx.os.environ.get(_TF_NEED_CUDA) == "1":
13+
# Clang is handled separately for CUDA configs.
14+
# See cuda_configure.bzl for more details.
15+
return
16+
17+
download_clang(repo_ctx, out_folder='extra_tools')
18+
overriden_tools = {'gcc': 'extra_tools/bin/clang'}
19+
cc_autoconf_impl(repo_ctx, overriden_tools)
20+
21+
cc_download_clang_toolchain = repository_rule(
22+
environ = [
23+
_TF_DOWNLOAD_CLANG,
24+
_TF_NEED_CUDA,
25+
],
26+
implementation = _cc_clang_autoconf,
27+
)

third_party/gpus/download_clang.bzl renamed to third_party/clang_toolchain/download_clang.bzl

File renamed without changes.

‎third_party/gpus/cuda_configure.bzl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ NVVM_LIBDEVICE_PATHS = [
9696
"share/cuda/",
9797
]
9898

99-
load(":download_clang.bzl", "download_clang")
99+
load("//third_party/clang_toolchain:download_clang.bzl", "download_clang")
100100

101101
# TODO(dzc): Once these functions have been factored out of Bazel's
102102
# cc_configure.bzl, load them from @bazel_tools instead.

‎third_party/mkl_dnn/mkldnn.BUILD‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ config_setting(
44
name = "clang_linux_x86_64",
55
values = {
66
"cpu": "k8",
7-
"define": "using_cuda_clang=true",
7+
"define": "using_clang=true",
88
},
99
)
1010

‎tools/bazel.rc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,14 @@ build --define framework_shared_object=true
2727
build:mkl --define=using_mkl=true
2828
build:mkl -c opt
2929

30+
build:download_clang --crosstool_top=@local_config_download_clang//:toolchain
31+
build:download_clang --define=using_clang=true
32+
3033
build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
3134
build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true
3235

3336
build:cuda_clang --crosstool_top=@local_config_cuda//crosstool:toolchain
34-
build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true
37+
build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true --define=using_clang=true
3538

3639
build:win-cuda --define=using_cuda=true --define=using_cuda_nvcc=true
3740

0 commit comments

Comments
 (0)