Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bazel: Add pre-compiled clang header poc #13788

Merged
4 changes: 4 additions & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ build:sanitizer --test_tag_filters=-no_san
build:clang --action_env=BAZEL_COMPILER=clang
build:clang --linkopt=-fuse-ld=lld

# Flags for Clang + PCH
build:clang-pch --spawn_strategy=local
build:clang-pch --define=ENVOY_CLANG_PCH=1

# Basic ASAN/UBSAN that works for gcc
build:asan --action_env=ENVOY_ASAN=1
build:asan --config=sanitizer
Expand Down
5 changes: 5 additions & 0 deletions bazel/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,11 @@ config_setting(
},
)

config_setting(
name = "clang_pch_build",
values = {"define": "ENVOY_CLANG_PCH=1"},
)
Comment on lines +172 to +175
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use a bool_flag instead of define?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't get this option to work, started a thread on it here https://envoyproxy.slack.com/archives/C7E6C71QB/p1618869455054400

Either way it should be an easy swap if we want because no one should ever use this directly, they should only ever use the --config=clang-pch flag instead. We might actually prefer this way because it only requires 1 target vs 2

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We talked offline and you can use --@// for this omitting the envoy part. But what do you think about merging this way since it's not meant for direct consumption anyways and this leaves us 1 less option to maintain since for the flags we have to have both.


config_setting(
name = "gcc_build_gcc",
flag_values = {
Expand Down
2 changes: 2 additions & 0 deletions bazel/envoy_build_system.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ load(
_envoy_cc_win32_library = "envoy_cc_win32_library",
_envoy_proto_library = "envoy_proto_library",
)
load(":envoy_pch.bzl", _envoy_pch_library = "envoy_pch_library")
load(
":envoy_select.bzl",
_envoy_select_boringssl = "envoy_select_boringssl",
Expand Down Expand Up @@ -225,6 +226,7 @@ envoy_cc_posix_library = _envoy_cc_posix_library
envoy_cc_posix_without_linux_library = _envoy_cc_posix_without_linux_library
envoy_cc_win32_library = _envoy_cc_win32_library
envoy_proto_library = _envoy_proto_library
envoy_pch_library = _envoy_pch_library

# Test wrappers (from envoy_test.bzl)
envoy_cc_fuzz_test = _envoy_cc_fuzz_test
Expand Down
5 changes: 3 additions & 2 deletions bazel/envoy_library.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ load(
"envoy_external_dep_path",
"envoy_linkstatic",
)
load(":envoy_pch.bzl", "envoy_pch_copts")
load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library")
load(
"@envoy_build_config//:extensions_build_config.bzl",
Expand Down Expand Up @@ -92,17 +93,17 @@ def envoy_cc_library(
name = name,
srcs = srcs,
hdrs = hdrs,
copts = envoy_copts(repository) + copts,
copts = envoy_copts(repository) + envoy_pch_copts(repository, "//source/common/common:common_pch") + copts,
visibility = visibility,
tags = tags,
textual_hdrs = textual_hdrs,
deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + [
repository + "//envoy/common:base_includes",
repository + "//source/common/common:fmt_lib",
repository + "//source/common/common:common_pch",
envoy_external_dep_path("abseil_flat_hash_map"),
envoy_external_dep_path("abseil_flat_hash_set"),
envoy_external_dep_path("abseil_strings"),
envoy_external_dep_path("spdlog"),
envoy_external_dep_path("fmtlib"),
],
alwayslink = 1,
Expand Down
51 changes: 51 additions & 0 deletions bazel/envoy_pch.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
load("@rules_cc//cc:defs.bzl", "cc_library")

# DO NOT LOAD THIS FILE. Load envoy_build_system.bzl instead.
# Envoy library targets
load(
":envoy_internal.bzl",
"envoy_copts",
"envoy_external_dep_path",
"envoy_linkstatic",
)
load(":pch.bzl", "pch")

def envoy_pch_copts(repository, target):
return select({
repository + "//bazel:clang_pch_build": [
"-include-pch",
"$(location {}{})".format(repository, target),
],
"//conditions:default": [],
})

def envoy_pch_library(
name,
includes,
deps,
external_deps,
visibility,
testonly = False,
repository = ""):
cc_library(
name = name + "_libs",
visibility = ["//visibility:private"],
copts = envoy_copts(repository),
deps = deps + [envoy_external_dep_path(dep) for dep in external_deps],
alwayslink = 1,
testonly = testonly,
linkstatic = envoy_linkstatic(),
)

pch(
name = name,
deps = [name + "_libs"],
includes = includes,
visibility = visibility,
testonly = testonly,
tags = ["no-remote"],
enabled = select({
repository + "//bazel:clang_pch_build": True,
"//conditions:default": False,
}),
)
23 changes: 17 additions & 6 deletions bazel/envoy_test.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
load("@rules_fuzzing//fuzzing:cc_defs.bzl", "fuzzing_decoration")
load(":envoy_binary.bzl", "envoy_cc_binary")
load(":envoy_library.bzl", "tcmalloc_external_deps")
load(":envoy_pch.bzl", "envoy_pch_copts")
load(
":envoy_internal.bzl",
"envoy_copts",
Expand All @@ -29,19 +30,26 @@ def _envoy_cc_test_infrastructure_library(
include_prefix = None,
copts = [],
alwayslink = 1,
disable_pch = False,
**kargs):
# Add implicit tcmalloc external dependency(if available) in order to enable CPU and heap profiling in tests.
deps += tcmalloc_external_deps(repository)
extra_deps = []
pch_copts = []
if disable_pch:
extra_deps = [envoy_external_dep_path("googletest")]
else:
extra_deps = [repository + "//test:test_pch"]
pch_copts = envoy_pch_copts(repository, "//test:test_pch")

cc_library(
name = name,
srcs = srcs,
hdrs = hdrs,
data = data,
copts = envoy_copts(repository, test = True) + copts,
copts = envoy_copts(repository, test = True) + copts + pch_copts,
testonly = 1,
deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + [
envoy_external_dep_path("googletest"),
],
deps = deps + [envoy_external_dep_path(dep) for dep in external_deps] + extra_deps,
tags = tags,
include_prefix = include_prefix,
alwayslink = alwayslink,
Expand Down Expand Up @@ -159,11 +167,12 @@ def envoy_cc_test(
name = name,
srcs = srcs,
data = data,
copts = envoy_copts(repository, test = True) + copts,
copts = envoy_copts(repository, test = True) + copts + envoy_pch_copts(repository, "//test:test_pch"),
linkopts = _envoy_test_linkopts(),
linkstatic = envoy_linkstatic(),
malloc = tcmalloc_external_dep(repository),
deps = envoy_stdlib_deps() + deps + [envoy_external_dep_path(dep) for dep in external_deps + ["googletest"]] + [
repository + "//test:test_pch",
repository + "//test:main",
repository + "//test/test_common:test_version_linkstamp",
],
Expand Down Expand Up @@ -192,6 +201,7 @@ def envoy_cc_test_library(
copts = [],
alwayslink = 1,
**kargs):
disable_pch = kargs.pop("disable_pch", True)
_envoy_cc_test_infrastructure_library(
name,
srcs,
Expand All @@ -205,6 +215,7 @@ def envoy_cc_test_library(
copts,
visibility = ["//visibility:public"],
alwayslink = alwayslink,
disable_pch = disable_pch,
**kargs
)

Expand Down Expand Up @@ -283,7 +294,7 @@ def envoy_py_test(

# Envoy C++ mock targets should be specified with this function.
def envoy_cc_mock(name, **kargs):
envoy_cc_test_library(name = name, **kargs)
envoy_cc_test_library(name = name, disable_pch = True, **kargs)

# Envoy shell tests that need to be included in coverage run should be specified with this function.
def envoy_sh_test(
Expand Down
114 changes: 114 additions & 0 deletions bazel/pch.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
load(
"@bazel_tools//tools/build_defs/cc:action_names.bzl",
"CPP_COMPILE_ACTION_NAME",
)

def _pch(ctx):
deps_cc_info = cc_common.merge_cc_infos(
cc_infos = [dep[CcInfo] for dep in ctx.attr.deps],
)

if not ctx.attr.enabled:
return [deps_cc_info]

cc_toolchain = ctx.attr._cc_toolchain[cc_common.CcToolchainInfo]
feature_configuration = cc_common.configure_features(
ctx = ctx,
cc_toolchain = cc_toolchain,
requested_features = ctx.features,
unsupported_features = ctx.disabled_features,
)

cc_compiler_path = cc_common.get_tool_for_action(
feature_configuration = feature_configuration,
action_name = CPP_COMPILE_ACTION_NAME,
)

if "clang" not in cc_compiler_path:
fail("error: attempting to use clang PCH without clang: {}".format(cc_compiler_path))

generated_header_file = ctx.actions.declare_file(ctx.label.name + ".h")
ctx.actions.write(
generated_header_file,
"\n".join(["#include \"{}\"".format(include) for include in ctx.attr.includes]) + "\n",
)

# TODO: -fno-pch-timestamp / invalidation in that case doesn't work
pch_flags = ["-x", "c++-header"]
keith marked this conversation as resolved.
Show resolved Hide resolved
pch_file = ctx.actions.declare_file(ctx.label.name + ".pch")

deps_ctx = deps_cc_info.compilation_context
cc_compile_variables = cc_common.create_compile_variables(
feature_configuration = feature_configuration,
cc_toolchain = cc_toolchain,
user_compile_flags = ctx.fragments.cpp.copts + ctx.fragments.cpp.cxxopts + pch_flags,
source_file = generated_header_file.path,
output_file = pch_file.path,
preprocessor_defines = depset(deps_ctx.defines.to_list() + deps_ctx.local_defines.to_list()),
include_directories = deps_ctx.includes,
quote_include_directories = deps_ctx.quote_includes,
system_include_directories = deps_ctx.system_includes,
framework_include_directories = deps_ctx.framework_includes,
)

env = cc_common.get_environment_variables(
feature_configuration = feature_configuration,
action_name = CPP_COMPILE_ACTION_NAME,
variables = cc_compile_variables,
)

command_line = cc_common.get_memory_inefficient_command_line(
feature_configuration = feature_configuration,
action_name = CPP_COMPILE_ACTION_NAME,
variables = cc_compile_variables,
)

transitive_headers = []
for dep in ctx.attr.deps:
transitive_headers.append(dep[CcInfo].compilation_context.headers)
ctx.actions.run(
executable = cc_compiler_path,
arguments = command_line,
env = env,
inputs = depset(
items = [generated_header_file],
transitive = [cc_toolchain.all_files] + transitive_headers,
),
outputs = [pch_file],
)

return [
DefaultInfo(files = depset(items = [pch_file])),
cc_common.merge_cc_infos(
direct_cc_infos = [
CcInfo(
compilation_context = cc_common.create_compilation_context(
headers = depset([pch_file, generated_header_file]),
),
),
],
cc_infos = [deps_cc_info],
),
]

pch = rule(
attrs = dict(
includes = attr.string_list(
mandatory = True,
allow_empty = False,
),
deps = attr.label_list(
mandatory = True,
allow_empty = False,
providers = [CcInfo],
),
enabled = attr.bool(
mandatory = True,
),
_cc_toolchain = attr.label(default = Label("@bazel_tools//tools/cpp:current_cc_toolchain")),
),
fragments = ["cpp"],
provides = [CcInfo],
toolchains = ["@bazel_tools//tools/cpp:toolchain_type"],
implementation = _pch,
)
28 changes: 28 additions & 0 deletions source/common/common/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ load(
"envoy_cc_posix_library",
"envoy_cc_win32_library",
"envoy_package",
"envoy_pch_library",
)

licenses(["notice"]) # Apache 2
Expand Down Expand Up @@ -469,3 +470,30 @@ envoy_cc_library(
"@com_google_absl//absl/status:statusor",
],
)

envoy_pch_library(
name = "common_pch",
external_deps = [
"spdlog",
],
includes = [
"envoy/config/bootstrap/v3/bootstrap.pb.h",
"envoy/config/cluster/v3/cluster.pb.h",
"envoy/config/core/v3/base.pb.h",
"envoy/config/core/v3/config_source.pb.h",
"envoy/config/route/v3/route_components.pb.h",
"envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h",
"envoy/service/discovery/v3/discovery.pb.h",
"spdlog/sinks/android_sink.h",
"spdlog/spdlog.h",
],
Comment on lines +479 to +489
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not fan of directly listing those headers here. Can they be extracted from deps headers?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So the problem is that the CcInfos of our deps include all headers transitively that they depend on, so we can't (or at least don't want to) just dump them all in the generated header file we use to create the pch. I tried some of the other attrs like direct_public_headers but they all appear to be empty. I'm not sure how those are supposed to fit in with this. One benefit here is that we're quite explicit about what things we want to be pre-compiled vs potentially blowing up this file with all transitive headers

visibility = ["//visibility:public"],
deps = [
"@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
"@envoy_api//envoy/config/cluster/v3:pkg_cc_proto",
"@envoy_api//envoy/config/core/v3:pkg_cc_proto",
"@envoy_api//envoy/config/route/v3:pkg_cc_proto",
"@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
"@envoy_api//envoy/service/discovery/v3:pkg_cc_proto",
],
)
Loading