benchmark updates: hermetic-sandbox doesn't matter
Motiejus Jakštys
parent
d0cdfd013e
commit
23ea22f307
106
README.md
106
README.md
|
|
@ -8,55 +8,93 @@ The test
|
||||||
--------
|
--------
|
||||||
|
|
||||||
Compiles 64 small binaries with [bazel-zig-cc][1] and [llvm14][2]. The tests
|
Compiles 64 small binaries with [bazel-zig-cc][1] and [llvm14][2]. The tests
|
||||||
were run on an x86_64 8-core machine running Ubuntu 22.04.
|
were run on an x86_64 8-core machine running Ubuntu 22.04. The llvm's sysroot
|
||||||
|
is this:
|
||||||
|
|
||||||
Steps to reproduce
|
sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
|
||||||
------------------
|
urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
|
||||||
|
|
||||||
Baseline is llvm14: 12-13 seconds:
|
... but with extra ~8k files, so the number of files in the sysroot is exactly
|
||||||
|
the same as in zig sdk.
|
||||||
|
|
||||||
```
|
Baseline
|
||||||
bazel --batch clean; bazel --batch build --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux ...
|
--------
|
||||||
INFO: Elapsed time: 12.454s, Critical Path: 1.11s
|
|
||||||
```
|
|
||||||
|
|
||||||
zig cc without the sandbox. Which means different invocations of `zig cc` will
|
Baseline llvm14 without sandbox: 12 seconds:
|
||||||
see that all files in `zig_lib_dir` are the same file. 17 seconds:
|
|
||||||
|
|
||||||
```
|
bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||||
bazel --batch clean; bazel --batch build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
bazel clean; bazel shutdown
|
||||||
INFO: Elapsed time: 17.021s, Critical Path: 1.67s
|
bazel --batch build --spawn_strategy=local --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||||
```
|
INFO: Elapsed time: 11.971s, Critical Path: 0.41s
|
||||||
|
|
||||||
zig cc plain: 61 seconds. Sandbox is on a real disk, which means it will take
|
zig cc without the sandbox. 15 seconds:
|
||||||
even longer to re-hash all its dependencies:
|
|
||||||
|
|
||||||
```
|
bazel build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||||
bazel --batch clean; bazel --batch build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
bazel clean; bazel shutdown
|
||||||
INFO: Elapsed time: 61.128s, Critical Path: 8.35s
|
bazel --batch build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||||
```
|
INFO: Elapsed time: 15.631s, Critical Path: 1.99s
|
||||||
|
|
||||||
zig cc with hardlinks: `--config=hermetic-sandbox`. This uses hardlinks to
|
Cost of Sandbox
|
||||||
zig_sdk instead of symlinks:
|
---------------
|
||||||
|
|
||||||
```
|
llvm 14:
|
||||||
bazel --batch clean; bazel --batch build --config=hermetic-sandbox --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
|
||||||
INFO: Elapsed time: 45.886s, Critical Path: 9.56s
|
|
||||||
```
|
|
||||||
|
|
||||||
llvm with sysroot (this sysroot is about half the size of zig):
|
bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||||
|
bazel clean; bazel shutdown
|
||||||
|
bazel --batch build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||||
|
INFO: Elapsed time: 109.677s, Critical Path: 13.60s
|
||||||
|
|
||||||
|
zig cc:
|
||||||
|
|
||||||
|
bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||||
|
bazel clean; bazel shutdown
|
||||||
|
bazel --batch build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||||
|
INFO: Elapsed time: 116.370s, Critical Path: 16.23s
|
||||||
|
|
||||||
|
Possible mitigation: Sandboxfs
|
||||||
|
------------------------------
|
||||||
|
|
||||||
|
No dice.
|
||||||
|
|
||||||
|
llvm 14:
|
||||||
|
|
||||||
|
bazel build --experimental_sandboxfs_path=sandboxfs --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||||
|
bazel clean; bazel shutdown
|
||||||
|
bazel --batch build --experimental_sandboxfs_path=sandboxfs --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||||
|
INFO: Elapsed time: 96.803s, Critical Path: 12.42s
|
||||||
|
|
||||||
|
zig cc:
|
||||||
|
|
||||||
|
bazel build --experimental_sandboxfs_path=sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||||
|
bazel clean; bazel shutdown
|
||||||
|
bazel --batch build --experimental_sandboxfs_path=sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||||
|
INFO: Elapsed time: 109.347s, Critical Path: 13.94s
|
||||||
|
|
||||||
|
Possible mitigation: `--experimental_reuse_sandbox_directories`
|
||||||
|
---------------------------------------------------------------
|
||||||
|
|
||||||
|
Dice.
|
||||||
|
|
||||||
|
llvm 14:
|
||||||
|
|
||||||
|
bazel build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||||
|
bazel clean; bazel shutdown
|
||||||
|
bazel --batch build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||||
|
INFO: Elapsed time: 25.252s, Critical Path: 4.44s
|
||||||
|
|
||||||
|
zig cc:
|
||||||
|
|
||||||
|
bazel build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||||
|
bazel clean; bazel shutdown
|
||||||
|
bazel --batch build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||||
|
INFO: Elapsed time: 29.132s, Critical Path: 3.88s
|
||||||
|
|
||||||
```
|
|
||||||
bazel --batch clean; bazel --batch build --config=hermetic-sandbox --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
|
||||||
INFO: Elapsed time: 25.644s, Critical Path: 3.03s
|
|
||||||
```
|
|
||||||
|
|
||||||
Flame graphs and discussion
|
Flame graphs and discussion
|
||||||
---------------------------
|
---------------------------
|
||||||
|
|
||||||
Flame graphs are in results/. As of 2022-12-11 most of the overhead comes from
|
Flame graphs and more tests are in results/. As of 2022-12-11 most of the
|
||||||
creating and deleting the sandboxes. Time for a new sandboxfs!
|
overhead comes from creating and deleting the sandboxes.
|
||||||
|
|
||||||
|
|
||||||
[1]: https://sr.ht/~motiejus/bazel-zig-cc
|
[1]: https://sr.ht/~motiejus/bazel-zig-cc
|
||||||
[2]: https://github.com/grailbio/bazel-toolchain
|
[2]: https://github.com/grailbio/bazel-toolchain
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||||
|
|
||||||
## LLVM-14
|
## LLVM
|
||||||
|
|
||||||
BAZEL_TOOLCHAIN_TAG = "0.7.2"
|
BAZEL_TOOLCHAIN_TAG = "0.7.2"
|
||||||
BAZEL_TOOLCHAIN_SHA = "f7aa8e59c9d3cafde6edb372d9bd25fb4ee7293ab20b916d867cd0baaa642529"
|
BAZEL_TOOLCHAIN_SHA = "f7aa8e59c9d3cafde6edb372d9bd25fb4ee7293ab20b916d867cd0baaa642529"
|
||||||
|
|
@ -37,8 +37,8 @@ filegroup(
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
)
|
)
|
||||||
""",
|
""",
|
||||||
sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
|
sha256 = "4498fa8ce2df8830afd88e0bbf7c8d353a13c3f2454abc507a057889075af904",
|
||||||
urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
|
urls = ["https://dl.jakstys.lt/zig/debian_stretch_amd64_sysroot-padding15353.tar.xz"],
|
||||||
)
|
)
|
||||||
|
|
||||||
llvm_toolchain(
|
llvm_toolchain(
|
||||||
|
|
|
||||||
|
|
@ -5,24 +5,17 @@ _build() {
|
||||||
name=$1
|
name=$1
|
||||||
shift
|
shift
|
||||||
bazel build --color=no --curses=no "$@" //:zigzag-0
|
bazel build --color=no --curses=no "$@" //:zigzag-0
|
||||||
bazel shutdown
|
bazel clean --color=no --curses=no
|
||||||
bazel --batch clean --color=no --curses=no
|
bazel shutdown --color=no --curses=no
|
||||||
perf record --call-graph=dwarf -g -o "$HOME/$name.data" bazel --batch build --color=no --curses=no "$@" ...
|
perf record --call-graph=dwarf -g -o "$HOME/$name.data" bazel --batch build --color=no --curses=no "$@" ...
|
||||||
}
|
}
|
||||||
|
|
||||||
_build zigcc-hermetic-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
_build zigcc-nosandbox --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||||
_build zigcc-hermetic-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
_build llvm-nosandbox --spawn_strategy=local --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux
|
||||||
_build llvm-hermetic-j1 --jobs=1 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux --config=hermetic-sandbox
|
_build zigcc --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||||
_build llvm-hermetic-j8 --jobs=8 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux --config=hermetic-sandbox
|
_build llvm --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux
|
||||||
_build llvm-sysroot-hermetic-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
_build zigcc-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||||
_build llvm-sysroot-hermetic-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
_build llvm-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux
|
||||||
|
|
||||||
_build zigcc-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
|
||||||
_build zigcc-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
|
||||||
_build llvm-j1 --jobs=1 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux
|
|
||||||
_build llvm-j8 --jobs=8 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux
|
|
||||||
_build llvm-sysroot-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
|
||||||
_build llvm-sysroot-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
|
||||||
|
|
||||||
for d in $HOME/*.data; do
|
for d in $HOME/*.data; do
|
||||||
f=$(basename "$d")
|
f=$(basename "$d")
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue