benchmark updates: hermetic-sandbox doesn't matter
Motiejus Jakštys
parent
d0cdfd013e
commit
23ea22f307
106
README.md
106
README.md
|
|
@ -8,55 +8,93 @@ The test
|
|||
--------
|
||||
|
||||
Compiles 64 small binaries with [bazel-zig-cc][1] and [llvm14][2]. The tests
|
||||
were run on an x86_64 8-core machine running Ubuntu 22.04.
|
||||
were run on an x86_64 8-core machine running Ubuntu 22.04. The llvm's sysroot
|
||||
is this:
|
||||
|
||||
Steps to reproduce
|
||||
------------------
|
||||
sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
|
||||
urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
|
||||
|
||||
Baseline is llvm14: 12-13 seconds:
|
||||
... but with extra ~8k files, so the number of files in the sysroot is exactly
|
||||
the same as in zig sdk.
|
||||
|
||||
```
|
||||
bazel --batch clean; bazel --batch build --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 12.454s, Critical Path: 1.11s
|
||||
```
|
||||
Baseline
|
||||
--------
|
||||
|
||||
zig cc without the sandbox. Which means different invocations of `zig cc` will
|
||||
see that all files in `zig_lib_dir` are the same file. 17 seconds:
|
||||
Baseline llvm14 without sandbox: 12 seconds:
|
||||
|
||||
```
|
||||
bazel --batch clean; bazel --batch build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 17.021s, Critical Path: 1.67s
|
||||
```
|
||||
bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --spawn_strategy=local --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 11.971s, Critical Path: 0.41s
|
||||
|
||||
zig cc plain: 61 seconds. Sandbox is on a real disk, which means it will take
|
||||
even longer to re-hash all its dependencies:
|
||||
zig cc without the sandbox. 15 seconds:
|
||||
|
||||
```
|
||||
bazel --batch clean; bazel --batch build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 61.128s, Critical Path: 8.35s
|
||||
```
|
||||
bazel build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 15.631s, Critical Path: 1.99s
|
||||
|
||||
zig cc with hardlinks: `--config=hermetic-sandbox`. This uses hardlinks to
|
||||
zig_sdk instead of symlinks:
|
||||
Cost of Sandbox
|
||||
---------------
|
||||
|
||||
```
|
||||
bazel --batch clean; bazel --batch build --config=hermetic-sandbox --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 45.886s, Critical Path: 9.56s
|
||||
```
|
||||
llvm 14:
|
||||
|
||||
llvm with sysroot (this sysroot is about half the size of zig):
|
||||
bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 109.677s, Critical Path: 13.60s
|
||||
|
||||
zig cc:
|
||||
|
||||
bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 116.370s, Critical Path: 16.23s
|
||||
|
||||
Possible mitigation: Sandboxfs
|
||||
------------------------------
|
||||
|
||||
No dice.
|
||||
|
||||
llvm 14:
|
||||
|
||||
bazel build --experimental_sandboxfs_path=sandboxfs --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --experimental_sandboxfs_path=sandboxfs --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 96.803s, Critical Path: 12.42s
|
||||
|
||||
zig cc:
|
||||
|
||||
bazel build --experimental_sandboxfs_path=sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --experimental_sandboxfs_path=sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 109.347s, Critical Path: 13.94s
|
||||
|
||||
Possible mitigation: `--experimental_reuse_sandbox_directories`
|
||||
---------------------------------------------------------------
|
||||
|
||||
Dice.
|
||||
|
||||
llvm 14:
|
||||
|
||||
bazel build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 25.252s, Critical Path: 4.44s
|
||||
|
||||
zig cc:
|
||||
|
||||
bazel build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 29.132s, Critical Path: 3.88s
|
||||
|
||||
```
|
||||
bazel --batch clean; bazel --batch build --config=hermetic-sandbox --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 25.644s, Critical Path: 3.03s
|
||||
```
|
||||
|
||||
Flame graphs and discussion
|
||||
---------------------------
|
||||
|
||||
Flame graphs are in results/. As of 2022-12-11 most of the overhead comes from
|
||||
creating and deleting the sandboxes. Time for a new sandboxfs!
|
||||
|
||||
Flame graphs and more tests are in results/. As of 2022-12-11 most of the
|
||||
overhead comes from creating and deleting the sandboxes.
|
||||
|
||||
[1]: https://sr.ht/~motiejus/bazel-zig-cc
|
||||
[2]: https://github.com/grailbio/bazel-toolchain
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
|
||||
|
||||
## LLVM-14
|
||||
## LLVM
|
||||
|
||||
BAZEL_TOOLCHAIN_TAG = "0.7.2"
|
||||
BAZEL_TOOLCHAIN_SHA = "f7aa8e59c9d3cafde6edb372d9bd25fb4ee7293ab20b916d867cd0baaa642529"
|
||||
|
|
@ -37,8 +37,8 @@ filegroup(
|
|||
visibility = ["//visibility:public"],
|
||||
)
|
||||
""",
|
||||
sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
|
||||
urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
|
||||
sha256 = "4498fa8ce2df8830afd88e0bbf7c8d353a13c3f2454abc507a057889075af904",
|
||||
urls = ["https://dl.jakstys.lt/zig/debian_stretch_amd64_sysroot-padding15353.tar.xz"],
|
||||
)
|
||||
|
||||
llvm_toolchain(
|
||||
|
|
|
|||
|
|
@ -5,24 +5,17 @@ _build() {
|
|||
name=$1
|
||||
shift
|
||||
bazel build --color=no --curses=no "$@" //:zigzag-0
|
||||
bazel shutdown
|
||||
bazel --batch clean --color=no --curses=no
|
||||
bazel clean --color=no --curses=no
|
||||
bazel shutdown --color=no --curses=no
|
||||
perf record --call-graph=dwarf -g -o "$HOME/$name.data" bazel --batch build --color=no --curses=no "$@" ...
|
||||
}
|
||||
|
||||
_build zigcc-hermetic-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
||||
_build zigcc-hermetic-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
||||
_build llvm-hermetic-j1 --jobs=1 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux --config=hermetic-sandbox
|
||||
_build llvm-hermetic-j8 --jobs=8 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux --config=hermetic-sandbox
|
||||
_build llvm-sysroot-hermetic-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
||||
_build llvm-sysroot-hermetic-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox
|
||||
|
||||
_build zigcc-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build zigcc-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build llvm-j1 --jobs=1 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux
|
||||
_build llvm-j8 --jobs=8 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux
|
||||
_build llvm-sysroot-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build llvm-sysroot-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build zigcc-nosandbox --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build llvm-nosandbox --spawn_strategy=local --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux
|
||||
_build zigcc --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build llvm --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux
|
||||
_build zigcc-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build llvm-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux
|
||||
|
||||
for d in $HOME/*.data; do
|
||||
f=$(basename "$d")
|
||||
|
|
|
|||
Loading…
Reference in New Issue