update results
This commit is contained in:
parent
3c883f1979
commit
49bb148282
129
README.md
129
README.md
@ -8,101 +8,72 @@ The test
|
||||
--------
|
||||
|
||||
Compiles 64 small binaries with [bazel-zig-cc][1] and [llvm14][2]. The tests
|
||||
were run on an x86_64 8-core machine running Ubuntu 22.04. The llvm's sysroot
|
||||
is this:
|
||||
were run on an x86_64 8-core machine running Ubuntu 22.04.
|
||||
|
||||
sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3",
|
||||
urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"],
|
||||
Results
|
||||
-------
|
||||
|
||||
... but with extra ~8k files, so the number of files in the sysroot is exactly
|
||||
the same as in zig sdk.
|
||||
Benchmark 1: bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=local //...
|
||||
Time (mean ± σ): 9.257 s ± 0.287 s [User: 0.083 s, System: 0.026 s]
|
||||
Range (min … max): 8.818 s … 9.515 s 5 runs
|
||||
|
||||
Baseline
|
||||
--------
|
||||
Benchmark 2: bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=local //...
|
||||
Time (mean ± σ): 9.613 s ± 0.214 s [User: 0.077 s, System: 0.038 s]
|
||||
Range (min … max): 9.418 s … 9.866 s 5 runs
|
||||
|
||||
Baseline llvm14 without sandbox: 12 seconds:
|
||||
Benchmark 3: bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=sandboxed --experimental_reuse_sandbox_directories //...
|
||||
Time (mean ± σ): 13.314 s ± 0.201 s [User: 0.109 s, System: 0.028 s]
|
||||
Range (min … max): 13.093 s … 13.536 s 5 runs
|
||||
|
||||
bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --spawn_strategy=local --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 13.268s, Critical Path: 1.19s
|
||||
Benchmark 4: bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=sandboxed --experimental_reuse_sandbox_directories //...
|
||||
Time (mean ± σ): 13.829 s ± 0.376 s [User: 0.093 s, System: 0.046 s]
|
||||
Range (min … max): 13.573 s … 14.489 s 5 runs
|
||||
|
||||
zig cc without the sandbox. 15 seconds:
|
||||
Benchmark 5: bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=sandboxed //...
|
||||
Time (mean ± σ): 35.101 s ± 0.365 s [User: 0.103 s, System: 0.028 s]
|
||||
Range (min … max): 34.747 s … 35.673 s 5 runs
|
||||
|
||||
bazel build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 23.836s, Critical Path: 3.47s
|
||||
Benchmark 6: bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=sandboxed //...
|
||||
Time (mean ± σ): 21.071 s ± 0.378 s [User: 0.091 s, System: 0.034 s]
|
||||
Range (min … max): 20.611 s … 21.635 s 5 runs
|
||||
|
||||
Cost of Sandbox
|
||||
---------------
|
||||
|
||||
llvm 14:
|
||||
|
||||
bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 89.776s, Critical Path: 12.01s
|
||||
|
||||
zig cc:
|
||||
|
||||
bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 58.919s, Critical Path: 8.38s
|
||||
|
||||
Possible mitigation: Sandboxfs
|
||||
------------------------------
|
||||
|
||||
No dice.
|
||||
|
||||
llvm 14: does not build at all:
|
||||
|
||||
$ bazel build --experimental_sandboxfs_path=sandboxfs --experimental_use_sandboxfs --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
ERROR: /code/test-zigcc/BUILD:4:14: Compiling main.cc failed: (Exit 5): cc_wrapper.sh failed: error executing command external/llvm_toolchain_with_sysroot/bin/cc_wrapper.sh -U_FORTIFY_SOURCE '--target=x86_64-unknown-linux-gnu' -U_FORTIFY_SOURCE -fstack-protector -fno-omit-frame-pointer -fcolor-diagnostics -Wall ... (remaining 29 arguments skipped)
|
||||
|
||||
Use --sandbox_debug to see verbose messages from the sandbox and retain the sandbox build root for debugging
|
||||
ERROR: could not find clang; PWD="/proc/self/cwd"; PATH="<...>".
|
||||
Target //:zigzag-0 failed to build
|
||||
Use --verbose_failures to see the command lines of failed build steps.
|
||||
INFO: Elapsed time: 7.578s, Critical Path: 0.26s
|
||||
INFO: 3 processes: 3 internal.
|
||||
FAILED: Build did NOT complete successfully
|
||||
Summary
|
||||
'bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=local //...' ran
|
||||
1.04 ± 0.04 times faster than 'bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=local //...'
|
||||
1.44 ± 0.05 times faster than 'bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=sandboxed --experimental_reuse_sandbox_directories //...'
|
||||
1.49 ± 0.06 times faster than 'bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=sandboxed --experimental_reuse_sandbox_directories //...'
|
||||
2.28 ± 0.08 times faster than 'bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=sandboxed //...'
|
||||
3.79 ± 0.12 times faster than 'bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=sandboxed //...'
|
||||
|
||||
|
||||
zig cc:
|
||||
| Command | Mean [s] | Min [s] | Max [s] | Relative |
|
||||
|:---|---:|---:|---:|---:|
|
||||
| `bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=local //...` | 9.257 ± 0.287 | 8.818 | 9.515 | 1.00 |
|
||||
| `bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=local //...` | 9.613 ± 0.214 | 9.418 | 9.866 | 1.04 ± 0.04 |
|
||||
| `bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=sandboxed --experimental_reuse_sandbox_directories //...` | 13.314 ± 0.201 | 13.093 | 13.536 | 1.44 ± 0.05 |
|
||||
| `bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=sandboxed --experimental_reuse_sandbox_directories //...` | 13.829 ± 0.376 | 13.573 | 14.489 | 1.49 ± 0.06 |
|
||||
| `bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux --spawn_strategy=sandboxed //...` | 35.101 ± 0.365 | 34.747 | 35.673 | 3.79 ± 0.12 |
|
||||
| `bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --spawn_strategy=sandboxed //...` | 21.071 ± 0.378 | 20.611 | 21.635 | 2.28 ± 0.08 |
|
||||
|
||||
bazel build --experimental_sandboxfs_path=sandboxfs --experimental_use_sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --experimental_sandboxfs_path=sandboxfs --experimental_use_sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 126.613s, Critical Path: 17.63s
|
||||
|
||||
|
||||
Possible mitigation: `--experimental_reuse_sandbox_directories`
|
||||
---------------------------------------------------------------
|
||||
|
||||
Dice.
|
||||
|
||||
llvm 14:
|
||||
|
||||
bazel build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ...
|
||||
INFO: Elapsed time: 18.335s, Critical Path: 2.00s
|
||||
|
||||
zig cc:
|
||||
|
||||
bazel build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0
|
||||
bazel clean; bazel shutdown
|
||||
bazel --batch build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ...
|
||||
INFO: Elapsed time: 34.777s, Critical Path: 4.71s
|
||||
Explanation:
|
||||
|
||||
* `--spawn_strategy=local` is the baseline. This is the fastest possible
|
||||
approach (no sandbox), but does never runs in production.
|
||||
* `--spawn_strategy=sandboxed` is the default for Bazel. This is what happens
|
||||
if one does not pass any arguments.
|
||||
* `--spawn_strategy=sandboxed --experimental_reuse_sandbox_directories` is an
|
||||
optimization which is meaningful on sandboxes with many files. As of Bazel 6
|
||||
this has been [promoted to stable][3], so safe to use and, as one can see in
|
||||
the numbers, recommended with both bazel-zig-cc and a nontrivial sysroot. As
|
||||
a result, this is the most important benchmark to look at.
|
||||
|
||||
Flame graphs and discussion
|
||||
---------------------------
|
||||
|
||||
Flame graphs and more tests are in results/. As of 2022-12-11 most of the
|
||||
overhead comes from creating and deleting the sandboxes.
|
||||
Flame graphs in results/. Only of historical interest, because the original
|
||||
performance issues have been resolved with a combination of zig and
|
||||
bazel-zig-cc changes.
|
||||
|
||||
[1]: https://sr.ht/~motiejus/bazel-zig-cc
|
||||
[2]: https://github.com/grailbio/bazel-toolchain
|
||||
[3]: https://github.com/bazelbuild/bazel/issues/16138
|
||||
|
||||
@ -1,13 +1,23 @@
|
||||
#!/bin/bash
|
||||
set -xeuo pipefail
|
||||
|
||||
# set to 1 to enable profiling
|
||||
profile=
|
||||
|
||||
_build() {
|
||||
if [[ "$profile" == 1 ]]; then
|
||||
perf=(perf record --call-graph=dwarf -g -o "$HOME/$name.data")
|
||||
else
|
||||
perf=()
|
||||
fi
|
||||
|
||||
name=$1
|
||||
shift
|
||||
bazel build --color=no --curses=no "$@" //:zigzag-0
|
||||
bazel clean --color=no --curses=no
|
||||
bazel shutdown --color=no --curses=no
|
||||
perf record --call-graph=dwarf -g -o "$HOME/$name.data" bazel --batch build --color=no --curses=no "$@" ...
|
||||
bazel build --color=no --curses=no "$@" //:zigzag-0
|
||||
bazel clean --color=no --curses=no
|
||||
bazel shutdown --color=no --curses=no
|
||||
( "${perf[@]}" bazel --batch build --color=no --curses=no "$@" ... ) |& \
|
||||
tee "results/${name}.log"
|
||||
}
|
||||
|
||||
_build zigcc-nosandbox --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
@ -17,7 +27,9 @@ _build llvm --extr
|
||||
_build zigcc-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28
|
||||
_build llvm-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux
|
||||
|
||||
for d in $HOME/*.data; do
|
||||
f=$(basename "$d")
|
||||
echo "perf script -i $d | inferno-collapse-perf | inferno-flamegraph > results/${f%data}svg"
|
||||
done | parallel -v --eta
|
||||
if [[ $profile == 1 ]]; then
|
||||
for d in $HOME/*.data; do
|
||||
f=$(basename "$d")
|
||||
echo "perf script -i $d | inferno-collapse-perf | inferno-flamegraph > results/${f%data}svg"
|
||||
done | parallel -v --eta
|
||||
fi
|
||||
|
||||
Loading…
Reference in New Issue
Block a user