From 23ea22f307bf1144d2415402a229f053463a8bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Mon, 12 Dec 2022 09:43:08 +0200 Subject: [PATCH] benchmark updates: hermetic-sandbox doesn't matter --- README.md | 106 ++++++++++++++++++++++++++++++--------------- WORKSPACE | 6 +-- results/profile.sh | 23 ++++------ 3 files changed, 83 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index b7f4840..1cf5972 100644 --- a/README.md +++ b/README.md @@ -8,55 +8,93 @@ The test -------- Compiles 64 small binaries with [bazel-zig-cc][1] and [llvm14][2]. The tests -were run on an x86_64 8-core machine running Ubuntu 22.04. +were run on an x86_64 8-core machine running Ubuntu 22.04. The llvm's sysroot +is this: -Steps to reproduce ------------------- + sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3", + urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"], -Baseline is llvm14: 12-13 seconds: +... but with extra ~8k files, so the number of files in the sysroot is exactly +the same as in zig sdk. -``` -bazel --batch clean; bazel --batch build --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux ... -INFO: Elapsed time: 12.454s, Critical Path: 1.11s -``` +Baseline +-------- -zig cc without the sandbox. Which means different invocations of `zig cc` will -see that all files in `zig_lib_dir` are the same file. 17 seconds: +Baseline llvm14 without sandbox: 12 seconds: -``` -bazel --batch clean; bazel --batch build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ... -INFO: Elapsed time: 17.021s, Critical Path: 1.67s -``` + bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --spawn_strategy=local --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ... + INFO: Elapsed time: 11.971s, Critical Path: 0.41s -zig cc plain: 61 seconds. Sandbox is on a real disk, which means it will take -even longer to re-hash all its dependencies: +zig cc without the sandbox. 15 seconds: -``` -bazel --batch clean; bazel --batch build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ... -INFO: Elapsed time: 61.128s, Critical Path: 8.35s -``` + bazel build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ... + INFO: Elapsed time: 15.631s, Critical Path: 1.99s -zig cc with hardlinks: `--config=hermetic-sandbox`. This uses hardlinks to -zig_sdk instead of symlinks: +Cost of Sandbox +--------------- -``` -bazel --batch clean; bazel --batch build --config=hermetic-sandbox --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ... -INFO: Elapsed time: 45.886s, Critical Path: 9.56s -``` +llvm 14: -llvm with sysroot (this sysroot is about half the size of zig): + bazel build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ... + INFO: Elapsed time: 109.677s, Critical Path: 13.60s + +zig cc: + + bazel build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ... + INFO: Elapsed time: 116.370s, Critical Path: 16.23s + +Possible mitigation: Sandboxfs +------------------------------ + +No dice. + +llvm 14: + + bazel build --experimental_sandboxfs_path=sandboxfs --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --experimental_sandboxfs_path=sandboxfs --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ... + INFO: Elapsed time: 96.803s, Critical Path: 12.42s + +zig cc: + + bazel build --experimental_sandboxfs_path=sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --experimental_sandboxfs_path=sandboxfs --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ... + INFO: Elapsed time: 109.347s, Critical Path: 13.94s + +Possible mitigation: `--experimental_reuse_sandbox_directories` +--------------------------------------------------------------- + +Dice. + +llvm 14: + + bazel build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ... + INFO: Elapsed time: 25.252s, Critical Path: 4.44s + +zig cc: + + bazel build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 //:zigzag-0 + bazel clean; bazel shutdown + bazel --batch build --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 ... + INFO: Elapsed time: 29.132s, Critical Path: 3.88s -``` -bazel --batch clean; bazel --batch build --config=hermetic-sandbox --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux ... -INFO: Elapsed time: 25.644s, Critical Path: 3.03s -``` Flame graphs and discussion --------------------------- -Flame graphs are in results/. As of 2022-12-11 most of the overhead comes from -creating and deleting the sandboxes. Time for a new sandboxfs! - +Flame graphs and more tests are in results/. As of 2022-12-11 most of the +overhead comes from creating and deleting the sandboxes. [1]: https://sr.ht/~motiejus/bazel-zig-cc [2]: https://github.com/grailbio/bazel-toolchain diff --git a/WORKSPACE b/WORKSPACE index 4cc380f..af62515 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,6 +1,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -## LLVM-14 +## LLVM BAZEL_TOOLCHAIN_TAG = "0.7.2" BAZEL_TOOLCHAIN_SHA = "f7aa8e59c9d3cafde6edb372d9bd25fb4ee7293ab20b916d867cd0baaa642529" @@ -37,8 +37,8 @@ filegroup( visibility = ["//visibility:public"], ) """, - sha256 = "84656a6df544ecef62169cfe3ab6e41bb4346a62d3ba2a045dc5a0a2ecea94a3", - urls = ["https://commondatastorage.googleapis.com/chrome-linux-sysroot/toolchain/2202c161310ffde63729f29d27fe7bb24a0bc540/debian_stretch_amd64_sysroot.tar.xz"], + sha256 = "4498fa8ce2df8830afd88e0bbf7c8d353a13c3f2454abc507a057889075af904", + urls = ["https://dl.jakstys.lt/zig/debian_stretch_amd64_sysroot-padding15353.tar.xz"], ) llvm_toolchain( diff --git a/results/profile.sh b/results/profile.sh index 0296db0..8936fbd 100755 --- a/results/profile.sh +++ b/results/profile.sh @@ -5,24 +5,17 @@ _build() { name=$1 shift bazel build --color=no --curses=no "$@" //:zigzag-0 - bazel shutdown - bazel --batch clean --color=no --curses=no + bazel clean --color=no --curses=no + bazel shutdown --color=no --curses=no perf record --call-graph=dwarf -g -o "$HOME/$name.data" bazel --batch build --color=no --curses=no "$@" ... } -_build zigcc-hermetic-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox -_build zigcc-hermetic-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox -_build llvm-hermetic-j1 --jobs=1 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux --config=hermetic-sandbox -_build llvm-hermetic-j8 --jobs=8 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux --config=hermetic-sandbox -_build llvm-sysroot-hermetic-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox -_build llvm-sysroot-hermetic-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 --config=hermetic-sandbox - -_build zigcc-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 -_build zigcc-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 -_build llvm-j1 --jobs=1 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux -_build llvm-j8 --jobs=8 --extra_toolchains=@llvm_toolchain//:cc-toolchain-x86_64-linux -_build llvm-sysroot-j1 --jobs=1 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 -_build llvm-sysroot-j8 --jobs=8 --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 +_build zigcc-nosandbox --spawn_strategy=local --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 +_build llvm-nosandbox --spawn_strategy=local --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux +_build zigcc --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 +_build llvm --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux +_build zigcc-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --platforms=@zig_sdk//libc_aware/platform:linux_amd64_gnu.2.28 +_build llvm-reuse-sandbox-dirs --experimental_reuse_sandbox_directories --extra_toolchains=@llvm_toolchain_with_sysroot//:cc-toolchain-x86_64-linux for d in $HOME/*.data; do f=$(basename "$d")