Restructure tests to all run in bazel

- add rules to run tests for a specific platform - use downloaded buildifier - move lint to a script - rename ci tasks - stop running under qemu-aarch64-static as it doesn't do anything
2022-04-13 12:58:11 +00:00 · 2022-04-13 12:58:11 +00:00 · 7a81e2a129
commit 7a81e2a129
parent 557e75efda
14 changed files with 163 additions and 110 deletions
--- a/.bazelrc
+++ b/.bazelrc
@ -10,6 +10,3 @@ build --extra_toolchains @zig_sdk//toolchain:linux_amd64_gnu.2.19
 build --extra_toolchains @zig_sdk//toolchain:linux_arm64_gnu.2.28
 build --extra_toolchains @zig_sdk//toolchain:darwin_amd64
 build --extra_toolchains @zig_sdk//toolchain:darwin_arm64
-
-test:qemu-aarch64 --test_env=QEMU_LD_PREFIX=/usr/aarch64-linux-gnu/
-test:qemu-aarch64 --run_under=qemu-aarch64-static
--- a/.build.yml
+++ b/.build.yml
@ -3,7 +3,6 @@ packages:
  - direnv
  - shellcheck
  - qemu-user-static
-  - libc6-arm64-cross
  - binfmt-support
 sources:
  - https://git.sr.ht/~motiejus/bazel-zig-cc
@ -16,17 +15,19 @@ triggers:
 tasks:
  - setup: |
      sudo apt-get purge gcc -y && sudo apt-get autoremove -y
-  - test_list_toolchains_platforms: |
+      sudo dpkg --add-architecture arm64
+      sudo apt-get update
+      sudo apt-get install libc6:arm64 -y
+  - list_toolchains_platforms: |
      cd bazel-zig-cc; . .envrc
      echo "Available toolchains:"
      bazel query @zig_sdk//toolchain:*
      echo "Available platforms:"
      bazel query @zig_sdk//platform:*
-  - test_hello_on_toolchains: |
-      cd bazel-zig-cc
+  - test: |
+      cd bazel-zig-cc; . .envrc
      ./ci/test --color=yes --curses=yes
  - lint: |
      cd bazel-zig-cc; . .envrc
-      shellcheck -x $(awk '/#!\/bin\/(ba)?sh/&&FNR==1{print FILENAME}' $(git ls-files))
-      bazel run //:buildifier
+      ./ci/lint
      git diff --exit-code
--- a/.envrc
+++ b/.envrc
@ -1,27 +1,45 @@
 set -eu

-export PATH="$(git rev-parse --show-toplevel)/bin:$PATH"
-_u=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-
+BIN_DIR="$(git rev-parse --show-toplevel)/bin"
+export PATH="$BIN_DIR:$PATH"

-#for os in linux darwin; do
-#    for arch in amd64 arm64; do
-#        hash=$(direnv fetchurl "${_u}$os-$arch")
-#        echo -e "$os-$arch\t$hash"
-#    done
-#done
+_u_bzl=https://github.com/bazelbuild/bazelisk/releases/download/v1.10.1/bazelisk-
+_u_bldf=https://github.com/bazelbuild/buildtools/releases/download/5.0.1/buildifier-

+if [[ "${PRINT_TOOL_HASHES:-no}" = "yes" ]]; then
+    for os in linux darwin; do
+        for arch in amd64 arm64; do
+            hash_bzl=$(direnv fetchurl "${_u_bzl}$os-$arch")
+            hash_bldf=$(direnv fetchurl "${_u_bldf}$os-$arch")
+            echo -e "bzl:  $os-$arch\t$hash_bzl"
+            echo -e "bldf: $os-$arch\t$hash_bldf"
+        done
+    done
+fi
+
+# to fetch the hashes, run:
+# $ PRINT_TOOL_HASHES=yes bash .envrc
 case "$(uname | tr A-Z a-z)-$(uname -m)" in
    linux-x86_64)
-        bzl=$(direnv fetchurl "${_u}linux-amd64" sha256-TLU0xSzdR6YiPUWW1TDnyceFQ4qzsKSf80fpkcIQss0=);;
+        bzl=$(direnv fetchurl "${_u_bzl}linux-amd64" sha256-TLU0xSzdR6YiPUWW1TDnyceFQ4qzsKSf80fpkcIQss0=)
+        bldf=$(direnv fetchurl "${_u_bldf}linux-amd64" sha256-Ptc1jHxqHKIW3FZukFT9C5ehSCywt+YQkr6IfUJhXF0=)
+        ;;
    linux-aarch64)
-        bzl=$(direnv fetchurl "${_u}linux-arm64" sha256-wd5oYN1PjV4uwnAJe9RtaiEblxoLizhVl4S9BR6pUKE=);;
+        bzl=$(direnv fetchurl "${_u_bzl}linux-arm64" sha256-wd5oYN1PjV4uwnAJe9RtaiEblxoLizhVl4S9BR6pUKE=)
+        bldf=$(direnv fetchurl "${_u_bldf}linux-arm64" sha256-xlfGKPynK34ERvGlQiMXIqELpDIVl71vYkml2mBgtv8==)
+        ;;
    darwin-x86_64)
-        bzl=$(direnv fetchurl "${_u}darwin-amd64" sha256-5IW7+EUy0CpgsOsjxwJhC1QI3zoZkIek8rXgmVu/LVo=);;
+        bzl=$(direnv fetchurl "${_u_bzl}darwin-amd64" sha256-5IW7+EUy0CpgsOsjxwJhC1QI3zoZkIek8rXgmVu/LVo=)
+        bldf=$(direnv fetchurl "${_u_bldf}darwin-amd64" sha256-LLClRoNjPvbeTgSRBy4i5mrJxjiQUUMrdiAN7u6vk/s=)
+        ;;
    darwin-arm64)
-        bzl=$(direnv fetchurl "${_u}darwin-arm64" sha256-wi1IYBRm2dOwQ8zXQFHy9CMPm59FCfCXAXyXMDqojRM=);;
+        bzl=$(direnv fetchurl "${_u_bzl}darwin-arm64" sha256-wi1IYBRm2dOwQ8zXQFHy9CMPm59FCfCXAXyXMDqojRM=)
+        bldf=$(direnv fetchurl "${_u_bldf}darwin-arm64" sha256-TaIzFfDcyr+HjIIn/dvM81VFsjs8tiJb/PMQdonMQ2Q=)
+        ;;
    *)
        >&2 echo "unsupported architecture tuple $(uname | tr A-Z a-z)-$(uname -m)"
        exit 1;;
 esac

-ln -sf "${bzl}" bin/bazel
+ln -sf "${bzl}" "$BIN_DIR/bazel"
+ln -sf "${bldf}" "$BIN_DIR/buildifier"
--- a/.gitignore
+++ b/.gitignore
@ -2,6 +2,7 @@
 *.sw[op]

 /bin/bazel
+/bin/buildifier
 /bin/bazelisk-*

 /bazel-bazel-zig-cc
--- a/3
+++ b/3
@ -1,10 +1,7 @@
 load("@bazel_gazelle//:def.bzl", "gazelle")
-load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier")

 # gazelle:map_kind go_binary go_binary //rules:rules_go.bzl

 # gazelle:build_file_name BUILD
 # gazelle:prefix git.sr.ht/~motiejus/bazel-zig-cc
 gazelle(name = "gazelle")
-
-buildifier(name = "buildifier")
--- a/README.md
+++ b/README.md
@ -69,7 +69,7 @@ For example, the toolchain `linux_amd64_gnu.2.28` is aliased to
 used, run:

 ```
-$ bazel query @zig_sdk//... | grep _toolchain$
+$ bazel query @zig_sdk//toolchain/...
 ```

 ## Specifying non-default toolchains
@ -151,48 +151,22 @@ may apply to aarch64, but the author didn't find a need to test it (yet).
 - [golang/go #46644 cmd/link: with CC=zig: SIGSERV when cross-compiling to darwin/amd64](https://github.com/golang/go/issues/46644) (CLOSED, thanks kubkon)

 # Testing
-
-## build & run linux cgo + glibc
-
-```
-$ bazel build --platforms @zig_sdk//platform:linux_amd64 //test/go:go
-$ file bazel-out/k8-opt-ST-d17813c235ce/bin/test/go/go_/go
-bazel-out/k8-opt-ST-d17813c235ce/bin/test/go/go_/go: ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.0.0, Go BuildID=redacted, with debug_info, not stripped
-$ bazel-out/k8-opt-ST-d17813c235ce/bin/test/go/go_/go
-hello, world
-```
-
-## test linux cgo + musl on arm64 (under qemu-aarch64)
-
-```
-$ bazel test \
-    --config=qemu-aarch64 \
-    --platforms @zig_sdk//platform:linux_arm64 \
-    --extra_toolchains @zig_sdk//toolchain:linux_arm64_musl //test/...
-...
-INFO: Build completed successfully, 10 total actions
-//test/go:go_test                                                        PASSED in 0.2s
-```
-
-## macos cgo
-
-```
-$ bazel build --platforms @zig_sdk//platform:darwin_amd64 //test/go:go
-...
-$ file bazel-out/k8-opt-ST-d17813c235ce/bin/test/go/go_/go
-bazel-out/k8-opt-ST-d17813c235ce/bin/test/go/go_/go: Mach-O 64-bit x86_64 executable, flags:<NOUNDEFS|DYLDLINK|TWOLEVEL|PIE|HAS_TLV_DESCRIPTORS>
-```
-
 ## Transient docker environment

+First of all, make sure that your kernel is configured to run arm64 binaries.
+You can either `apt install qemu-user-static binfmt-support`; this should setup
+`binfmt_misc` to handle arm64 binaries. Or you can use this handy dockerized
+script `docker run --rm --privileged multiarch/qemu-user-static --reset -p yes`.
+
 ```
-$ docker run -e CC=/usr/bin/false -ti --rm -v $(pwd):/x -w /x debian:bullseye-slim
-# apt update && apt install -y direnv git
+$ docker run -e CC=/usr/bin/false -ti --rm -v $(git rev-parse --show-toplevel):/x -w /x debian:bullseye-slim
+# dpkg --add-architecture arm64 && apt update && apt install -y direnv git shellcheck libc6:arm64
 # . .envrc
+# ./ci/test
+# ./ci/lint
 ```

-And run the `bazel build` commands above. Take a look at `.build.yml` and see
-how CI does it.
+See `ci/test` for how tests are run.

 # Questions & Contributions

--- a/12
+++ b/12
@ -42,18 +42,6 @@ go_repositories()

 gazelle_dependencies(go_repository_default_config = "@//:WORKSPACE")

-# protobuf is required for //:buildifier
-http_archive(
-    name = "com_google_protobuf",
-    sha256 = "25f1292d4ea6666f460a2a30038eef121e6c3937ae0f61d610611dfb14b0bd32",
-    strip_prefix = "protobuf-3.19.1",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.19.1.zip"],
-)
-
-load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
-
-protobuf_deps()
-
 load(
    "//toolchain:defs.bzl",
    zig_register_toolchains = "register_toolchains",
--- a/bin/mod-tidy
+++ b/bin/mod-tidy
@ -2,9 +2,8 @@

 set -xeuo pipefail

-cd "$(git rev-parse --show-toplevel)"
-bazel build @go_sdk//:go_sdk
-bazel-bazel-zig-cc/external/go_sdk/bin/go mod tidy
+cd "$(git rev-parse --show-toplevel)/"
+bazel run @go_sdk//:bin/go -- mod tidy
 exec bazel run //:gazelle -- update-repos \
    -from_file=go.mod \
    -prune \
--- a/ci/lint
+++ b/ci/lint
@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+REPO_ROOT=$(git rev-parse --show-toplevel)
+cd "$REPO_ROOT"
+
+# shellcheck disable=SC2046
+shellcheck -x $(awk '/#!\/bin\/(ba)?sh/&&FNR==1{print FILENAME}' $(git ls-files))
+find . \( -name 'WORKSPACE' -o -name 'BUILD' -o -name '*.bzl' \) -exec buildifier {} +
--- a/ci/test
+++ b/ci/test
@ -1,34 +1,4 @@
 #!/bin/bash
 set -euo pipefail

-cd "$(dirname "$0")/.."
-. .envrc
-
-_run() {
-    >&2 echo
-    >&2 echo "  $*"
-    >&2 echo
-    "$@"
-}
-
-while read -r action platform toolchain config; do
-    args=("$@")
-    if [[ $config != : ]]; then
-        args+=(--config="$config")
-    fi
-
-    args+=(\
-        --platforms "@zig_sdk//platform:${platform}" \
-        --extra_toolchains "@zig_sdk//toolchain:${toolchain}" \
-        //test/... \
-    )
-
-    _run bazel "$action" "${args[@]}"
-
-done <<EOF
-test linux_amd64 linux_amd64_musl :
-test linux_amd64 linux_amd64_gnu.2.19 :
-test linux_arm64 linux_arm64_musl qemu-aarch64
-test linux_arm64 linux_arm64_gnu.2.28 qemu-aarch64
-build darwin_amd64 darwin_amd64 :
-EOF
+bazel test ...
--- a/go.mod
+++ b/go.mod
@ -1,3 +1,3 @@
 module git.sr.ht/~motiejus/bazel-zig-cc

-go 1.16
+go 1.18
--- a/rules/BUILD
+++ b/rules/BUILD
@ -0,0 +1 @@
+package(default_visibility = ["//test:__pkg__"])
--- a/rules/platform.bzl
+++ b/rules/platform.bzl
@ -0,0 +1,67 @@
+def _platform_transition_impl(settings, attr):
+    _ignore = settings
+    return {
+        "//command_line_option:platforms": "@zig_sdk//platform:{}".format(attr.platform),
+        "//command_line_option:extra_toolchains": ["@zig_sdk//toolchain:{}".format(tc) for tc in attr.extra_toolchains],
+    }
+
+_platform_transition = transition(
+    implementation = _platform_transition_impl,
+    inputs = [],
+    outputs = [
+        "//command_line_option:platforms",
+        "//command_line_option:extra_toolchains",
+    ],
+)
+
+def _platform_binary_impl(ctx):
+    source_info = ctx.attr.src[DefaultInfo]
+
+    executable = None
+    if source_info.files_to_run and source_info.files_to_run.executable:
+        executable = ctx.actions.declare_file("{}_{}".format(ctx.file.src.basename, ctx.attr.platform))
+        ctx.actions.run_shell(
+            command = "cp {} {}".format(source_info.files_to_run.executable.path, executable.path),
+            inputs = [source_info.files_to_run.executable],
+            outputs = [executable],
+        )
+
+    return [DefaultInfo(
+        files = depset(ctx.files.src),
+        executable = executable,
+    )]
+
+_attrs = {
+    "src": attr.label(
+        allow_single_file = True,
+        mandatory = True,
+        doc = "Target to build.",
+    ),
+    "platform": attr.string(
+        doc = "The platform to build the target for.",
+    ),
+    "extra_toolchains": attr.string_list(
+        doc = "The toolchains to provide as extra_toolchains.",
+    ),
+    "_allowlist_function_transition": attr.label(
+        default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
+    ),
+}
+
+# wrap a single exectable and build it for the specified platform passing in
+# the extra_toolchains.
+platform_binary = rule(
+    implementation = _platform_binary_impl,
+    cfg = _platform_transition,
+    attrs = _attrs,
+    executable = True,
+)
+
+# wrap a single test target and build it for the specified platform passing in
+# the extra_toolchains.
+platform_test = rule(
+    implementation = _platform_binary_impl,
+    cfg = _platform_transition,
+    attrs = _attrs,
+    test = True,
+)
--- a/test/go/BUILD
+++ b/test/go/BUILD
@ -1,5 +1,6 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 load("//rules:rules_go.bzl", "go_binary")
+load("@bazel-zig-cc//rules:platform.bzl", "platform_binary", "platform_test")

 go_library(
    name = "go_lib",
@ -12,7 +13,6 @@ go_library(
 go_binary(
    name = "go",
    embed = [":go_lib"],
-    static = "on",
    visibility = ["//visibility:public"],
 )

@ -21,3 +21,34 @@ go_test(
    srcs = ["hello_test.go"],
    embed = [":go_lib"],
 )
+
+[
+    platform_binary(
+        name = "go_{}".format(toolchain),
+        src = "go",
+        extra_toolchains = [toolchain],
+        platform = platform,
+    )
+    for platform, toolchain in [
+        ("linux_amd64", "linux_amd64_musl"),
+        ("linux_amd64", "linux_amd64_gnu.2.19"),
+        ("linux_arm64", "linux_arm64_musl"),
+        ("linux_arm64", "linux_arm64_gnu.2.28"),
+        ("darwin_amd64", "darwin_amd64"),
+    ]
+]
+
+[
+    platform_test(
+        name = "go_test_{}".format(toolchain),
+        src = "go_test",
+        extra_toolchains = [toolchain],
+        platform = platform,
+    )
+    for platform, toolchain in [
+        ("linux_amd64", "linux_amd64_musl"),
+        ("linux_amd64", "linux_amd64_gnu.2.19"),
+        ("linux_arm64", "linux_arm64_musl"),
+        ("linux_arm64", "linux_arm64_gnu.2.28"),
+    ]
+]
				`@ -0,0 +1 @@`
				`package(default_visibility = ["//test:__pkg__"])`