From 6d2ee8cad0b1170883dd13ffddc7b355ee232f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= Date: Tue, 14 Mar 2023 10:15:43 +0200 Subject: [PATCH] replace zopfli with encoding/gzip --- .build.yml | 44 -------------- BUILD | 1 - WORKSPACE | 10 ---- ci/release | 15 +---- release | 39 ------------- relnotes.awk | 37 ------------ third_party/BUILD.zopfli | 20 ------- tools/releaser/BUILD | 6 +- tools/releaser/WORKSPACE | 1 + tools/releaser/main.go | 120 +++++++++++++++++---------------------- tools/releaser/zopfli.go | 10 ---- 11 files changed, 56 insertions(+), 247 deletions(-) delete mode 100644 .build.yml delete mode 100755 release delete mode 100755 relnotes.awk delete mode 100644 third_party/BUILD.zopfli create mode 100644 tools/releaser/WORKSPACE delete mode 100644 tools/releaser/zopfli.go diff --git a/.build.yml b/.build.yml deleted file mode 100644 index 5a232df..0000000 --- a/.build.yml +++ /dev/null @@ -1,44 +0,0 @@ -image: debian/stable -packages: - - direnv - - shellcheck - - qemu-user-static - - binfmt-support - - moreutils - - file - - wine64 -sources: - - https://git.sr.ht/~motiejus/bazel-zig-cc -environment: - CC: /usr/bin/false -triggers: - - action: email - condition: failure - to: motiejus+srht@jakstys.lt -tasks: - - setup: | - sudo apt-get purge gcc -y && sudo apt-get autoremove -y - sudo dpkg --add-architecture arm64 - sudo apt-get update - sudo apt-get install libc6:arm64 -y - - lint_gazelle: | - cd bazel-zig-cc; . .envrc - ./ci/lint - bazel run --color=yes --curses=yes //:gazelle - git diff --exit-code - - test_release: | - cd bazel-zig-cc; . .envrc - ./ci/release - - list_toolchains_platforms: | - cd bazel-zig-cc; . .envrc - ./ci/list_toolchains_platforms - - test_launcher: | - cd bazel-zig-cc; . .envrc - ./ci/launcher --color=yes --curses=yes - - test: | - cd bazel-zig-cc; . .envrc - export BAZEL_ZIG_CC_CACHE_PREFIX=/tmp/bazel-zig-cc-2 - ./ci/test \ - --color=yes --curses=yes \ - --repo_env BAZEL_ZIG_CC_CACHE_PREFIX=$BAZEL_ZIG_CC_CACHE_PREFIX \ - --sandbox_writable_path "$BAZEL_ZIG_CC_CACHE_PREFIX" diff --git a/BUILD b/BUILD index 7180402..dcedea4 100644 --- a/BUILD +++ b/BUILD @@ -8,7 +8,6 @@ load("@bazel_gazelle//:def.bzl", "gazelle") # gazelle:build_file_name BUILD # gazelle:prefix github.com/uber/bazel-zig-cc # gazelle:exclude tools.go -# gazelle:exclude tools/releaser/zopfli.go gazelle(name = "gazelle") diff --git a/WORKSPACE b/WORKSPACE index b8ad0b4..e1b4454 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -82,13 +82,3 @@ http_archive( load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") protobuf_deps() - -ZOPFLI_COMMIT = "831773bc28e318b91a3255fa12c9fcde1606058b" - -http_archive( - name = "zopfli", - build_file = "//:third_party/BUILD.zopfli", - sha256 = "dbc695ae30c815973dc38b39e2f95dc2b249263b7222427d96e5e785092b0f78", - strip_prefix = "zopfli-{}".format(ZOPFLI_COMMIT), - urls = ["https://github.com/google/zopfli/archive/{}.tar.gz".format(ZOPFLI_COMMIT)], -) diff --git a/ci/release b/ci/release index 2d2a2f0..2d89c64 100755 --- a/ci/release +++ b/ci/release @@ -8,17 +8,8 @@ set -xeuo pipefail cd "$(git rev-parse --show-toplevel)" prev_ref=$(git rev-parse HEAD) git commit --allow-empty -m "this is a test commit" -./release --nosign v99.0 -cleanup() { git tag -d v99.0; git reset --hard "$prev_ref"; } +tools/bazel run //tools/releaser -- -tag v99.0.0 +cleanup() { git tag -d v99.0.0; git reset --hard "$prev_ref"; } trap cleanup EXIT -want=" 1 file changed, 2 insertions(+), 2 deletions(-)" -got=$(git show --shortstat HEAD | tail -1) - -if [[ "$want" != "$got" ]]; then - echo wanted: - echo \ \ "$want" - echo got: - echo \ \ "$got" - exit 1 -fi +file bazel-zig-cc-v99.0.0.tar.gz | grep -q 'gzip compressed data' diff --git a/release b/release deleted file mode 100755 index 8c6dae7..0000000 --- a/release +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2023 Uber Technologies, Inc. -# Licensed under the Apache License, Version 2.0 - -set -xeuo pipefail - -sign=(-u motiejus@jakstys.lt) -[[ $1 == "--nosign" ]] && { sign=(); shift; } - -_err(){ >&2 echo "ERROR: $*"; exit 1; } - -git status --porcelain | grep -q "" && - _err "working tree is dirty, commit your changes first." - -[[ "$1" =~ ^v([0-9]+)\.([0-9]+)(\.([0-9]+))?(-rc([0-9]+))?$ ]] || \ - _err "arg1 accepts the following formats: v1.0 v1.0.0 v1.0-rc1 v1.0.1-rc1" - -git tag | grep -q "^$1$" && - _err "tag $1 already exists" - -last_tag=$(git -c 'versionsort.suffix=-' tag -l --sort=v:refname | tail -1) - -{ - echo bazel-zig-cc "$1" - echo - echo Changelog since "$last_tag": - git log --pretty=format:"- [%an] %s" "$last_tag"..HEAD | \ - grep -v "Update release notes for $last_tag" -} | git tag "${sign[@]+"${sign[@]}"}" -F - "$1" - -shasum=$(git archive --prefix="bazel-zig-cc-$1/" --format=tar "$1" | \ - gzip -n | sha256sum | cut -f1 -d" ") - -./relnotes.awk -v tag="$1" -v sha256sum="$shasum" README.md > .README.md -mv .README.md README.md - -git add README.md -git commit -m "Update release notes for $1" diff --git a/relnotes.awk b/relnotes.awk deleted file mode 100755 index 06c6435..0000000 --- a/relnotes.awk +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/awk -f - -# Copyright 2023 Uber Technologies, Inc. -# Licensed under the Apache License, Version 2.0 - -BEGIN {stage=0}; - -!/```/ && stage==0 { - print -} - -/```/ && stage==0 { - print "```" - print "BAZEL_ZIG_CC_VERSION = \""tag"\"" - print "" - print "http_archive(" - print " name = \"bazel-zig-cc\"," - print " sha256 = \""sha256sum"\"," - print " strip_prefix = \"bazel-zig-cc-{}\".format(BAZEL_ZIG_CC_VERSION)," - print " urls = [\"https://git.sr.ht/~motiejus/bazel-zig-cc/archive/{}.tar.gz\".format(BAZEL_ZIG_CC_VERSION)]," - print ")" - stage=1 - next -} - -!/^)$/ && stage==1 { - next -}; - -/^)$/ && stage==1 { - stage=2 - next -}; - -stage==2 { - print; -}; diff --git a/third_party/BUILD.zopfli b/third_party/BUILD.zopfli deleted file mode 100644 index 152c3dc..0000000 --- a/third_party/BUILD.zopfli +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 Uber Technologies, Inc. -# Licensed under the Apache License, Version 2.0 - -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "go_zopfli", - srcs = glob( - [ - "go/zopfli/zopfli.go", - "src/zopfli/*.c", - "src/zopfli/*.h", - ], - exclude = ["src/zopfli/zopfli_bin.c"], - ), - cgo = True, - copts = ["-O2"], - importpath = "github.com/google/zopfli/go/zopfli", - visibility = ["//visibility:public"], -) diff --git a/tools/releaser/BUILD b/tools/releaser/BUILD index 05419f6..f63900c 100644 --- a/tools/releaser/BUILD +++ b/tools/releaser/BUILD @@ -6,13 +6,9 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "releaser_lib", - srcs = [ - "main.go", - "zopfli.go", # keep - ], + srcs = ["main.go"], importpath = "github.com/uber/bazel-zig-cc/tools/releaser", visibility = ["//visibility:private"], - deps = ["@zopfli//:go_zopfli"], # keep ) go_binary( diff --git a/tools/releaser/WORKSPACE b/tools/releaser/WORKSPACE new file mode 100644 index 0000000..92be942 --- /dev/null +++ b/tools/releaser/WORKSPACE @@ -0,0 +1 @@ +workspace(name = "bazel-zig-cc") diff --git a/tools/releaser/main.go b/tools/releaser/main.go index 89289e2..8a74ee6 100644 --- a/tools/releaser/main.go +++ b/tools/releaser/main.go @@ -5,17 +5,18 @@ package main import ( + "bytes" + "compress/gzip" "crypto/sha256" "errors" "flag" "fmt" + "io" "os" "os/exec" "path" "regexp" - "strconv" "strings" - ) var ( @@ -28,14 +29,14 @@ var ( } // regexp for valid tags - tagRegexp = regexp.MustCompile(`^v([0-9]+)\\.([0-9]+)(\\.([0-9]+))(-rc([0-9]+))?$`) + tagRegexp = regexp.MustCompile(`^v([0-9]+)\.([0-9]+)(\.([0-9]+))(-rc([0-9]+))?$`) errTag = errors.New("tag accepts the following formats: v1.0.0 v1.0.1-rc1") ) func main() { if err := run(); err != nil { - fmt.Fprintln(os.Stderr, err) + fmt.Fprintf(os.Stderr, "error: %s\n", err) os.Exit(1) } } @@ -44,18 +45,14 @@ func log(msg string, format ...any) { fmt.Fprintf(flag.CommandLine.Output(), msg+"\n", format...) } -func run() error { +func run() (_err error) { var ( - goVersion string - repoRoot string - skipUpgrades bool - tag string + repoRoot string + tag string ) - flag.StringVar(&goVersion, "go_version", "", "go version for go.mod") flag.StringVar(&repoRoot, "repo_root", os.Getenv("BUILD_WORKSPACE_DIRECTORY"), "root directory of bazel-zig-cc repo") flag.StringVar(&tag, "tag", "", "tag for this release") - flag.BoolVar(&skipUpgrades, "skip_upgrades", false, "skip upgrade checks (testing only)") flag.Usage = func() { fmt.Fprint(flag.CommandLine.Output(), `usage: bazel run //tools/releaser -- -go_version -tag @@ -69,54 +66,27 @@ This utility is intended to handle many of the steps to release a new version. flag.Parse() if tag == "" { - return fmt.Errorf("ERROR: tag is required") + return fmt.Errorf("tag is required") } if !tagRegexp.MatchString(tag) { return errTag } - var goVersionArgs []string - if goVersion != "" { - versionParts := strings.Split(goVersion, ".") - if len(versionParts) < 2 { - flag.Usage() - return errors.New("please provide a valid Go version") - } - if minorVersion, err := strconv.Atoi(versionParts[1]); err != nil { - return fmt.Errorf("%q is not a valid Go version", goVersion) - } else if minorVersion > 0 { - versionParts[1] = strconv.Itoa(minorVersion - 1) - } - goVersionArgs = append(goVersionArgs, "-go", goVersion, "-compat", strings.Join(versionParts, ".")) - } - - // external dependency checks - depChecks := [][]string{ - {"go", "get", "-t", "-u", "./..."}, - append([]string{"tools/mod-tidy"}, goVersionArgs...), - } - // commands that Must Not Fail cmds := [][]string{ - {"tools/bazel", "run", "//:gazelle"}, {"git", "diff", "--stat", "--exit-code"}, {"git", "tag", tag}, } log("Cutting a release:") - if skipUpgrades { - log("SKIPPING: go update commands") - } else { - cmds = append(depChecks, cmds...) - } for _, c := range cmds { cmd := exec.Command(c[0], c[1:]...) cmd.Dir = repoRoot if out, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf( - "ERROR: running %s:%w\n%s", + "run %s: %w\n%s", strings.Join(c, " "), err, out, @@ -124,35 +94,56 @@ This utility is intended to handle many of the steps to release a new version. } } - log("Creating archive bazel-zig-cc-%s.tar", tag) + fpath := path.Join(repoRoot, fmt.Sprintf("bazel-zig-cc-%s.tar.gz", tag)) + tgz, err := os.Create(fpath) + if err != nil { + return err + } + defer func() { + if _err != nil { + os.Remove(fpath) + } + }() + hashw := sha256.New() + gzw, err := gzip.NewWriterLevel(io.MultiWriter(tgz, hashw), gzip.BestCompression) + if err != nil { + return fmt.Errorf("create gzip writer: %w", err) + } + + log("- creating %s", fpath) + + var stderr bytes.Buffer cmd := exec.Command( "git", - append([]string{"archive", "--format=tar", tag}, _paths...)..., + append([]string{ + "archive", + "--format=tar", + // WORKSPACE in the resulting tarball needs to be much + // smaller than of bazel-zig-cc. See #15. + "--add-file=tools/releaser/WORKSPACE", + tag, + }, _paths...)..., ) cmd.Dir = repoRoot + cmd.Stdout = gzw + cmd.Stderr = &stderr - out, err := cmd.Output() - if err != nil { + if err := cmd.Run(); err != nil { var exitError *exec.ExitError errors.As(err, &exitError) - return fmt.Errorf("ERROR: failed to create git archive: %w\n%s", err, exitError.Stderr) + return fmt.Errorf("create git archive: %w\n%s", err, stderr.Bytes()) } - log("Compressing bazel-zig-cc-%s.tar", tag) - - tgz := Gzip(out) - - fpath := path.Join(repoRoot, fmt.Sprintf("bazel-zig-cc-%s.tar.gz", tag)) - if err := os.WriteFile(fpath, tgz, 0o644); err != nil { - return fmt.Errorf("ERROR: write %q: %w", fpath, err) + if err := gzw.Close(); err != nil { + return fmt.Errorf("close gzip stream: %w", err) } - log("Wrote %s", fpath) - - shasum := sha256.Sum256(tgz) - - log("Release boilerplate:\n-----\n" + genBoilerplate(tag, fmt.Sprintf("%x", shasum))) + if err := tgz.Close(); err != nil { + return err + } + log("- wrote %s", fpath) + log("Release:\n-----\n" + genBoilerplate(tag, fmt.Sprintf("%x", hashw.Sum(nil)))) return nil } @@ -171,16 +162,7 @@ http_archive( load("@bazel-zig-cc//toolchain:defs.bzl", zig_toolchains = "toolchains") -# Argument-free will pick reasonable defaults. -zig_toolchains() - -# version, url_formats and host_platform_sha256 are can be set for those who -# wish to control their Zig SDK version and where it is downloaded from -zig_toolchains( - version = "<...>", - url_formats = [ - "https://example.org/zig/zig-{host_platform}-{version}.{_ext}", - ], - host_platform_sha256 = { ... }, -)`, version, shasum) +# plain zig_toolchains() will pick reasonable defaults. See +# toolchain/defs.bzl:toolchains on how to change the Zig SDK path and version. +zig_toolchains()`, version, shasum) } diff --git a/tools/releaser/zopfli.go b/tools/releaser/zopfli.go deleted file mode 100644 index 40a00d5..0000000 --- a/tools/releaser/zopfli.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import "github.com/google/zopfli/go/zopfli" - -// Gzip compresses a byte array with zopfli. -// -// We use a separate file, because we can make Gazelle ignore it. -func Gzip(in []byte) []byte { - return zopfli.Gzip(in) -}