commit 5001faa690ce12374260334beb5f26dfa3f61613 (tree)
parent 10046f9a52c089c656bfd4578e4ef17671ff1f75
Author: Andrew Kelley <andrew@ziglang.org>
Date: Sun, 3 Nov 2019 14:00:28 -0500
ci: stop assuming that azure will complete last
Each of the 3 CI services now trigger a sr.ht build via
the on_master_success script. The sr.ht build checks if all builds have
completed successfully by trying to download the JSON file for the
particular version. If all builds have completed successfully then this
sr.ht job will update the download page.
Diffstat:
11 files changed, 142 insertions(+), 64 deletions(-)
diff --git a/.builds/freebsd.yml b/.builds/freebsd.yml
@@ -3,8 +3,11 @@ packages:
- cmake
- py27-s3cmd
- wget
+ - curl
+ - jq
secrets:
- 6c60aaee-92e7-4e7d-812c-114817689b4d
+ - 2cd8c203-ba06-4504-be49-1e2b9b71efdd
sources:
- https://github.com/ziglang/zig
tasks:
diff --git a/ci/azure/linux_script b/ci/azure/linux_script
@@ -25,6 +25,7 @@ cd build
cmake .. -DCMAKE_BUILD_TYPE=Release
make -j2 install
./zig build test -Denable-qemu
+VERSION="$(./zig version)"
if [ "${BUILD_REASON}" != "PullRequest" ]; then
ARTIFACTSDIR="$BUILDDIR/artifacts"
@@ -44,7 +45,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
- s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/linux-$VERSION.json"
+ s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-linux-$VERSION.json"
# `set -x` causes these variables to be mangled.
# See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
@@ -52,4 +53,5 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL"
echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM"
echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE"
+ echo "##vso[task.setvariable variable=version;isOutput=true]$VERSION"
fi
diff --git a/ci/azure/macos_script b/ci/azure/macos_script
@@ -104,7 +104,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
- s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/macos-$VERSION.json"
+ s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-macos-$VERSION.json"
# `set -x` causes these variables to be mangled.
# See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
diff --git a/ci/azure/on_master_success b/ci/azure/on_master_success
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -x
+set -e
+
+sudo apt-get update -y
+sudo apt-get install -y curl jq
+
+OAUTH_TOKEN="$(cat "$DOWNLOADSECUREFILE_SECUREFILEPATH")"
+./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
diff --git a/ci/azure/pipelines.yml b/ci/azure/pipelines.yml
@@ -55,18 +55,10 @@ jobs:
pool:
vmImage: 'ubuntu-16.04'
variables:
- macos_tarball: $[ dependencies.BuildMacOS.outputs['main.tarball'] ]
- macos_shasum: $[ dependencies.BuildMacOS.outputs['main.shasum'] ]
- macos_bytesize: $[ dependencies.BuildMacOS.outputs['main.bytesize'] ]
- linux_tarball: $[ dependencies.BuildLinux.outputs['main.tarball'] ]
- linux_shasum: $[ dependencies.BuildLinux.outputs['main.shasum'] ]
- linux_bytesize: $[ dependencies.BuildLinux.outputs['main.bytesize'] ]
- windows_tarball: $[ dependencies.BuildWindows.outputs['main.tarball'] ]
- windows_shasum: $[ dependencies.BuildWindows.outputs['main.shasum'] ]
- windows_bytesize: $[ dependencies.BuildWindows.outputs['main.bytesize'] ]
+ version: $[ dependencies.BuildLinux.outputs['main.version'] ]
steps:
- task: DownloadSecureFile@1
inputs:
- secureFile: s3cfg
- - script: ci/azure/update_download_page
- displayName: 'Update download page'
+ secureFile: oauth_token
+ - script: ci/azure/on_master_success
+ displayName: 'master branch success hook'
diff --git a/ci/azure/update_download_page b/ci/azure/update_download_page
@@ -1,47 +0,0 @@
-#!/bin/sh
-
-set -x
-set -e
-
-SRCDIR=$(pwd)
-rm -rf .git
-sudo apt-get update -y
-sudo apt-get install -y s3cmd curl jq
-
-cd "$HOME"
-wget "https://ziglang.org/builds/$LINUX_TARBALL"
-tar xf $LINUX_TARBALL
-ZIGDIR=$(basename -s .tar.xz $LINUX_TARBALL)
-ZIG="$ZIGDIR/zig"
-LANGREF="$ZIGDIR/langref.html"
-VERSION=$($ZIG version)
-SRCTARBALLDIR="zig-$VERSION"
-export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
-mv "$SRCDIR" "$SRCTARBALLDIR"
-tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
-export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
-export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
-
-# the aarch64 build has to be there too
-AARCH64_LINUX_JSON=$(curl --fail "https://ziglang.org/builds/aarch64-linux-$VERSION.json" || exit 1)
-export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
-export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
-export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
-
-# the freebsd build has to be there too
-FREEBSD_JSON=$(curl --fail "https://ziglang.org/builds/freebsd-$VERSION.json" || exit 1)
-export FREEBSD_TARBALL="$(echo "$FREEBSD_JSON" | jq .tarball -r)"
-export FREEBSD_BYTESIZE="$(echo "$FREEBSD_JSON" | jq .size -r)"
-export FREEBSD_SHASUM="$(echo "$FREEBSD_JSON" | jq .shasum -r)"
-
-git clone https://github.com/ziglang/www.ziglang.org --depth 1
-cd www.ziglang.org
-export MASTER_DATE=$(date +%Y-%m-%d)
-env
-"../$ZIG" run update-download-page.zig
-
-mv "$DOWNLOADSECUREFILE_SECUREFILEPATH" "$HOME/.s3cfg"
-s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
-s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
-s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
diff --git a/ci/azure/windows_upload b/ci/azure/windows_upload
@@ -30,7 +30,7 @@ if [ "${BUILD_REASON}" != "PullRequest" ]; then
echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
- s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/windows-$VERSION.json"
+ s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json"
# `set -x` causes these variables to be mangled.
# See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html
diff --git a/ci/drone/linux_script b/ci/drone/linux_script
@@ -8,7 +8,7 @@ BUILDDIR="$(pwd)"
DISTDIR="$(pwd)/dist"
apk update
-apk add py3-pip xz perl-utils
+apk add py3-pip xz perl-utils jq curl
pip3 install s3cmd
# Make the `zig version` number consistent.
@@ -60,4 +60,8 @@ if [ -z "$DRONE_PULL_REQUEST" ]; then
s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/$TRIPLEARCH-linux-$VERSION.json"
+ if [ "$GITBRANCH" = "master" ]; then
+ cd "$BUILDDIR"
+ ./ci/srht/on_master_success "$VERSION" "$SRHT_OAUTH_TOKEN"
+ fi
fi
diff --git a/ci/srht/freebsd_script b/ci/srht/freebsd_script
@@ -85,5 +85,11 @@ if [ -f ~/.s3cfg ]; then
echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE
s3cmd put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE"
- s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/freebsd-$VERSION.json"
+ s3cmd put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
+
+ if [ "$GITBRANCH" = "master" ]; then
+ OAUTH_TOKEN="$(cat ~/.oauth_token)"
+ cd "$ZIGDIR"
+ ./ci/srht/on_master_success "$VERSION" "$OAUTH_TOKEN"
+ fi
fi
diff --git a/ci/srht/on_master_success b/ci/srht/on_master_success
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+# This script must run on a lot of different platforms.
+# It assumes that the following things are installed:
+# * curl
+# * jq
+
+set -x
+set -e
+
+VERSION="$1"
+OAUTH_TOKEN="$2"
+YML_FILE="tmp.yml"
+
+echo "image: ubuntu/xenial" >> "$YML_FILE"
+echo "packages:" >> "$YML_FILE"
+echo " - s3cmd" >> "$YML_FILE"
+echo " - curl" >> "$YML_FILE"
+echo " - jq" >> "$YML_FILE"
+echo "secrets:" >> "$YML_FILE"
+echo " - 6c60aaee-92e7-4e7d-812c-114817689b4d" >> "$YML_FILE"
+echo "sources:" >> "$YML_FILE"
+echo " - https://github.com/ziglang/zig" >> "$YML_FILE"
+echo "tasks:" >> "$YML_FILE"
+echo " - build: cd zig && ./ci/srht/update_download_page $VERSION" >> "$YML_FILE"
+
+jq <$YML_FILE -sR '{
+ "manifest": .,
+ }' | curl \
+ -H Authorization:"token $OAUTH_TOKEN" \
+ -H Content-Type:application/json \
+ -X POST \
+ -d @- "https://builds.sr.ht/api/jobs"
diff --git a/ci/srht/update_download_page b/ci/srht/update_download_page
@@ -0,0 +1,75 @@
+#!/bin/sh
+
+set -x
+set -e
+
+VERSION="$1"
+SRCDIR="$(pwd)"
+NATIVE_TARBALL="zig-linux-$(uname -m)-$VERSION.tar.xz"
+
+# Check for all the builds being completed. It's expected that this script is run several times
+# before they are all complete.
+AARCH64_LINUX_JSON_URL="https://ziglang.org/builds/aarch64-linux-$VERSION.json"
+X86_64_LINUX_JSON_URL="https://ziglang.org/builds/x86_64-linux-$VERSION.json"
+X86_64_WINDOWS_JSON_URL="https://ziglang.org/builds/x86_64-windows-$VERSION.json"
+X86_64_MACOS_JSON_URL="https://ziglang.org/builds/x86_64-macos-$VERSION.json"
+X86_64_FREEBSD_JSON_URL="https://ziglang.org/builds/x86_64-freebsd-$VERSION.json"
+
+# If any of these fail, it's not really this job failing; rather we have detected
+# that this job will be called again later when other jobs have completed.
+curl --fail -I "$AARCH64_LINUX_JSON_URL" || exit 0
+curl --fail -I "$X86_64_LINUX_JSON_URL" || exit 0
+curl --fail -I "$X86_64_WINDOWS_JSON_URL" || exit 0
+curl --fail -I "$X86_64_MACOS_JSON_URL" || exit 0
+curl --fail -I "$X86_64_FREEBSD_JSON_URL" || exit 0
+
+rm -rf .git
+
+cd "$HOME"
+wget "https://ziglang.org/builds/$NATIVE_TARBALL"
+tar xf "$NATIVE_TARBALL"
+ZIGDIR=$(basename -s .tar.xz $NATIVE_TARBALL)
+ZIG="$ZIGDIR/zig"
+LANGREF="$ZIGDIR/langref.html"
+SRCTARBALLDIR="zig-$VERSION"
+export SRC_TARBALL="$SRCTARBALLDIR.tar.xz"
+mv "$SRCDIR" "$SRCTARBALLDIR"
+tar cfJ "$SRC_TARBALL" "$SRCTARBALLDIR"
+export SRC_SHASUM=$(sha256sum $SRC_TARBALL | cut '-d ' -f1)
+export SRC_BYTESIZE=$(wc -c < $SRC_TARBALL)
+
+X86_64_WINDOWS_JSON=$(curl --fail "X86_64_WINDOWS_JSON_URL" || exit 1)
+export X86_64_WINDOWS_TARBALL="$(echo "$X86_64_WINDOWS_JSON" | jq .tarball -r)"
+export X86_64_WINDOWS_BYTESIZE="$(echo "$X86_64_WINDOWS_JSON" | jq .size -r)"
+export X86_64_WINDOWS_SHASUM="$(echo "$X86_64_WINDOWS_JSON" | jq .shasum -r)"
+
+X86_64_MACOS_JSON=$(curl --fail "X86_64_MACOS_JSON_URL" || exit 1)
+export X86_64_MACOS_TARBALL="$(echo "$X86_64_MACOS_JSON" | jq .tarball -r)"
+export X86_64_MACOS_BYTESIZE="$(echo "$X86_64_MACOS_JSON" | jq .size -r)"
+export X86_64_MACOS_SHASUM="$(echo "$X86_64_MACOS_JSON" | jq .shasum -r)"
+
+X86_64_LINUX_JSON=$(curl --fail "X86_64_LINUX_JSON_URL" || exit 1)
+export X86_64_LINUX_TARBALL="$(echo "$X86_64_LINUX_JSON" | jq .tarball -r)"
+export X86_64_LINUX_BYTESIZE="$(echo "$X86_64_LINUX_JSON" | jq .size -r)"
+export X86_64_LINUX_SHASUM="$(echo "$X86_64_LINUX_JSON" | jq .shasum -r)"
+
+AARCH64_LINUX_JSON=$(curl --fail "AARCH64_LINUX_JSON_URL" || exit 1)
+export AARCH64_LINUX_TARBALL="$(echo "$AARCH64_LINUX_JSON" | jq .tarball -r)"
+export AARCH64_LINUX_BYTESIZE="$(echo "$AARCH64_LINUX_JSON" | jq .size -r)"
+export AARCH64_LINUX_SHASUM="$(echo "$AARCH64_LINUX_JSON" | jq .shasum -r)"
+
+X86_64_FREEBSD_JSON=$(curl --fail "X86_64_FREEBSD_JSON_URL" || exit 1)
+export X86_64_FREEBSD_TARBALL="$(echo "$X86_64_FREEBSD_JSON" | jq .tarball -r)"
+export X86_64_FREEBSD_BYTESIZE="$(echo "$X86_64_FREEBSD_JSON" | jq .size -r)"
+export X86_64_FREEBSD_SHASUM="$(echo "$X86_64_FREEBSD_JSON" | jq .shasum -r)"
+
+git clone https://github.com/ziglang/www.ziglang.org --depth 1
+cd www.ziglang.org
+export MASTER_DATE="$(date +%Y-%m-%d)"
+env
+"../$ZIG" run update-download-page.zig
+
+s3cmd put -P --add-header="cache-control: public, max-age=31536000, immutable" "../$SRC_TARBALL" s3://ziglang.org/builds/
+s3cmd put -P "../$LANGREF" s3://ziglang.org/documentation/master/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+s3cmd put -P www/download/index.html s3://ziglang.org/download/index.html --add-header="Cache-Control: max-age=0, must-revalidate"
+s3cmd put -P www/download/index.json s3://ziglang.org/download/index.json --add-header="Cache-Control: max-age=0, must-revalidate"
