From 02902cc099d9736d589ac4f0e0ff671d7dfa9557 Mon Sep 17 00:00:00 2001 From: Hadrien Dorio Date: Thu, 17 Feb 2022 04:33:08 +0100 Subject: [PATCH] ci: azure: split build-and-test step (#10853) replace the .bat script by a pwsh script --- CMakeLists.txt | 2 +- ci/azure/pipelines.yml | 156 +++++++++++++++++++++++++++++++++++----- ci/azure/windows_script | 94 ------------------------ 3 files changed, 139 insertions(+), 113 deletions(-) delete mode 100755 ci/azure/windows_script diff --git a/CMakeLists.txt b/CMakeLists.txt index 505de972b9..247626f5e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,7 @@ if("${ZIG_VERSION}" STREQUAL "") endif() endif() endif() -message("Configuring zig version ${ZIG_VERSION}") +message(STATUS "Configuring zig version ${ZIG_VERSION}") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries") diff --git a/ci/azure/pipelines.yml b/ci/azure/pipelines.yml index b7c478178f..66c412f1e4 100644 --- a/ci/azure/pipelines.yml +++ b/ci/azure/pipelines.yml @@ -22,29 +22,149 @@ jobs: name: main displayName: 'Build' - job: BuildWindows + timeoutInMinutes: 360 pool: vmImage: 'windows-2019' - timeoutInMinutes: 360 + variables: + TARGET: 'x86_64-windows-gnu' + ZIG_LLVM_CLANG_LLD_NAME: 'zig+llvm+lld+clang-${{ variables.TARGET }}-0.9.1' + ZIG_LLVM_CLANG_LLD_URL: 'https://ziglang.org/deps/${{ variables.ZIG_LLVM_CLANG_LLD_NAME }}.zip' steps: - - powershell: | - (New-Object Net.WebClient).DownloadFile("https://github.com/msys2/msys2-installer/releases/download/2022-01-28/msys2-base-x86_64-20220128.sfx.exe", "sfx.exe") - .\sfx.exe -y -o\ - displayName: Download/Extract/Install MSYS2 - - script: | - @REM install updated filesystem package first without dependency checking - @REM because of: https://github.com/msys2/MSYS2-packages/issues/2021 - %CD:~0,2%\msys64\usr\bin\bash -lc "pacman --noconfirm -Sydd filesystem" - displayName: Workaround filesystem dash MSYS2 dependency issue - - script: | - %CD:~0,2%\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu" - %CD:~0,2%\msys64\usr\bin\bash -lc "pacman --noconfirm -Syuu" - displayName: Update MSYS2 + - pwsh: | + (New-Object Net.WebClient).DownloadFile("$(ZIG_LLVM_CLANG_LLD_URL)", "${ZIG_LLVM_CLANG_LLD_NAME}.zip") + & 'C:\Program Files\7-Zip\7z.exe' x "${ZIG_LLVM_CLANG_LLD_NAME}.zip" + name: install + displayName: 'Install ZIG/LLVM/CLANG/LLD' + + - pwsh: | + Set-Variable -Name ZIGBUILDDIR -Value "$(Get-Location)\build" + Set-Variable -Name ZIGINSTALLDIR -Value "${ZIGBUILDDIR}\dist" + Set-Variable -Name ZIGPREFIXPATH -Value "$(Get-Location)\$(ZIG_LLVM_CLANG_LLD_NAME)" + + # Make the `zig version` number consistent. + # This will affect the `zig build` command below which uses `git describe`. + git config core.abbrev 9 + git fetch --tags + if ((git rev-parse --is-shallow-repository) -eq "true") { + git fetch --unshallow # `git describe` won't work on a shallow repo + } + + # The dev kit zip file that we have here is old, and may be incompatible with + # the build.zig script of master branch. So we keep an old version of build.zig + # here in the CI directory. + mv build.zig build.zig.master + mv ci/azure/build.zig build.zig + + mkdir $ZIGBUILDDIR + cd $ZIGBUILDDIR + + & "${ZIGPREFIXPATH}/bin/zig.exe" build ` + --prefix "$ZIGINSTALLDIR" ` + --search-prefix "$ZIGPREFIXPATH" ` + -Dstage1 ` + <# stage2 is omitted until we resolve https://github.com/ziglang/zig/issues/6485 #> ` + -Domit-stage2 ` + -Dstatic-llvm ` + -Drelease ` + -Dstrip ` + -Duse-zig-libcxx ` + -Dtarget=$(TARGET) + + cd - + + # Now that we have built an up-to-date zig.exe, we restore the original + # build script from master branch. + rm build.zig + mv build.zig.master build.zig + + name: build + displayName: 'Build' + + - pwsh: | + Set-Variable -Name ZIGINSTALLDIR -Value "$(Get-Location)\build\dist" + + # Sadly, stage2 is omitted from this build to save memory on the CI server. Once self-hosted is + # built with itself and does not gobble as much memory, we can enable these tests. + #& "$ZIGINSTALLDIR\bin\zig.exe" test "..\test\behavior.zig" -fno-stage1 -fLLVM -I "..\test" 2>&1 + + & "$ZIGINSTALLDIR\bin\zig.exe" build test-toolchain -Dskip-non-native -Dskip-stage2-tests 2>&1 + & "$ZIGINSTALLDIR\bin\zig.exe" build test-std -Dskip-non-native 2>&1 + name: test + displayName: 'Test' + + - pwsh: | + Set-Variable -Name ZIGINSTALLDIR -Value "$(Get-Location)\build\dist" + + & "$ZIGINSTALLDIR\bin\zig.exe" build docs + timeoutInMinutes: 60 + name: doc + displayName: 'Documentation' + - task: DownloadSecureFile@1 inputs: - secureFile: s3cfg - - script: ci/azure/windows_script.bat - name: main - displayName: 'Build and test' + name: aws_credentials + secureFile: aws_credentials + + - pwsh: | + Set-Variable -Name ZIGBUILDDIR -Value "$(Get-Location)\build" + $Env:AWS_SHARED_CREDENTIALS_FILE = "$Env:DOWNLOADSECUREFILE_SECUREFILEPATH" + + # Workaround Azure networking issue + # https://github.com/aws/aws-cli/issues/5749 + $Env:AWS_EC2_METADATA_DISABLED = "true" + $Env:AWS_REGION = "us-west-2" + + cd "$ZIGBUILDDIR" + mv ../LICENSE dist/ + mv ../zig-cache/langref.html dist/ + mv dist/bin/zig.exe dist/ + rmdir dist/bin + + # Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig + mv dist/lib/zig dist/lib2 + rmdir dist/lib + mv dist/lib2 dist/lib + + Set-Variable -Name VERSION -Value $(./dist/zig.exe version) + Set-Variable -Name DIRNAME -Value "zig-windows-x86_64-$VERSION" + Set-Variable -Name TARBALL -Value "$DIRNAME.zip" + mv dist "$DIRNAME" + 7z a "$TARBALL" "$DIRNAME" + + aws s3 cp ` + "$TARBALL" ` + s3://ziglang.org/builds/ ` + --acl public-read ` + --cache-control 'public, max-age=31536000, immutable' + + Set-Variable -Name SHASUM -Value (Get-FileHash "$TARBALL" -Algorithm SHA256 | select-object -ExpandProperty Hash) + Set-Variable -Name BYTESIZE -Value (Get-Item "$TARBALL").length + + Set-Variable -Name JSONFILE -Value "windows-${Env:BUILD_SOURCEBRANCHNAME}.json" + echo $null > $JSONFILE + echo ('{"tarball": "' + $TARBALL + '",') >> $JSONFILE + echo ('"shasum": "' + $SHASUM + '",') >> $JSONFILE + echo ('"size": ' + $BYTESIZE + '}' ) >> $JSONFILE + + aws s3 cp ` + "$JSONFILE" ` + s3://ziglang.org/builds/ ` + --acl public-read ` + --cache-control 'max-age=0, must-revalidate' + + aws s3 cp ` + "$JSONFILE" ` + "s3://ziglang.org/builds/x86_64-windows-${VERSION}.json" ` + --acl public-read + + echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" + echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM" + echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE" + + name: upload + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + displayName: 'Upload' + - job: OnMasterSuccess dependsOn: - BuildMacOS diff --git a/ci/azure/windows_script b/ci/azure/windows_script deleted file mode 100755 index c487033fed..0000000000 --- a/ci/azure/windows_script +++ /dev/null @@ -1,94 +0,0 @@ -#!/bin/sh - -set -x -set -e - -pacman -Suy --needed --noconfirm -pacman -S --needed --noconfirm wget p7zip python3-pip git -pip install s3cmd - -ZIGDIR="$(pwd)" -ARCH="x86_64" -TARGET="$ARCH-windows-gnu" -CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.9.1" -PREFIX="$HOME/$CACHE_BASENAME" -ZIG="$PREFIX/bin/zig.exe" - -rm -rf $PREFIX -cd $HOME - -wget -nv "https://ziglang.org/deps/$CACHE_BASENAME.zip" -7z x "$CACHE_BASENAME.zip" - -cd $ZIGDIR - -# Make the `zig version` number consistent. -# This will affect the `zig build` command below. -git config core.abbrev 9 -git fetch --unshallow || true -git fetch --tags - -# The dev kit zip file that we have here is old, and may be incompatible with -# the build.zig script of master branch. So we keep an old version of build.zig -# here in the CI directory. -mv build.zig build.zig.master -mv ci/azure/build.zig build.zig - -# stage2 is omitted until we resolve https://github.com/ziglang/zig/issues/6485 -$ZIG build \ - --prefix dist \ - --search-prefix "$PREFIX" \ - -Dstage1 \ - -Domit-stage2 \ - -Dstatic-llvm \ - -Drelease \ - -Dstrip \ - -Duse-zig-libcxx \ - -Dtarget=$TARGET - -# Now that we have built an up-to-date zig.exe, we restore the original -# build script from master branch. -mv build.zig.master build.zig - -dist/bin/zig.exe build test-toolchain -Dskip-non-native -Dskip-stage2-tests -dist/bin/zig.exe build test-std -Dskip-non-native -dist/bin/zig.exe build docs - -if [ "${BUILD_REASON}" != "PullRequest" ]; then - mv LICENSE dist/ - mv zig-cache/langref.html dist/ - mv dist/bin/zig.exe dist/ - rmdir dist/bin - - # Remove the unnecessary zig dir in $prefix/lib/zig/std/std.zig - mv dist/lib/zig dist/lib2 - rmdir dist/lib - mv dist/lib2 dist/lib - - VERSION=$(dist/zig.exe version) - DIRNAME="zig-windows-x86_64-$VERSION" - TARBALL="$DIRNAME.zip" - mv dist "$DIRNAME" - 7z a "$TARBALL" "$DIRNAME" - - s3cmd -c "$DOWNLOADSECUREFILE_SECUREFILEPATH" put -P --add-header="cache-control: public, max-age=31536000, immutable" "$TARBALL" s3://ziglang.org/builds/ - - SHASUM=$(sha256sum $TARBALL | cut '-d ' -f1) - BYTESIZE=$(wc -c < $TARBALL) - - JSONFILE="windows-$GITBRANCH.json" - touch $JSONFILE - echo "{\"tarball\": \"$TARBALL\"," >>$JSONFILE - echo "\"shasum\": \"$SHASUM\"," >>$JSONFILE - echo "\"size\": \"$BYTESIZE\"}" >>$JSONFILE - - s3cmd -c "$DOWNLOADSECUREFILE_SECUREFILEPATH" put -P --add-header="Cache-Control: max-age=0, must-revalidate" "$JSONFILE" "s3://ziglang.org/builds/$JSONFILE" - s3cmd -c "$DOWNLOADSECUREFILE_SECUREFILEPATH" put -P "$JSONFILE" "s3://ziglang.org/builds/x86_64-windows-$VERSION.json" - - # `set -x` causes these variables to be mangled. - # See https://developercommunity.visualstudio.com/content/problem/375679/pipeline-variable-incorrectly-inserts-single-quote.html - set +x - echo "##vso[task.setvariable variable=tarball;isOutput=true]$TARBALL" - echo "##vso[task.setvariable variable=shasum;isOutput=true]$SHASUM" - echo "##vso[task.setvariable variable=bytesize;isOutput=true]$BYTESIZE" -fi