Setup buildkite
This commit is contained in:
zplin
Motiejus Jakštys
parent
e08cf8e6bf
commit
a1b1be2303
|
|
@ -0,0 +1,6 @@
|
|||
steps:
|
||||
- label: "Build"
|
||||
command: tools/bazel build //...
|
||||
agents:
|
||||
- "queue=init"
|
||||
- "docker=*"
|
||||
|
|
@ -0,0 +1,493 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Copyright 2018 Google Inc. All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
# Origin: https://github.com/bazelbuild/bazelisk/blob/fc3e3d68c42744dc1c01739f9710cc52f4a8258c/bazelisk.py
|
||||
|
||||
import base64
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
import netrc
|
||||
import os
|
||||
import os.path
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import urlopen, Request
|
||||
from urllib.error import HTTPError
|
||||
except ImportError:
|
||||
# Python 2.x compatibility hack.
|
||||
# http://python-future.org/compatible_idioms.html?highlight=urllib#urllib-module
|
||||
from urlparse import urlparse
|
||||
from urllib2 import urlopen, Request, HTTPError
|
||||
|
||||
FileNotFoundError = IOError
|
||||
|
||||
ONE_HOUR = 1 * 60 * 60
|
||||
|
||||
LATEST_PATTERN = re.compile(r"latest(-(?P<offset>\d+))?$")
|
||||
|
||||
LAST_GREEN_COMMIT_BASE_PATH = (
|
||||
"https://storage.googleapis.com/bazel-untrusted-builds/last_green_commit/"
|
||||
)
|
||||
|
||||
LAST_GREEN_COMMIT_PATH_SUFFIXES = {
|
||||
"last_green": "github.com/bazelbuild/bazel.git/bazel-bazel",
|
||||
"last_downstream_green": "downstream_pipeline",
|
||||
}
|
||||
|
||||
BAZEL_GCS_PATH_PATTERN = (
|
||||
"https://storage.googleapis.com/bazel-builds/artifacts/{platform}/{commit}/bazel"
|
||||
)
|
||||
|
||||
SUPPORTED_PLATFORMS = {"linux": "ubuntu1404", "windows": "windows", "darwin": "macos"}
|
||||
|
||||
TOOLS_BAZEL_PATH = "./tools/bazel"
|
||||
|
||||
BAZEL_REAL = "BAZEL_REAL"
|
||||
|
||||
BAZEL_UPSTREAM = "bazelbuild"
|
||||
|
||||
|
||||
def decide_which_bazel_version_to_use():
|
||||
# Check in this order:
|
||||
# - env var "USE_BAZEL_VERSION" is set to a specific version.
|
||||
# - env var "USE_NIGHTLY_BAZEL" or "USE_BAZEL_NIGHTLY" is set -> latest
|
||||
# nightly. (TODO)
|
||||
# - env var "USE_CANARY_BAZEL" or "USE_BAZEL_CANARY" is set -> latest
|
||||
# rc. (TODO)
|
||||
# - the file workspace_root/tools/bazel exists -> that version. (TODO)
|
||||
# - workspace_root/.bazelversion exists -> read contents, that version.
|
||||
# - workspace_root/WORKSPACE contains a version -> that version. (TODO)
|
||||
# - fallback: latest release
|
||||
if "USE_BAZEL_VERSION" in os.environ:
|
||||
return os.environ["USE_BAZEL_VERSION"]
|
||||
|
||||
workspace_root = find_workspace_root()
|
||||
if workspace_root:
|
||||
bazelversion_path = os.path.join(workspace_root, ".bazelversion")
|
||||
if os.path.exists(bazelversion_path):
|
||||
with open(bazelversion_path, "r") as f:
|
||||
return f.read().strip()
|
||||
|
||||
return "latest"
|
||||
|
||||
|
||||
def find_workspace_root(root=None):
|
||||
if root is None:
|
||||
root = os.getcwd()
|
||||
if os.path.exists(os.path.join(root, "WORKSPACE")):
|
||||
return root
|
||||
new_root = os.path.dirname(root)
|
||||
return find_workspace_root(new_root) if new_root != root else None
|
||||
|
||||
|
||||
def resolve_version_label_to_number_or_commit(bazelisk_directory, version):
|
||||
"""Resolves the given label to a released version of Bazel or a commit.
|
||||
|
||||
Args:
|
||||
bazelisk_directory: string; path to a directory that can store
|
||||
temporary data for Bazelisk.
|
||||
version: string; the version label that should be resolved.
|
||||
Returns:
|
||||
A (string, bool) tuple that consists of two parts:
|
||||
1. the resolved number of a Bazel release (candidate), or the commit
|
||||
of an unreleased Bazel binary,
|
||||
2. An indicator for whether the returned version refers to a commit.
|
||||
"""
|
||||
suffix = LAST_GREEN_COMMIT_PATH_SUFFIXES.get(version)
|
||||
if suffix:
|
||||
return get_last_green_commit(suffix), True
|
||||
|
||||
if "latest" in version:
|
||||
match = LATEST_PATTERN.match(version)
|
||||
if not match:
|
||||
raise Exception(
|
||||
'Invalid version "{}". In addition to using a version '
|
||||
'number such as "0.20.0", you can use values such as '
|
||||
'"latest" and "latest-N", with N being a non-negative '
|
||||
"integer.".format(version)
|
||||
)
|
||||
|
||||
history = get_version_history(bazelisk_directory)
|
||||
offset = int(match.group("offset") or "0")
|
||||
return resolve_latest_version(history, offset), False
|
||||
|
||||
return version, False
|
||||
|
||||
|
||||
def get_last_green_commit(path_suffix):
|
||||
return read_remote_text_file(LAST_GREEN_COMMIT_BASE_PATH + path_suffix).strip()
|
||||
|
||||
|
||||
def get_releases_json(bazelisk_directory):
|
||||
"""Returns the most recent versions of Bazel, in descending order."""
|
||||
releases = os.path.join(bazelisk_directory, "releases.json")
|
||||
|
||||
# Use a cached version if it's fresh enough.
|
||||
if os.path.exists(releases):
|
||||
if abs(time.time() - os.path.getmtime(releases)) < ONE_HOUR:
|
||||
with open(releases, "rb") as f:
|
||||
try:
|
||||
return json.loads(f.read().decode("utf-8"))
|
||||
except ValueError:
|
||||
print("WARN: Could not parse cached releases.json.")
|
||||
pass
|
||||
|
||||
with open(releases, "wb") as f:
|
||||
body = read_remote_text_file("https://api.github.com/repos/bazelbuild/bazel/releases")
|
||||
f.write(body.encode("utf-8"))
|
||||
return json.loads(body)
|
||||
|
||||
|
||||
def read_remote_text_file(url):
|
||||
with closing(urlopen(url)) as res:
|
||||
body = res.read()
|
||||
try:
|
||||
return body.decode(res.info().get_content_charset("iso-8859-1"))
|
||||
except AttributeError:
|
||||
# Python 2.x compatibility hack
|
||||
return body.decode(res.info().getparam("charset") or "iso-8859-1")
|
||||
|
||||
|
||||
def get_version_history(bazelisk_directory):
|
||||
return sorted(
|
||||
(
|
||||
release["tag_name"]
|
||||
for release in get_releases_json(bazelisk_directory)
|
||||
if not release["prerelease"]
|
||||
),
|
||||
# This only handles versions with numeric components, but that is fine
|
||||
# since prerelease versions have been excluded.
|
||||
key=lambda version: tuple(int(component)
|
||||
for component in version.split('.')),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def resolve_latest_version(version_history, offset):
|
||||
if offset >= len(version_history):
|
||||
version = "latest-{}".format(offset) if offset else "latest"
|
||||
raise Exception(
|
||||
'Cannot resolve version "{}": There are only {} Bazel '
|
||||
"releases.".format(version, len(version_history))
|
||||
)
|
||||
|
||||
# This only works since we store the history in descending order.
|
||||
return version_history[offset]
|
||||
|
||||
|
||||
def get_operating_system():
|
||||
operating_system = platform.system().lower()
|
||||
if operating_system not in ("linux", "darwin", "windows"):
|
||||
raise Exception(
|
||||
'Unsupported operating system "{}". '
|
||||
"Bazel currently only supports Linux, macOS and Windows.".format(operating_system)
|
||||
)
|
||||
return operating_system
|
||||
|
||||
|
||||
def determine_executable_filename_suffix():
|
||||
operating_system = get_operating_system()
|
||||
return ".exe" if operating_system == "windows" else ""
|
||||
|
||||
|
||||
def determine_bazel_filename(version):
|
||||
operating_system = get_operating_system()
|
||||
supported_machines = get_supported_machine_archs(version, operating_system)
|
||||
machine = normalized_machine_arch_name()
|
||||
if machine not in supported_machines:
|
||||
raise Exception(
|
||||
'Unsupported machine architecture "{}". Bazel {} only supports {} on {}.'.format(
|
||||
machine, version, ", ".join(supported_machines), operating_system.capitalize()
|
||||
)
|
||||
)
|
||||
|
||||
filename_suffix = determine_executable_filename_suffix()
|
||||
bazel_flavor = "bazel"
|
||||
if os.environ.get("BAZELISK_NOJDK", "0") != "0":
|
||||
bazel_flavor = "bazel_nojdk"
|
||||
return "{}-{}-{}-{}{}".format(bazel_flavor, version, operating_system, machine, filename_suffix)
|
||||
|
||||
|
||||
def get_supported_machine_archs(version, operating_system):
|
||||
supported_machines = ["x86_64"]
|
||||
versions = version.split(".")[:2]
|
||||
if len(versions) == 2:
|
||||
# released version
|
||||
major, minor = int(versions[0]), int(versions[1])
|
||||
if (
|
||||
operating_system == "darwin"
|
||||
and (major > 4 or major == 4 and minor >= 1)
|
||||
or operating_system == "linux"
|
||||
and (major > 3 or major == 3 and minor >= 4)
|
||||
):
|
||||
# Linux arm64 was supported since 3.4.0.
|
||||
# Darwin arm64 was supported since 4.1.0.
|
||||
supported_machines.append("arm64")
|
||||
elif operating_system in ("darwin", "linux"):
|
||||
# This is needed to run bazelisk_test.sh on Linux and Darwin arm64 machines, which are
|
||||
# becoming more and more popular.
|
||||
# It works because all recent commits of Bazel support arm64 on Darwin and Linux.
|
||||
# However, this would add arm64 by mistake if the commit is too old, which should be
|
||||
# a rare scenario.
|
||||
supported_machines.append("arm64")
|
||||
return supported_machines
|
||||
|
||||
|
||||
def normalized_machine_arch_name():
|
||||
machine = platform.machine().lower()
|
||||
if machine == "amd64":
|
||||
machine = "x86_64"
|
||||
elif machine == "aarch64":
|
||||
machine = "arm64"
|
||||
return machine
|
||||
|
||||
|
||||
def determine_url(version, is_commit, bazel_filename):
|
||||
if is_commit:
|
||||
sys.stderr.write("Using unreleased version at commit {}\n".format(version))
|
||||
# No need to validate the platform thanks to determine_bazel_filename().
|
||||
return BAZEL_GCS_PATH_PATTERN.format(
|
||||
platform=SUPPORTED_PLATFORMS[platform.system().lower()], commit=version
|
||||
)
|
||||
|
||||
# Split version into base version and optional additional identifier.
|
||||
# Example: '0.19.1' -> ('0.19.1', None), '0.20.0rc1' -> ('0.20.0', 'rc1')
|
||||
(version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups()
|
||||
|
||||
if "BAZELISK_BASE_URL" in os.environ:
|
||||
return "{}/{}/{}".format(os.environ["BAZELISK_BASE_URL"], version, bazel_filename)
|
||||
else:
|
||||
return "https://releases.bazel.build/{}/{}/{}".format(
|
||||
version, rc if rc else "release", bazel_filename
|
||||
)
|
||||
|
||||
|
||||
def trim_suffix(string, suffix):
|
||||
if string.endswith(suffix):
|
||||
return string[: len(string) - len(suffix)]
|
||||
else:
|
||||
return string
|
||||
|
||||
|
||||
def download_bazel_into_directory(version, is_commit, directory):
|
||||
bazel_filename = determine_bazel_filename(version)
|
||||
bazel_url = determine_url(version, is_commit, bazel_filename)
|
||||
|
||||
filename_suffix = determine_executable_filename_suffix()
|
||||
bazel_directory_name = trim_suffix(bazel_filename, filename_suffix)
|
||||
destination_dir = os.path.join(directory, bazel_directory_name, "bin")
|
||||
maybe_makedirs(destination_dir)
|
||||
|
||||
destination_path = os.path.join(destination_dir, "bazel" + filename_suffix)
|
||||
if not os.path.exists(destination_path):
|
||||
download(bazel_url, destination_path)
|
||||
os.chmod(destination_path, 0o755)
|
||||
|
||||
sha256_path = destination_path + ".sha256"
|
||||
expected_hash = ""
|
||||
if not os.path.exists(sha256_path):
|
||||
try:
|
||||
download(bazel_url + ".sha256", sha256_path)
|
||||
except HTTPError as e:
|
||||
if e.code == 404:
|
||||
sys.stderr.write(
|
||||
"The Bazel mirror does not have a checksum file; skipping checksum verification."
|
||||
)
|
||||
return destination_path
|
||||
raise e
|
||||
with open(sha256_path, "r") as sha_file:
|
||||
expected_hash = sha_file.read().split()[0]
|
||||
sha256_hash = hashlib.sha256()
|
||||
with open(destination_path, "rb") as bazel_file:
|
||||
for byte_block in iter(lambda: bazel_file.read(4096), b""):
|
||||
sha256_hash.update(byte_block)
|
||||
actual_hash = sha256_hash.hexdigest()
|
||||
if actual_hash != expected_hash:
|
||||
os.remove(destination_path)
|
||||
os.remove(sha256_path)
|
||||
print(
|
||||
"The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Please try again.".format(
|
||||
expected_hash, actual_hash
|
||||
)
|
||||
)
|
||||
# Exiting with a special exit code not used by Bazel, so the calling process may retry based on that.
|
||||
# https://docs.bazel.build/versions/0.21.0/guide.html#what-exit-code-will-i-get
|
||||
sys.exit(22)
|
||||
return destination_path
|
||||
|
||||
|
||||
def download(url, destination_path):
|
||||
sys.stderr.write("Downloading {}...\n".format(url))
|
||||
request = Request(url)
|
||||
if "BAZELISK_BASE_URL" in os.environ:
|
||||
parts = urlparse(url)
|
||||
creds = None
|
||||
try:
|
||||
creds = netrc.netrc().hosts.get(parts.netloc)
|
||||
except Exception:
|
||||
pass
|
||||
if creds is not None:
|
||||
auth = base64.b64encode(("%s:%s" % (creds[0], creds[2])).encode("ascii"))
|
||||
request.add_header("Authorization", "Basic %s" % auth.decode("utf-8"))
|
||||
with closing(urlopen(request)) as response, open(destination_path, "wb") as file:
|
||||
shutil.copyfileobj(response, file)
|
||||
|
||||
|
||||
def get_bazelisk_directory():
|
||||
bazelisk_home = os.environ.get("BAZELISK_HOME")
|
||||
if bazelisk_home is not None:
|
||||
return bazelisk_home
|
||||
|
||||
operating_system = get_operating_system()
|
||||
|
||||
base_dir = None
|
||||
|
||||
if operating_system == "windows":
|
||||
base_dir = os.environ.get("LocalAppData")
|
||||
if base_dir is None:
|
||||
raise Exception("%LocalAppData% is not defined")
|
||||
elif operating_system == "darwin":
|
||||
base_dir = os.environ.get("HOME")
|
||||
if base_dir is None:
|
||||
raise Exception("$HOME is not defined")
|
||||
base_dir = os.path.join(base_dir, "Library/Caches")
|
||||
elif operating_system == "linux":
|
||||
base_dir = os.environ.get("XDG_CACHE_HOME")
|
||||
if base_dir is None:
|
||||
base_dir = os.environ.get("HOME")
|
||||
if base_dir is None:
|
||||
raise Exception("neither $XDG_CACHE_HOME nor $HOME are defined")
|
||||
base_dir = os.path.join(base_dir, ".cache")
|
||||
else:
|
||||
raise Exception("Unsupported operating system '{}'".format(operating_system))
|
||||
|
||||
return os.path.join(base_dir, "bazelisk")
|
||||
|
||||
|
||||
def maybe_makedirs(path):
|
||||
"""
|
||||
Creates a directory and its parents if necessary.
|
||||
"""
|
||||
try:
|
||||
os.makedirs(path)
|
||||
except OSError as e:
|
||||
if not os.path.isdir(path):
|
||||
raise e
|
||||
|
||||
|
||||
def delegate_tools_bazel(bazel_path):
|
||||
"""Match Bazel's own delegation behavior in the builds distributed by most
|
||||
package managers: use tools/bazel if it's present, executable, and not this
|
||||
script.
|
||||
"""
|
||||
root = find_workspace_root()
|
||||
if root:
|
||||
wrapper = os.path.join(root, TOOLS_BAZEL_PATH)
|
||||
if os.path.exists(wrapper) and os.access(wrapper, os.X_OK):
|
||||
try:
|
||||
if not os.path.samefile(wrapper, __file__):
|
||||
return wrapper
|
||||
except AttributeError:
|
||||
# Python 2 on Windows does not support os.path.samefile
|
||||
if os.path.abspath(wrapper) != os.path.abspath(__file__):
|
||||
return wrapper
|
||||
return None
|
||||
|
||||
|
||||
def prepend_directory_to_path(env, directory):
|
||||
"""
|
||||
Prepend binary directory to PATH
|
||||
"""
|
||||
if "PATH" in env:
|
||||
env["PATH"] = directory + os.pathsep + env["PATH"]
|
||||
else:
|
||||
env["PATH"] = directory
|
||||
|
||||
|
||||
def make_bazel_cmd(bazel_path, argv):
|
||||
env = os.environ.copy()
|
||||
|
||||
wrapper = delegate_tools_bazel(bazel_path)
|
||||
if wrapper:
|
||||
env[BAZEL_REAL] = bazel_path
|
||||
bazel_path = wrapper
|
||||
|
||||
directory = os.path.dirname(bazel_path)
|
||||
prepend_directory_to_path(env, directory)
|
||||
return {
|
||||
"exec": bazel_path,
|
||||
"args": argv,
|
||||
"env": env,
|
||||
}
|
||||
|
||||
|
||||
def execute_bazel(bazel_path, argv):
|
||||
cmd = make_bazel_cmd(bazel_path, argv)
|
||||
|
||||
# We cannot use close_fds on Windows, so disable it there.
|
||||
p = subprocess.Popen([cmd["exec"]] + cmd["args"], close_fds=os.name != "nt", env=cmd["env"])
|
||||
while True:
|
||||
try:
|
||||
return p.wait()
|
||||
except KeyboardInterrupt:
|
||||
# Bazel will also get the signal and terminate.
|
||||
# We should continue waiting until it does so.
|
||||
pass
|
||||
|
||||
|
||||
def get_bazel_path():
|
||||
bazelisk_directory = get_bazelisk_directory()
|
||||
maybe_makedirs(bazelisk_directory)
|
||||
|
||||
bazel_version = decide_which_bazel_version_to_use()
|
||||
bazel_version, is_commit = resolve_version_label_to_number_or_commit(
|
||||
bazelisk_directory, bazel_version
|
||||
)
|
||||
|
||||
# TODO: Support other forks just like Go version
|
||||
bazel_directory = os.path.join(bazelisk_directory, "downloads", BAZEL_UPSTREAM)
|
||||
return download_bazel_into_directory(bazel_version, is_commit, bazel_directory)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
if argv is None:
|
||||
argv = sys.argv
|
||||
|
||||
bazel_path = get_bazel_path()
|
||||
|
||||
argv = argv[1:]
|
||||
|
||||
if argv and argv[0] == "--print_env":
|
||||
cmd = make_bazel_cmd(bazel_path, argv)
|
||||
env = cmd["env"]
|
||||
for key in env:
|
||||
print("{}={}".format(key, env[key]))
|
||||
return 0
|
||||
|
||||
return execute_bazel(bazel_path, argv)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue