494 lines
17 KiB
Python
Executable File
494 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Copyright 2018 Google Inc. All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
"""
|
|
# Origin: https://github.com/bazelbuild/bazelisk/blob/fc3e3d68c42744dc1c01739f9710cc52f4a8258c/bazelisk.py
|
|
|
|
import base64
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
import netrc
|
|
import os
|
|
import os.path
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
try:
|
|
from urllib.parse import urlparse
|
|
from urllib.request import urlopen, Request
|
|
from urllib.error import HTTPError
|
|
except ImportError:
|
|
# Python 2.x compatibility hack.
|
|
# http://python-future.org/compatible_idioms.html?highlight=urllib#urllib-module
|
|
from urlparse import urlparse
|
|
from urllib2 import urlopen, Request, HTTPError
|
|
|
|
FileNotFoundError = IOError
|
|
|
|
ONE_HOUR = 1 * 60 * 60
|
|
|
|
LATEST_PATTERN = re.compile(r"latest(-(?P<offset>\d+))?$")
|
|
|
|
LAST_GREEN_COMMIT_BASE_PATH = (
|
|
"https://storage.googleapis.com/bazel-untrusted-builds/last_green_commit/"
|
|
)
|
|
|
|
LAST_GREEN_COMMIT_PATH_SUFFIXES = {
|
|
"last_green": "github.com/bazelbuild/bazel.git/bazel-bazel",
|
|
"last_downstream_green": "downstream_pipeline",
|
|
}
|
|
|
|
BAZEL_GCS_PATH_PATTERN = (
|
|
"https://storage.googleapis.com/bazel-builds/artifacts/{platform}/{commit}/bazel"
|
|
)
|
|
|
|
SUPPORTED_PLATFORMS = {"linux": "ubuntu1404", "windows": "windows", "darwin": "macos"}
|
|
|
|
TOOLS_BAZEL_PATH = "./tools/bazel"
|
|
|
|
BAZEL_REAL = "BAZEL_REAL"
|
|
|
|
BAZEL_UPSTREAM = "bazelbuild"
|
|
|
|
|
|
def decide_which_bazel_version_to_use():
|
|
# Check in this order:
|
|
# - env var "USE_BAZEL_VERSION" is set to a specific version.
|
|
# - env var "USE_NIGHTLY_BAZEL" or "USE_BAZEL_NIGHTLY" is set -> latest
|
|
# nightly. (TODO)
|
|
# - env var "USE_CANARY_BAZEL" or "USE_BAZEL_CANARY" is set -> latest
|
|
# rc. (TODO)
|
|
# - the file workspace_root/tools/bazel exists -> that version. (TODO)
|
|
# - workspace_root/.bazelversion exists -> read contents, that version.
|
|
# - workspace_root/WORKSPACE contains a version -> that version. (TODO)
|
|
# - fallback: latest release
|
|
if "USE_BAZEL_VERSION" in os.environ:
|
|
return os.environ["USE_BAZEL_VERSION"]
|
|
|
|
workspace_root = find_workspace_root()
|
|
if workspace_root:
|
|
bazelversion_path = os.path.join(workspace_root, ".bazelversion")
|
|
if os.path.exists(bazelversion_path):
|
|
with open(bazelversion_path, "r") as f:
|
|
return f.read().strip()
|
|
|
|
return "latest"
|
|
|
|
|
|
def find_workspace_root(root=None):
|
|
if root is None:
|
|
root = os.getcwd()
|
|
if os.path.exists(os.path.join(root, "WORKSPACE")):
|
|
return root
|
|
new_root = os.path.dirname(root)
|
|
return find_workspace_root(new_root) if new_root != root else None
|
|
|
|
|
|
def resolve_version_label_to_number_or_commit(bazelisk_directory, version):
|
|
"""Resolves the given label to a released version of Bazel or a commit.
|
|
|
|
Args:
|
|
bazelisk_directory: string; path to a directory that can store
|
|
temporary data for Bazelisk.
|
|
version: string; the version label that should be resolved.
|
|
Returns:
|
|
A (string, bool) tuple that consists of two parts:
|
|
1. the resolved number of a Bazel release (candidate), or the commit
|
|
of an unreleased Bazel binary,
|
|
2. An indicator for whether the returned version refers to a commit.
|
|
"""
|
|
suffix = LAST_GREEN_COMMIT_PATH_SUFFIXES.get(version)
|
|
if suffix:
|
|
return get_last_green_commit(suffix), True
|
|
|
|
if "latest" in version:
|
|
match = LATEST_PATTERN.match(version)
|
|
if not match:
|
|
raise Exception(
|
|
'Invalid version "{}". In addition to using a version '
|
|
'number such as "0.20.0", you can use values such as '
|
|
'"latest" and "latest-N", with N being a non-negative '
|
|
"integer.".format(version)
|
|
)
|
|
|
|
history = get_version_history(bazelisk_directory)
|
|
offset = int(match.group("offset") or "0")
|
|
return resolve_latest_version(history, offset), False
|
|
|
|
return version, False
|
|
|
|
|
|
def get_last_green_commit(path_suffix):
|
|
return read_remote_text_file(LAST_GREEN_COMMIT_BASE_PATH + path_suffix).strip()
|
|
|
|
|
|
def get_releases_json(bazelisk_directory):
|
|
"""Returns the most recent versions of Bazel, in descending order."""
|
|
releases = os.path.join(bazelisk_directory, "releases.json")
|
|
|
|
# Use a cached version if it's fresh enough.
|
|
if os.path.exists(releases):
|
|
if abs(time.time() - os.path.getmtime(releases)) < ONE_HOUR:
|
|
with open(releases, "rb") as f:
|
|
try:
|
|
return json.loads(f.read().decode("utf-8"))
|
|
except ValueError:
|
|
print("WARN: Could not parse cached releases.json.")
|
|
pass
|
|
|
|
with open(releases, "wb") as f:
|
|
body = read_remote_text_file("https://api.github.com/repos/bazelbuild/bazel/releases")
|
|
f.write(body.encode("utf-8"))
|
|
return json.loads(body)
|
|
|
|
|
|
def read_remote_text_file(url):
|
|
with closing(urlopen(url)) as res:
|
|
body = res.read()
|
|
try:
|
|
return body.decode(res.info().get_content_charset("iso-8859-1"))
|
|
except AttributeError:
|
|
# Python 2.x compatibility hack
|
|
return body.decode(res.info().getparam("charset") or "iso-8859-1")
|
|
|
|
|
|
def get_version_history(bazelisk_directory):
|
|
return sorted(
|
|
(
|
|
release["tag_name"]
|
|
for release in get_releases_json(bazelisk_directory)
|
|
if not release["prerelease"]
|
|
),
|
|
# This only handles versions with numeric components, but that is fine
|
|
# since prerelease versions have been excluded.
|
|
key=lambda version: tuple(int(component)
|
|
for component in version.split('.')),
|
|
reverse=True,
|
|
)
|
|
|
|
|
|
def resolve_latest_version(version_history, offset):
|
|
if offset >= len(version_history):
|
|
version = "latest-{}".format(offset) if offset else "latest"
|
|
raise Exception(
|
|
'Cannot resolve version "{}": There are only {} Bazel '
|
|
"releases.".format(version, len(version_history))
|
|
)
|
|
|
|
# This only works since we store the history in descending order.
|
|
return version_history[offset]
|
|
|
|
|
|
def get_operating_system():
|
|
operating_system = platform.system().lower()
|
|
if operating_system not in ("linux", "darwin", "windows"):
|
|
raise Exception(
|
|
'Unsupported operating system "{}". '
|
|
"Bazel currently only supports Linux, macOS and Windows.".format(operating_system)
|
|
)
|
|
return operating_system
|
|
|
|
|
|
def determine_executable_filename_suffix():
|
|
operating_system = get_operating_system()
|
|
return ".exe" if operating_system == "windows" else ""
|
|
|
|
|
|
def determine_bazel_filename(version):
|
|
operating_system = get_operating_system()
|
|
supported_machines = get_supported_machine_archs(version, operating_system)
|
|
machine = normalized_machine_arch_name()
|
|
if machine not in supported_machines:
|
|
raise Exception(
|
|
'Unsupported machine architecture "{}". Bazel {} only supports {} on {}.'.format(
|
|
machine, version, ", ".join(supported_machines), operating_system.capitalize()
|
|
)
|
|
)
|
|
|
|
filename_suffix = determine_executable_filename_suffix()
|
|
bazel_flavor = "bazel"
|
|
if os.environ.get("BAZELISK_NOJDK", "0") != "0":
|
|
bazel_flavor = "bazel_nojdk"
|
|
return "{}-{}-{}-{}{}".format(bazel_flavor, version, operating_system, machine, filename_suffix)
|
|
|
|
|
|
def get_supported_machine_archs(version, operating_system):
|
|
supported_machines = ["x86_64"]
|
|
versions = version.split(".")[:2]
|
|
if len(versions) == 2:
|
|
# released version
|
|
major, minor = int(versions[0]), int(versions[1])
|
|
if (
|
|
operating_system == "darwin"
|
|
and (major > 4 or major == 4 and minor >= 1)
|
|
or operating_system == "linux"
|
|
and (major > 3 or major == 3 and minor >= 4)
|
|
):
|
|
# Linux arm64 was supported since 3.4.0.
|
|
# Darwin arm64 was supported since 4.1.0.
|
|
supported_machines.append("arm64")
|
|
elif operating_system in ("darwin", "linux"):
|
|
# This is needed to run bazelisk_test.sh on Linux and Darwin arm64 machines, which are
|
|
# becoming more and more popular.
|
|
# It works because all recent commits of Bazel support arm64 on Darwin and Linux.
|
|
# However, this would add arm64 by mistake if the commit is too old, which should be
|
|
# a rare scenario.
|
|
supported_machines.append("arm64")
|
|
return supported_machines
|
|
|
|
|
|
def normalized_machine_arch_name():
|
|
machine = platform.machine().lower()
|
|
if machine == "amd64":
|
|
machine = "x86_64"
|
|
elif machine == "aarch64":
|
|
machine = "arm64"
|
|
return machine
|
|
|
|
|
|
def determine_url(version, is_commit, bazel_filename):
|
|
if is_commit:
|
|
sys.stderr.write("Using unreleased version at commit {}\n".format(version))
|
|
# No need to validate the platform thanks to determine_bazel_filename().
|
|
return BAZEL_GCS_PATH_PATTERN.format(
|
|
platform=SUPPORTED_PLATFORMS[platform.system().lower()], commit=version
|
|
)
|
|
|
|
# Split version into base version and optional additional identifier.
|
|
# Example: '0.19.1' -> ('0.19.1', None), '0.20.0rc1' -> ('0.20.0', 'rc1')
|
|
(version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups()
|
|
|
|
if "BAZELISK_BASE_URL" in os.environ:
|
|
return "{}/{}/{}".format(os.environ["BAZELISK_BASE_URL"], version, bazel_filename)
|
|
else:
|
|
return "https://releases.bazel.build/{}/{}/{}".format(
|
|
version, rc if rc else "release", bazel_filename
|
|
)
|
|
|
|
|
|
def trim_suffix(string, suffix):
|
|
if string.endswith(suffix):
|
|
return string[: len(string) - len(suffix)]
|
|
else:
|
|
return string
|
|
|
|
|
|
def download_bazel_into_directory(version, is_commit, directory):
|
|
bazel_filename = determine_bazel_filename(version)
|
|
bazel_url = determine_url(version, is_commit, bazel_filename)
|
|
|
|
filename_suffix = determine_executable_filename_suffix()
|
|
bazel_directory_name = trim_suffix(bazel_filename, filename_suffix)
|
|
destination_dir = os.path.join(directory, bazel_directory_name, "bin")
|
|
maybe_makedirs(destination_dir)
|
|
|
|
destination_path = os.path.join(destination_dir, "bazel" + filename_suffix)
|
|
if not os.path.exists(destination_path):
|
|
download(bazel_url, destination_path)
|
|
os.chmod(destination_path, 0o755)
|
|
|
|
sha256_path = destination_path + ".sha256"
|
|
expected_hash = ""
|
|
if not os.path.exists(sha256_path):
|
|
try:
|
|
download(bazel_url + ".sha256", sha256_path)
|
|
except HTTPError as e:
|
|
if e.code == 404:
|
|
sys.stderr.write(
|
|
"The Bazel mirror does not have a checksum file; skipping checksum verification."
|
|
)
|
|
return destination_path
|
|
raise e
|
|
with open(sha256_path, "r") as sha_file:
|
|
expected_hash = sha_file.read().split()[0]
|
|
sha256_hash = hashlib.sha256()
|
|
with open(destination_path, "rb") as bazel_file:
|
|
for byte_block in iter(lambda: bazel_file.read(4096), b""):
|
|
sha256_hash.update(byte_block)
|
|
actual_hash = sha256_hash.hexdigest()
|
|
if actual_hash != expected_hash:
|
|
os.remove(destination_path)
|
|
os.remove(sha256_path)
|
|
print(
|
|
"The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Please try again.".format(
|
|
expected_hash, actual_hash
|
|
)
|
|
)
|
|
# Exiting with a special exit code not used by Bazel, so the calling process may retry based on that.
|
|
# https://docs.bazel.build/versions/0.21.0/guide.html#what-exit-code-will-i-get
|
|
sys.exit(22)
|
|
return destination_path
|
|
|
|
|
|
def download(url, destination_path):
|
|
sys.stderr.write("Downloading {}...\n".format(url))
|
|
request = Request(url)
|
|
if "BAZELISK_BASE_URL" in os.environ:
|
|
parts = urlparse(url)
|
|
creds = None
|
|
try:
|
|
creds = netrc.netrc().hosts.get(parts.netloc)
|
|
except Exception:
|
|
pass
|
|
if creds is not None:
|
|
auth = base64.b64encode(("%s:%s" % (creds[0], creds[2])).encode("ascii"))
|
|
request.add_header("Authorization", "Basic %s" % auth.decode("utf-8"))
|
|
with closing(urlopen(request)) as response, open(destination_path, "wb") as file:
|
|
shutil.copyfileobj(response, file)
|
|
|
|
|
|
def get_bazelisk_directory():
|
|
bazelisk_home = os.environ.get("BAZELISK_HOME")
|
|
if bazelisk_home is not None:
|
|
return bazelisk_home
|
|
|
|
operating_system = get_operating_system()
|
|
|
|
base_dir = None
|
|
|
|
if operating_system == "windows":
|
|
base_dir = os.environ.get("LocalAppData")
|
|
if base_dir is None:
|
|
raise Exception("%LocalAppData% is not defined")
|
|
elif operating_system == "darwin":
|
|
base_dir = os.environ.get("HOME")
|
|
if base_dir is None:
|
|
raise Exception("$HOME is not defined")
|
|
base_dir = os.path.join(base_dir, "Library/Caches")
|
|
elif operating_system == "linux":
|
|
base_dir = os.environ.get("XDG_CACHE_HOME")
|
|
if base_dir is None:
|
|
base_dir = os.environ.get("HOME")
|
|
if base_dir is None:
|
|
raise Exception("neither $XDG_CACHE_HOME nor $HOME are defined")
|
|
base_dir = os.path.join(base_dir, ".cache")
|
|
else:
|
|
raise Exception("Unsupported operating system '{}'".format(operating_system))
|
|
|
|
return os.path.join(base_dir, "bazelisk")
|
|
|
|
|
|
def maybe_makedirs(path):
|
|
"""
|
|
Creates a directory and its parents if necessary.
|
|
"""
|
|
try:
|
|
os.makedirs(path)
|
|
except OSError as e:
|
|
if not os.path.isdir(path):
|
|
raise e
|
|
|
|
|
|
def delegate_tools_bazel(bazel_path):
|
|
"""Match Bazel's own delegation behavior in the builds distributed by most
|
|
package managers: use tools/bazel if it's present, executable, and not this
|
|
script.
|
|
"""
|
|
root = find_workspace_root()
|
|
if root:
|
|
wrapper = os.path.join(root, TOOLS_BAZEL_PATH)
|
|
if os.path.exists(wrapper) and os.access(wrapper, os.X_OK):
|
|
try:
|
|
if not os.path.samefile(wrapper, __file__):
|
|
return wrapper
|
|
except AttributeError:
|
|
# Python 2 on Windows does not support os.path.samefile
|
|
if os.path.abspath(wrapper) != os.path.abspath(__file__):
|
|
return wrapper
|
|
return None
|
|
|
|
|
|
def prepend_directory_to_path(env, directory):
|
|
"""
|
|
Prepend binary directory to PATH
|
|
"""
|
|
if "PATH" in env:
|
|
env["PATH"] = directory + os.pathsep + env["PATH"]
|
|
else:
|
|
env["PATH"] = directory
|
|
|
|
|
|
def make_bazel_cmd(bazel_path, argv):
|
|
env = os.environ.copy()
|
|
|
|
wrapper = delegate_tools_bazel(bazel_path)
|
|
if wrapper:
|
|
env[BAZEL_REAL] = bazel_path
|
|
bazel_path = wrapper
|
|
|
|
directory = os.path.dirname(bazel_path)
|
|
prepend_directory_to_path(env, directory)
|
|
return {
|
|
"exec": bazel_path,
|
|
"args": argv,
|
|
"env": env,
|
|
}
|
|
|
|
|
|
def execute_bazel(bazel_path, argv):
|
|
cmd = make_bazel_cmd(bazel_path, argv)
|
|
|
|
# We cannot use close_fds on Windows, so disable it there.
|
|
p = subprocess.Popen([cmd["exec"]] + cmd["args"], close_fds=os.name != "nt", env=cmd["env"])
|
|
while True:
|
|
try:
|
|
return p.wait()
|
|
except KeyboardInterrupt:
|
|
# Bazel will also get the signal and terminate.
|
|
# We should continue waiting until it does so.
|
|
pass
|
|
|
|
|
|
def get_bazel_path():
|
|
bazelisk_directory = get_bazelisk_directory()
|
|
maybe_makedirs(bazelisk_directory)
|
|
|
|
bazel_version = decide_which_bazel_version_to_use()
|
|
bazel_version, is_commit = resolve_version_label_to_number_or_commit(
|
|
bazelisk_directory, bazel_version
|
|
)
|
|
|
|
# TODO: Support other forks just like Go version
|
|
bazel_directory = os.path.join(bazelisk_directory, "downloads", BAZEL_UPSTREAM)
|
|
return download_bazel_into_directory(bazel_version, is_commit, bazel_directory)
|
|
|
|
|
|
def main(argv=None):
|
|
if argv is None:
|
|
argv = sys.argv
|
|
|
|
bazel_path = get_bazel_path()
|
|
|
|
argv = argv[1:]
|
|
|
|
if argv and argv[0] == "--print_env":
|
|
cmd = make_bazel_cmd(bazel_path, argv)
|
|
env = cmd["env"]
|
|
for key in env:
|
|
print("{}={}".format(key, env[key]))
|
|
return 0
|
|
|
|
return execute_bazel(bazel_path, argv)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|