diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000..1399d08 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,6 @@ +steps: + - label: "Build" + command: tools/bazel build //... +agents: + - "queue=init" + - "docker=*" diff --git a/tools/bazel b/tools/bazel new file mode 100755 index 0000000..cddbe2b --- /dev/null +++ b/tools/bazel @@ -0,0 +1,493 @@ +#!/usr/bin/env python3 +""" +Copyright 2018 Google Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +# Origin: https://github.com/bazelbuild/bazelisk/blob/fc3e3d68c42744dc1c01739f9710cc52f4a8258c/bazelisk.py + +import base64 +from contextlib import closing +import hashlib +import json +import netrc +import os +import os.path +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import time + +try: + from urllib.parse import urlparse + from urllib.request import urlopen, Request + from urllib.error import HTTPError +except ImportError: + # Python 2.x compatibility hack. + # http://python-future.org/compatible_idioms.html?highlight=urllib#urllib-module + from urlparse import urlparse + from urllib2 import urlopen, Request, HTTPError + + FileNotFoundError = IOError + +ONE_HOUR = 1 * 60 * 60 + +LATEST_PATTERN = re.compile(r"latest(-(?P\d+))?$") + +LAST_GREEN_COMMIT_BASE_PATH = ( + "https://storage.googleapis.com/bazel-untrusted-builds/last_green_commit/" +) + +LAST_GREEN_COMMIT_PATH_SUFFIXES = { + "last_green": "github.com/bazelbuild/bazel.git/bazel-bazel", + "last_downstream_green": "downstream_pipeline", +} + +BAZEL_GCS_PATH_PATTERN = ( + "https://storage.googleapis.com/bazel-builds/artifacts/{platform}/{commit}/bazel" +) + +SUPPORTED_PLATFORMS = {"linux": "ubuntu1404", "windows": "windows", "darwin": "macos"} + +TOOLS_BAZEL_PATH = "./tools/bazel" + +BAZEL_REAL = "BAZEL_REAL" + +BAZEL_UPSTREAM = "bazelbuild" + + +def decide_which_bazel_version_to_use(): + # Check in this order: + # - env var "USE_BAZEL_VERSION" is set to a specific version. + # - env var "USE_NIGHTLY_BAZEL" or "USE_BAZEL_NIGHTLY" is set -> latest + # nightly. (TODO) + # - env var "USE_CANARY_BAZEL" or "USE_BAZEL_CANARY" is set -> latest + # rc. (TODO) + # - the file workspace_root/tools/bazel exists -> that version. (TODO) + # - workspace_root/.bazelversion exists -> read contents, that version. + # - workspace_root/WORKSPACE contains a version -> that version. (TODO) + # - fallback: latest release + if "USE_BAZEL_VERSION" in os.environ: + return os.environ["USE_BAZEL_VERSION"] + + workspace_root = find_workspace_root() + if workspace_root: + bazelversion_path = os.path.join(workspace_root, ".bazelversion") + if os.path.exists(bazelversion_path): + with open(bazelversion_path, "r") as f: + return f.read().strip() + + return "latest" + + +def find_workspace_root(root=None): + if root is None: + root = os.getcwd() + if os.path.exists(os.path.join(root, "WORKSPACE")): + return root + new_root = os.path.dirname(root) + return find_workspace_root(new_root) if new_root != root else None + + +def resolve_version_label_to_number_or_commit(bazelisk_directory, version): + """Resolves the given label to a released version of Bazel or a commit. + + Args: + bazelisk_directory: string; path to a directory that can store + temporary data for Bazelisk. + version: string; the version label that should be resolved. + Returns: + A (string, bool) tuple that consists of two parts: + 1. the resolved number of a Bazel release (candidate), or the commit + of an unreleased Bazel binary, + 2. An indicator for whether the returned version refers to a commit. + """ + suffix = LAST_GREEN_COMMIT_PATH_SUFFIXES.get(version) + if suffix: + return get_last_green_commit(suffix), True + + if "latest" in version: + match = LATEST_PATTERN.match(version) + if not match: + raise Exception( + 'Invalid version "{}". In addition to using a version ' + 'number such as "0.20.0", you can use values such as ' + '"latest" and "latest-N", with N being a non-negative ' + "integer.".format(version) + ) + + history = get_version_history(bazelisk_directory) + offset = int(match.group("offset") or "0") + return resolve_latest_version(history, offset), False + + return version, False + + +def get_last_green_commit(path_suffix): + return read_remote_text_file(LAST_GREEN_COMMIT_BASE_PATH + path_suffix).strip() + + +def get_releases_json(bazelisk_directory): + """Returns the most recent versions of Bazel, in descending order.""" + releases = os.path.join(bazelisk_directory, "releases.json") + + # Use a cached version if it's fresh enough. + if os.path.exists(releases): + if abs(time.time() - os.path.getmtime(releases)) < ONE_HOUR: + with open(releases, "rb") as f: + try: + return json.loads(f.read().decode("utf-8")) + except ValueError: + print("WARN: Could not parse cached releases.json.") + pass + + with open(releases, "wb") as f: + body = read_remote_text_file("https://api.github.com/repos/bazelbuild/bazel/releases") + f.write(body.encode("utf-8")) + return json.loads(body) + + +def read_remote_text_file(url): + with closing(urlopen(url)) as res: + body = res.read() + try: + return body.decode(res.info().get_content_charset("iso-8859-1")) + except AttributeError: + # Python 2.x compatibility hack + return body.decode(res.info().getparam("charset") or "iso-8859-1") + + +def get_version_history(bazelisk_directory): + return sorted( + ( + release["tag_name"] + for release in get_releases_json(bazelisk_directory) + if not release["prerelease"] + ), + # This only handles versions with numeric components, but that is fine + # since prerelease versions have been excluded. + key=lambda version: tuple(int(component) + for component in version.split('.')), + reverse=True, + ) + + +def resolve_latest_version(version_history, offset): + if offset >= len(version_history): + version = "latest-{}".format(offset) if offset else "latest" + raise Exception( + 'Cannot resolve version "{}": There are only {} Bazel ' + "releases.".format(version, len(version_history)) + ) + + # This only works since we store the history in descending order. + return version_history[offset] + + +def get_operating_system(): + operating_system = platform.system().lower() + if operating_system not in ("linux", "darwin", "windows"): + raise Exception( + 'Unsupported operating system "{}". ' + "Bazel currently only supports Linux, macOS and Windows.".format(operating_system) + ) + return operating_system + + +def determine_executable_filename_suffix(): + operating_system = get_operating_system() + return ".exe" if operating_system == "windows" else "" + + +def determine_bazel_filename(version): + operating_system = get_operating_system() + supported_machines = get_supported_machine_archs(version, operating_system) + machine = normalized_machine_arch_name() + if machine not in supported_machines: + raise Exception( + 'Unsupported machine architecture "{}". Bazel {} only supports {} on {}.'.format( + machine, version, ", ".join(supported_machines), operating_system.capitalize() + ) + ) + + filename_suffix = determine_executable_filename_suffix() + bazel_flavor = "bazel" + if os.environ.get("BAZELISK_NOJDK", "0") != "0": + bazel_flavor = "bazel_nojdk" + return "{}-{}-{}-{}{}".format(bazel_flavor, version, operating_system, machine, filename_suffix) + + +def get_supported_machine_archs(version, operating_system): + supported_machines = ["x86_64"] + versions = version.split(".")[:2] + if len(versions) == 2: + # released version + major, minor = int(versions[0]), int(versions[1]) + if ( + operating_system == "darwin" + and (major > 4 or major == 4 and minor >= 1) + or operating_system == "linux" + and (major > 3 or major == 3 and minor >= 4) + ): + # Linux arm64 was supported since 3.4.0. + # Darwin arm64 was supported since 4.1.0. + supported_machines.append("arm64") + elif operating_system in ("darwin", "linux"): + # This is needed to run bazelisk_test.sh on Linux and Darwin arm64 machines, which are + # becoming more and more popular. + # It works because all recent commits of Bazel support arm64 on Darwin and Linux. + # However, this would add arm64 by mistake if the commit is too old, which should be + # a rare scenario. + supported_machines.append("arm64") + return supported_machines + + +def normalized_machine_arch_name(): + machine = platform.machine().lower() + if machine == "amd64": + machine = "x86_64" + elif machine == "aarch64": + machine = "arm64" + return machine + + +def determine_url(version, is_commit, bazel_filename): + if is_commit: + sys.stderr.write("Using unreleased version at commit {}\n".format(version)) + # No need to validate the platform thanks to determine_bazel_filename(). + return BAZEL_GCS_PATH_PATTERN.format( + platform=SUPPORTED_PLATFORMS[platform.system().lower()], commit=version + ) + + # Split version into base version and optional additional identifier. + # Example: '0.19.1' -> ('0.19.1', None), '0.20.0rc1' -> ('0.20.0', 'rc1') + (version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups() + + if "BAZELISK_BASE_URL" in os.environ: + return "{}/{}/{}".format(os.environ["BAZELISK_BASE_URL"], version, bazel_filename) + else: + return "https://releases.bazel.build/{}/{}/{}".format( + version, rc if rc else "release", bazel_filename + ) + + +def trim_suffix(string, suffix): + if string.endswith(suffix): + return string[: len(string) - len(suffix)] + else: + return string + + +def download_bazel_into_directory(version, is_commit, directory): + bazel_filename = determine_bazel_filename(version) + bazel_url = determine_url(version, is_commit, bazel_filename) + + filename_suffix = determine_executable_filename_suffix() + bazel_directory_name = trim_suffix(bazel_filename, filename_suffix) + destination_dir = os.path.join(directory, bazel_directory_name, "bin") + maybe_makedirs(destination_dir) + + destination_path = os.path.join(destination_dir, "bazel" + filename_suffix) + if not os.path.exists(destination_path): + download(bazel_url, destination_path) + os.chmod(destination_path, 0o755) + + sha256_path = destination_path + ".sha256" + expected_hash = "" + if not os.path.exists(sha256_path): + try: + download(bazel_url + ".sha256", sha256_path) + except HTTPError as e: + if e.code == 404: + sys.stderr.write( + "The Bazel mirror does not have a checksum file; skipping checksum verification." + ) + return destination_path + raise e + with open(sha256_path, "r") as sha_file: + expected_hash = sha_file.read().split()[0] + sha256_hash = hashlib.sha256() + with open(destination_path, "rb") as bazel_file: + for byte_block in iter(lambda: bazel_file.read(4096), b""): + sha256_hash.update(byte_block) + actual_hash = sha256_hash.hexdigest() + if actual_hash != expected_hash: + os.remove(destination_path) + os.remove(sha256_path) + print( + "The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Please try again.".format( + expected_hash, actual_hash + ) + ) + # Exiting with a special exit code not used by Bazel, so the calling process may retry based on that. + # https://docs.bazel.build/versions/0.21.0/guide.html#what-exit-code-will-i-get + sys.exit(22) + return destination_path + + +def download(url, destination_path): + sys.stderr.write("Downloading {}...\n".format(url)) + request = Request(url) + if "BAZELISK_BASE_URL" in os.environ: + parts = urlparse(url) + creds = None + try: + creds = netrc.netrc().hosts.get(parts.netloc) + except Exception: + pass + if creds is not None: + auth = base64.b64encode(("%s:%s" % (creds[0], creds[2])).encode("ascii")) + request.add_header("Authorization", "Basic %s" % auth.decode("utf-8")) + with closing(urlopen(request)) as response, open(destination_path, "wb") as file: + shutil.copyfileobj(response, file) + + +def get_bazelisk_directory(): + bazelisk_home = os.environ.get("BAZELISK_HOME") + if bazelisk_home is not None: + return bazelisk_home + + operating_system = get_operating_system() + + base_dir = None + + if operating_system == "windows": + base_dir = os.environ.get("LocalAppData") + if base_dir is None: + raise Exception("%LocalAppData% is not defined") + elif operating_system == "darwin": + base_dir = os.environ.get("HOME") + if base_dir is None: + raise Exception("$HOME is not defined") + base_dir = os.path.join(base_dir, "Library/Caches") + elif operating_system == "linux": + base_dir = os.environ.get("XDG_CACHE_HOME") + if base_dir is None: + base_dir = os.environ.get("HOME") + if base_dir is None: + raise Exception("neither $XDG_CACHE_HOME nor $HOME are defined") + base_dir = os.path.join(base_dir, ".cache") + else: + raise Exception("Unsupported operating system '{}'".format(operating_system)) + + return os.path.join(base_dir, "bazelisk") + + +def maybe_makedirs(path): + """ + Creates a directory and its parents if necessary. + """ + try: + os.makedirs(path) + except OSError as e: + if not os.path.isdir(path): + raise e + + +def delegate_tools_bazel(bazel_path): + """Match Bazel's own delegation behavior in the builds distributed by most + package managers: use tools/bazel if it's present, executable, and not this + script. + """ + root = find_workspace_root() + if root: + wrapper = os.path.join(root, TOOLS_BAZEL_PATH) + if os.path.exists(wrapper) and os.access(wrapper, os.X_OK): + try: + if not os.path.samefile(wrapper, __file__): + return wrapper + except AttributeError: + # Python 2 on Windows does not support os.path.samefile + if os.path.abspath(wrapper) != os.path.abspath(__file__): + return wrapper + return None + + +def prepend_directory_to_path(env, directory): + """ + Prepend binary directory to PATH + """ + if "PATH" in env: + env["PATH"] = directory + os.pathsep + env["PATH"] + else: + env["PATH"] = directory + + +def make_bazel_cmd(bazel_path, argv): + env = os.environ.copy() + + wrapper = delegate_tools_bazel(bazel_path) + if wrapper: + env[BAZEL_REAL] = bazel_path + bazel_path = wrapper + + directory = os.path.dirname(bazel_path) + prepend_directory_to_path(env, directory) + return { + "exec": bazel_path, + "args": argv, + "env": env, + } + + +def execute_bazel(bazel_path, argv): + cmd = make_bazel_cmd(bazel_path, argv) + + # We cannot use close_fds on Windows, so disable it there. + p = subprocess.Popen([cmd["exec"]] + cmd["args"], close_fds=os.name != "nt", env=cmd["env"]) + while True: + try: + return p.wait() + except KeyboardInterrupt: + # Bazel will also get the signal and terminate. + # We should continue waiting until it does so. + pass + + +def get_bazel_path(): + bazelisk_directory = get_bazelisk_directory() + maybe_makedirs(bazelisk_directory) + + bazel_version = decide_which_bazel_version_to_use() + bazel_version, is_commit = resolve_version_label_to_number_or_commit( + bazelisk_directory, bazel_version + ) + + # TODO: Support other forks just like Go version + bazel_directory = os.path.join(bazelisk_directory, "downloads", BAZEL_UPSTREAM) + return download_bazel_into_directory(bazel_version, is_commit, bazel_directory) + + +def main(argv=None): + if argv is None: + argv = sys.argv + + bazel_path = get_bazel_path() + + argv = argv[1:] + + if argv and argv[0] == "--print_env": + cmd = make_bazel_cmd(bazel_path, argv) + env = cmd["env"] + for key in env: + print("{}={}".format(key, env[key])) + return 0 + + return execute_bazel(bazel_path, argv) + + +if __name__ == "__main__": + sys.exit(main())