Setup buildkite

.buildkite/pipeline.yml

@@ -0,0 +1,6 @@
+steps:
+  - label: "Build"
+    command: tools/bazel build //...
+agents:
+    - "queue=init"
+    - "docker=*"

tools/bazel

@@ -0,0 +1,493 @@
+#!/usr/bin/env python3
+"""
+Copyright 2018 Google Inc. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+# Origin: https://github.com/bazelbuild/bazelisk/blob/fc3e3d68c42744dc1c01739f9710cc52f4a8258c/bazelisk.py
+
+import base64
+from contextlib import closing
+import hashlib
+import json
+import netrc
+import os
+import os.path
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+try:
+    from urllib.parse import urlparse
+    from urllib.request import urlopen, Request
+    from urllib.error import HTTPError
+except ImportError:
+    # Python 2.x compatibility hack.
+    # http://python-future.org/compatible_idioms.html?highlight=urllib#urllib-module
+    from urlparse import urlparse
+    from urllib2 import urlopen, Request, HTTPError
+
+    FileNotFoundError = IOError
+
+ONE_HOUR = 1 * 60 * 60
+
+LATEST_PATTERN = re.compile(r"latest(-(?P<offset>\d+))?$")
+
+LAST_GREEN_COMMIT_BASE_PATH = (
+    "https://storage.googleapis.com/bazel-untrusted-builds/last_green_commit/"
+)
+
+LAST_GREEN_COMMIT_PATH_SUFFIXES = {
+    "last_green": "github.com/bazelbuild/bazel.git/bazel-bazel",
+    "last_downstream_green": "downstream_pipeline",
+}
+
+BAZEL_GCS_PATH_PATTERN = (
+    "https://storage.googleapis.com/bazel-builds/artifacts/{platform}/{commit}/bazel"
+)
+
+SUPPORTED_PLATFORMS = {"linux": "ubuntu1404", "windows": "windows", "darwin": "macos"}
+
+TOOLS_BAZEL_PATH = "./tools/bazel"
+
+BAZEL_REAL = "BAZEL_REAL"
+
+BAZEL_UPSTREAM = "bazelbuild"
+
+
+def decide_which_bazel_version_to_use():
+    # Check in this order:
+    # - env var "USE_BAZEL_VERSION" is set to a specific version.
+    # - env var "USE_NIGHTLY_BAZEL" or "USE_BAZEL_NIGHTLY" is set -> latest
+    #   nightly. (TODO)
+    # - env var "USE_CANARY_BAZEL" or "USE_BAZEL_CANARY" is set -> latest
+    #   rc. (TODO)
+    # - the file workspace_root/tools/bazel exists -> that version. (TODO)
+    # - workspace_root/.bazelversion exists -> read contents, that version.
+    # - workspace_root/WORKSPACE contains a version -> that version. (TODO)
+    # - fallback: latest release
+    if "USE_BAZEL_VERSION" in os.environ:
+        return os.environ["USE_BAZEL_VERSION"]
+
+    workspace_root = find_workspace_root()
+    if workspace_root:
+        bazelversion_path = os.path.join(workspace_root, ".bazelversion")
+        if os.path.exists(bazelversion_path):
+            with open(bazelversion_path, "r") as f:
+                return f.read().strip()
+
+    return "latest"
+
+
+def find_workspace_root(root=None):
+    if root is None:
+        root = os.getcwd()
+    if os.path.exists(os.path.join(root, "WORKSPACE")):
+        return root
+    new_root = os.path.dirname(root)
+    return find_workspace_root(new_root) if new_root != root else None
+
+
+def resolve_version_label_to_number_or_commit(bazelisk_directory, version):
+    """Resolves the given label to a released version of Bazel or a commit.
+
+    Args:
+        bazelisk_directory: string; path to a directory that can store
+            temporary data for Bazelisk.
+        version: string; the version label that should be resolved.
+    Returns:
+        A (string, bool) tuple that consists of two parts:
+        1. the resolved number of a Bazel release (candidate), or the commit
+            of an unreleased Bazel binary,
+        2. An indicator for whether the returned version refers to a commit.
+    """
+    suffix = LAST_GREEN_COMMIT_PATH_SUFFIXES.get(version)
+    if suffix:
+        return get_last_green_commit(suffix), True
+
+    if "latest" in version:
+        match = LATEST_PATTERN.match(version)
+        if not match:
+            raise Exception(
+                'Invalid version "{}". In addition to using a version '
+                'number such as "0.20.0", you can use values such as '
+                '"latest" and "latest-N", with N being a non-negative '
+                "integer.".format(version)
+            )
+
+        history = get_version_history(bazelisk_directory)
+        offset = int(match.group("offset") or "0")
+        return resolve_latest_version(history, offset), False
+
+    return version, False
+
+
+def get_last_green_commit(path_suffix):
+    return read_remote_text_file(LAST_GREEN_COMMIT_BASE_PATH + path_suffix).strip()
+
+
+def get_releases_json(bazelisk_directory):
+    """Returns the most recent versions of Bazel, in descending order."""
+    releases = os.path.join(bazelisk_directory, "releases.json")
+
+    # Use a cached version if it's fresh enough.
+    if os.path.exists(releases):
+        if abs(time.time() - os.path.getmtime(releases)) < ONE_HOUR:
+            with open(releases, "rb") as f:
+                try:
+                    return json.loads(f.read().decode("utf-8"))
+                except ValueError:
+                    print("WARN: Could not parse cached releases.json.")
+                    pass
+
+    with open(releases, "wb") as f:
+        body = read_remote_text_file("https://api.github.com/repos/bazelbuild/bazel/releases")
+        f.write(body.encode("utf-8"))
+        return json.loads(body)
+
+
+def read_remote_text_file(url):
+    with closing(urlopen(url)) as res:
+        body = res.read()
+        try:
+            return body.decode(res.info().get_content_charset("iso-8859-1"))
+        except AttributeError:
+            # Python 2.x compatibility hack
+            return body.decode(res.info().getparam("charset") or "iso-8859-1")
+
+
+def get_version_history(bazelisk_directory):
+    return sorted(
+        (
+            release["tag_name"]
+            for release in get_releases_json(bazelisk_directory)
+            if not release["prerelease"]
+        ),
+        # This only handles versions with numeric components, but that is fine
+        # since prerelease versions have been excluded.
+        key=lambda version: tuple(int(component)
+                                  for component in version.split('.')),
+        reverse=True,
+    )
+
+
+def resolve_latest_version(version_history, offset):
+    if offset >= len(version_history):
+        version = "latest-{}".format(offset) if offset else "latest"
+        raise Exception(
+            'Cannot resolve version "{}": There are only {} Bazel '
+            "releases.".format(version, len(version_history))
+        )
+
+    # This only works since we store the history in descending order.
+    return version_history[offset]
+
+
+def get_operating_system():
+    operating_system = platform.system().lower()
+    if operating_system not in ("linux", "darwin", "windows"):
+        raise Exception(
+            'Unsupported operating system "{}". '
+            "Bazel currently only supports Linux, macOS and Windows.".format(operating_system)
+        )
+    return operating_system
+
+
+def determine_executable_filename_suffix():
+    operating_system = get_operating_system()
+    return ".exe" if operating_system == "windows" else ""
+
+
+def determine_bazel_filename(version):
+    operating_system = get_operating_system()
+    supported_machines = get_supported_machine_archs(version, operating_system)
+    machine = normalized_machine_arch_name()
+    if machine not in supported_machines:
+        raise Exception(
+            'Unsupported machine architecture "{}". Bazel {} only supports {} on {}.'.format(
+                machine, version, ", ".join(supported_machines), operating_system.capitalize()
+            )
+        )
+
+    filename_suffix = determine_executable_filename_suffix()
+    bazel_flavor = "bazel"
+    if os.environ.get("BAZELISK_NOJDK", "0") != "0":
+        bazel_flavor = "bazel_nojdk"
+    return "{}-{}-{}-{}{}".format(bazel_flavor, version, operating_system, machine, filename_suffix)
+
+
+def get_supported_machine_archs(version, operating_system):
+    supported_machines = ["x86_64"]
+    versions = version.split(".")[:2]
+    if len(versions) == 2:
+        # released version
+        major, minor = int(versions[0]), int(versions[1])
+        if (
+            operating_system == "darwin"
+            and (major > 4 or major == 4 and minor >= 1)
+            or operating_system == "linux"
+            and (major > 3 or major == 3 and minor >= 4)
+        ):
+            # Linux arm64 was supported since 3.4.0.
+            # Darwin arm64 was supported since 4.1.0.
+            supported_machines.append("arm64")
+    elif operating_system in ("darwin", "linux"):
+        # This is needed to run bazelisk_test.sh on Linux and Darwin arm64 machines, which are
+        # becoming more and more popular.
+        # It works because all recent commits of Bazel support arm64 on Darwin and Linux.
+        # However, this would add arm64 by mistake if the commit is too old, which should be
+        # a rare scenario.
+        supported_machines.append("arm64")
+    return supported_machines
+
+
+def normalized_machine_arch_name():
+    machine = platform.machine().lower()
+    if machine == "amd64":
+        machine = "x86_64"
+    elif machine == "aarch64":
+        machine = "arm64"
+    return machine
+
+
+def determine_url(version, is_commit, bazel_filename):
+    if is_commit:
+        sys.stderr.write("Using unreleased version at commit {}\n".format(version))
+        # No need to validate the platform thanks to determine_bazel_filename().
+        return BAZEL_GCS_PATH_PATTERN.format(
+            platform=SUPPORTED_PLATFORMS[platform.system().lower()], commit=version
+        )
+
+    # Split version into base version and optional additional identifier.
+    # Example: '0.19.1' -> ('0.19.1', None), '0.20.0rc1' -> ('0.20.0', 'rc1')
+    (version, rc) = re.match(r"(\d*\.\d*(?:\.\d*)?)(rc\d+)?", version).groups()
+
+    if "BAZELISK_BASE_URL" in os.environ:
+        return "{}/{}/{}".format(os.environ["BAZELISK_BASE_URL"], version, bazel_filename)
+    else:
+        return "https://releases.bazel.build/{}/{}/{}".format(
+            version, rc if rc else "release", bazel_filename
+        )
+
+
+def trim_suffix(string, suffix):
+    if string.endswith(suffix):
+        return string[: len(string) - len(suffix)]
+    else:
+        return string
+
+
+def download_bazel_into_directory(version, is_commit, directory):
+    bazel_filename = determine_bazel_filename(version)
+    bazel_url = determine_url(version, is_commit, bazel_filename)
+
+    filename_suffix = determine_executable_filename_suffix()
+    bazel_directory_name = trim_suffix(bazel_filename, filename_suffix)
+    destination_dir = os.path.join(directory, bazel_directory_name, "bin")
+    maybe_makedirs(destination_dir)
+
+    destination_path = os.path.join(destination_dir, "bazel" + filename_suffix)
+    if not os.path.exists(destination_path):
+        download(bazel_url, destination_path)
+        os.chmod(destination_path, 0o755)
+
+    sha256_path = destination_path + ".sha256"
+    expected_hash = ""
+    if not os.path.exists(sha256_path):
+        try:
+            download(bazel_url + ".sha256", sha256_path)
+        except HTTPError as e:
+            if e.code == 404:
+                sys.stderr.write(
+                    "The Bazel mirror does not have a checksum file; skipping checksum verification."
+                )
+                return destination_path
+            raise e
+    with open(sha256_path, "r") as sha_file:
+        expected_hash = sha_file.read().split()[0]
+    sha256_hash = hashlib.sha256()
+    with open(destination_path, "rb") as bazel_file:
+        for byte_block in iter(lambda: bazel_file.read(4096), b""):
+            sha256_hash.update(byte_block)
+    actual_hash = sha256_hash.hexdigest()
+    if actual_hash != expected_hash:
+        os.remove(destination_path)
+        os.remove(sha256_path)
+        print(
+            "The downloaded Bazel binary is corrupted. Expected SHA-256 {}, got {}. Please try again.".format(
+                expected_hash, actual_hash
+            )
+        )
+        # Exiting with a special exit code not used by Bazel, so the calling process may retry based on that.
+        # https://docs.bazel.build/versions/0.21.0/guide.html#what-exit-code-will-i-get
+        sys.exit(22)
+    return destination_path
+
+
+def download(url, destination_path):
+    sys.stderr.write("Downloading {}...\n".format(url))
+    request = Request(url)
+    if "BAZELISK_BASE_URL" in os.environ:
+        parts = urlparse(url)
+        creds = None
+        try:
+            creds = netrc.netrc().hosts.get(parts.netloc)
+        except Exception:
+            pass
+        if creds is not None:
+            auth = base64.b64encode(("%s:%s" % (creds[0], creds[2])).encode("ascii"))
+            request.add_header("Authorization", "Basic %s" % auth.decode("utf-8"))
+    with closing(urlopen(request)) as response, open(destination_path, "wb") as file:
+        shutil.copyfileobj(response, file)
+
+
+def get_bazelisk_directory():
+    bazelisk_home = os.environ.get("BAZELISK_HOME")
+    if bazelisk_home is not None:
+        return bazelisk_home
+
+    operating_system = get_operating_system()
+
+    base_dir = None
+
+    if operating_system == "windows":
+        base_dir = os.environ.get("LocalAppData")
+        if base_dir is None:
+            raise Exception("%LocalAppData% is not defined")
+    elif operating_system == "darwin":
+        base_dir = os.environ.get("HOME")
+        if base_dir is None:
+            raise Exception("$HOME is not defined")
+        base_dir = os.path.join(base_dir, "Library/Caches")
+    elif operating_system == "linux":
+        base_dir = os.environ.get("XDG_CACHE_HOME")
+        if base_dir is None:
+            base_dir = os.environ.get("HOME")
+            if base_dir is None:
+                raise Exception("neither $XDG_CACHE_HOME nor $HOME are defined")
+            base_dir = os.path.join(base_dir, ".cache")
+    else:
+        raise Exception("Unsupported operating system '{}'".format(operating_system))
+
+    return os.path.join(base_dir, "bazelisk")
+
+
+def maybe_makedirs(path):
+    """
+    Creates a directory and its parents if necessary.
+    """
+    try:
+        os.makedirs(path)
+    except OSError as e:
+        if not os.path.isdir(path):
+            raise e
+
+
+def delegate_tools_bazel(bazel_path):
+    """Match Bazel's own delegation behavior in the builds distributed by most
+    package managers: use tools/bazel if it's present, executable, and not this
+    script.
+    """
+    root = find_workspace_root()
+    if root:
+        wrapper = os.path.join(root, TOOLS_BAZEL_PATH)
+        if os.path.exists(wrapper) and os.access(wrapper, os.X_OK):
+            try:
+                if not os.path.samefile(wrapper, __file__):
+                    return wrapper
+            except AttributeError:
+                # Python 2 on Windows does not support os.path.samefile
+                if os.path.abspath(wrapper) != os.path.abspath(__file__):
+                    return wrapper
+    return None
+
+
+def prepend_directory_to_path(env, directory):
+    """
+    Prepend binary directory to PATH
+    """
+    if "PATH" in env:
+        env["PATH"] = directory + os.pathsep + env["PATH"]
+    else:
+        env["PATH"] = directory
+
+
+def make_bazel_cmd(bazel_path, argv):
+    env = os.environ.copy()
+
+    wrapper = delegate_tools_bazel(bazel_path)
+    if wrapper:
+        env[BAZEL_REAL] = bazel_path
+        bazel_path = wrapper
+
+    directory = os.path.dirname(bazel_path)
+    prepend_directory_to_path(env, directory)
+    return {
+        "exec": bazel_path,
+        "args": argv,
+        "env": env,
+    }
+
+
+def execute_bazel(bazel_path, argv):
+    cmd = make_bazel_cmd(bazel_path, argv)
+
+    # We cannot use close_fds on Windows, so disable it there.
+    p = subprocess.Popen([cmd["exec"]] + cmd["args"], close_fds=os.name != "nt", env=cmd["env"])
+    while True:
+        try:
+            return p.wait()
+        except KeyboardInterrupt:
+            # Bazel will also get the signal and terminate.
+            # We should continue waiting until it does so.
+            pass
+
+
+def get_bazel_path():
+    bazelisk_directory = get_bazelisk_directory()
+    maybe_makedirs(bazelisk_directory)
+
+    bazel_version = decide_which_bazel_version_to_use()
+    bazel_version, is_commit = resolve_version_label_to_number_or_commit(
+        bazelisk_directory, bazel_version
+    )
+
+    # TODO: Support other forks just like Go version
+    bazel_directory = os.path.join(bazelisk_directory, "downloads", BAZEL_UPSTREAM)
+    return download_bazel_into_directory(bazel_version, is_commit, bazel_directory)
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+
+    bazel_path = get_bazel_path()
+
+    argv = argv[1:]
+
+    if argv and argv[0] == "--print_env":
+        cmd = make_bazel_cmd(bazel_path, argv)
+        env = cmd["env"]
+        for key in env:
+            print("{}={}".format(key, env[key]))
+        return 0
+
+    return execute_bazel(bazel_path, argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main())