bazel-prerelease/fetch.py

#!/usr/bin/env python
# Copyright 2022 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import html.parser
import json
import os
import re
import urllib.request

# NOTE: There are three potential sources for Bazel binaries:
#
# - The latest stable release (e.g. 5.3.2 currently), available from GitHub,
#   and whose version is obtainable from the following API file:
#
#     https://api.github.com/repos/bazelbuild/bazel/releases/latest
#
# - The latest pre-release binary, which are no longer available from GitHub.
#   Since there is no API to get it, the latest version and corresponding
#   download link can be extracted by parsing the HTML content of:
#
#     https://releases.bazel.build/rolling.html
#
# - The release candidates, which are _not_ available from GitHub, but from
#   releases.bazel.build instead, and which do not have an API file, and
#   must be specified with explicit version/candidate name below.
#
# Which version is downloaded depends on the definition of the _BAZEL_VERSION
# configuration variable below, which should be one of the following constants:
_BAZEL_VERSION_LATEST_PRERELEASE = "latest-prerelease"
_BAZEL_VERSION_LATEST_RELEASE = "latest-release"
_BAZEL_VERSION_RELEASE_CANDIDATE = "6.0.0/rc4"  # Must be <version>/<candidate>

# This value is used to auto-detect which version to fetch
# by looking at the name of this script's parent directory. which
# can be either "bazel" or "bazel-prerelease". This allows generating
# CIPD packages for both versions using a simple directory symlink.
_BAZEL_VERSION_AUTO_DETECT = "auto-detect"

# Which Bazel binary to download, see notes above.
_BAZEL_VERSION = _BAZEL_VERSION_AUTO_DETECT

# Consistency check.
assert _BAZEL_VERSION in (
    _BAZEL_VERSION_AUTO_DETECT,
    _BAZEL_VERSION_LATEST_PRERELEASE,
    _BAZEL_VERSION_LATEST_RELEASE,
    _BAZEL_VERSION_RELEASE_CANDIDATE,
)


class BazelPreReleaseHtmlParser(html.parser.HTMLParser):
    """Parser used to extract the latest Bazel pre-release version from ROLLING_URL

    Usage is:
      1) Create instance, e.g. parser = BazelPreReleaseHtmlParser()
      2) Load input HTML from parser.INPUT_URL
      3) Call parser.feed(input_data)
      4) Read parser.version value to retrieve latest pre-release version as a string.
    """

    # Where to find the input data.
    INPUT_URL = "https://releases.bazel.build/rolling.html"

    def __init__(self):
        super().__init__()
        self._version = None

    def handle_starttag(self, tag, attrs):
        """ "Parse input tags to extract the first prelease version.

        Expected format of input is:

        <h1>8.0.0</h1>
        <ul>
        <li><a href="https://releases.bazel.build/8.0.0/rolling/8.0.0-pre.20240128.3/index.html">8.0.0-pre.20240128.3</a> (2024-02-12)</li>
        <li><a href="https://releases.bazel.build/8.0.0/rolling/8.0.0-pre.20240108.7/index.html">8.0.0-pre.20240108.7</a> (2024-02-05)</li>
        ...
        </ul>
        <h1>7.0.0</h1>
        <ul>
        <li><a href="https://releases.bazel.build/7.0.0/rolling/7.0.0-pre.20231018.3/index.html">7.0.0-pre.20231018.3</a> (2023-11-07)</li>
        <li><a href="https://releases.bazel.build/7.0.0/rolling/7.0.0-pre.20231011.2/index.html">7.0.0-pre.20231011.2</a> (2023-10-23)</li>
        <li><a href="https://releases.bazel.build/7.0.0/rolling/7.0.0-pre.20230926.1/index.html">7.0.0-pre.20230926.1</a> (2023-10-06)</li>
        ...
        </ul>
        ...
        """
        if (
            self._version is None
            and tag == "a"
            and len(attrs) == 1
            and attrs[0][0] == "href"
        ):
            href = attrs[0][1]
            components = href.split("/")
            assert (
                len(components) > 3 and components[-1] == "index.html"
            ), f"Unexpected item in {self.INPUT_URL}: tag={tag} attrs={attrs}"
            self._version = components[-2]

    @property
    def version(self):
        return self._version


def auto_detect_bazel_version():
    """Auto-detect correct _BAZEL_VERSION value base on current script directory name."""
    script_dir_name = os.path.basename(os.path.dirname(__file__))
    version = {
        "bazel": _BAZEL_VERSION_LATEST_RELEASE,
        "bazel-prerelease": _BAZEL_VERSION_LATEST_PRERELEASE,
    }.get(script_dir_name)
    assert (
        version is not None
    ), f"Unknown name of parent directory {script_dir_name}, cannot auto-detect Bazel version!"
    return version


def get_bazel_version():
    """Return _BAZEL_VERSION after optional auto-detection."""
    bazel_version = _BAZEL_VERSION
    if bazel_version == _BAZEL_VERSION_AUTO_DETECT:
        bazel_version = auto_detect_bazel_version()
    return bazel_version


def natural_sort_key(item):
    """Convert a version number to a sorting key for natural ordering."""
    # Split the input item into a list of words and numbers (ignore everything else).
    # E.g. "6.0-pre2022:1" -> ("6", "0", "pre", "2022", "1")
    tups = re.findall("[^\W\d_]|\d+", item)
    # Convert all numbers to their integer representation in the output list, but
    # keep words as is. For example (6, 0, "pre", 2022, 1)
    # Python will naturally sort tuple items in ascending order.
    return [int(tup) if tup.isdigit() else tup for tup in tups]


def get_release_candidate_version(bazel_version):
    """Return (version, candidate) information from formatted version string."""
    version, _, candidate = bazel_version.partition("/")
    assert candidate, "Missing release candidate /<suffix> in: " + bazel_version
    return version, candidate


def do_latest():
    """Retrieve tag of the latest pre-release to fetch."""
    bazel_version = get_bazel_version()

    if bazel_version == _BAZEL_VERSION_RELEASE_CANDIDATE:
        # A release candidate is only available from releases.bazel.build
        # and there is no API file provided to list them, which forces the use
        # of hard-coded version + candidate
        version, candidate = get_release_candidate_version(_BAZEL_VERSION)
        print(f"{version}{candidate}")
    elif bazel_version == _BAZEL_VERSION_LATEST_PRERELEASE:
        # Prebuilt pre-release binaries are not longer available from GitHub.
        # Extract the latest prerelease version from rolling HTML page.
        prerelease_parser = BazelPreReleaseHtmlParser()
        with urllib.request.urlopen(prerelease_parser.INPUT_URL) as input:
            prerelease_parser.feed(input.read().decode("utf-8"))
        print(prerelease_parser.version)
    elif bazel_version == _BAZEL_VERSION_LATEST_RELEASE:
        # This codes get the latest official release tag directly instead.
        print(
            json.load(
                urllib.request.urlopen(
                    "https://api.github.com/repos/bazelbuild/bazel/releases/latest"
                )
            )["tag_name"]
        )
    else:
        assert (
            False
        ), f"Invalid or unrecognized value for _BAZEL_VERSION: {bazel_version}"


_PLATFORMS = {
    "linux-amd64": "linux-x86_64",
    "linux-arm64": "linux-arm64",
    "mac-amd64": "darwin-x86_64",
    "mac-arm64": "darwin-arm64",
    "windows-amd64": "windows-x86_64",
    "windows-arm64": "windows-arm64",
}

_EXTENSION = {
    "linux": "",
    "mac": "",
    "windows": ".zip",
}


def get_download_url(version, platform):
    if platform not in _PLATFORMS:
        raise ValueError("unsupported platform {}".format(platform))

    extension = _EXTENSION[platform.split("-")[0]]

    bazel_version = _BAZEL_VERSION
    if bazel_version == _BAZEL_VERSION_AUTO_DETECT:
        bazel_version = auto_detect_bazel_version()

    if bazel_version == _BAZEL_VERSION_RELEASE_CANDIDATE:
        version, candidate = get_release_candidate_version(_BAZEL_VERSION)
        url = (
            "https://releases.bazel.build/{version}/{candidate}/"
            "bazel-{version}{candidate}-{platform}{extension}".format(
                version=version,
                candidate=candidate,
                platform=_PLATFORMS[platform],
                extension=extension,
            )
        )
    elif bazel_version == _BAZEL_VERSION_LATEST_PRERELEASE:
        # Example for version == "8.0.0-pre.20240128.3" and platform == "linux-amd64"
        # https://releases.bazel.build/8.0.0/rolling/8.0.0-pre.20240128.3/bazel-8.0.0-pre.20240128.3-linux-x86_64
        pos = version.find("-pre")
        assert pos > 0, f"Unexpected pre-release version: {version}"
        url = (
            "https://releases.bazel.build/{major_version}/rolling/{version}/"
            "bazel-{version}-{platform}{extension}".format(
                major_version=version[:pos],
                version=version,
                platform=_PLATFORMS[platform],
                extension=extension,
            )
        )
    else:
        url = (
            "https://github.com/bazelbuild/bazel/releases/download/{version}/"
            "bazel-{version}-{platform}{extension}".format(
                version=version,
                platform=_PLATFORMS[platform],
                extension=extension,
            )
        )

    manifest = {
        "url": [url],
        "ext": extension,
    }

    print(json.dumps(manifest))


def main():
    ap = argparse.ArgumentParser()
    sub = ap.add_subparsers()

    latest = sub.add_parser("latest")
    latest.set_defaults(func=lambda _opts: do_latest())

    download = sub.add_parser("get_url")
    download.set_defaults(
        func=lambda opts: get_download_url(
            os.environ["_3PP_VERSION"], os.environ["_3PP_PLATFORM"]
        )
    )

    opts = ap.parse_args()
    opts.func(opts)


if __name__ == "__main__":
    main()