feat: Support specifying multiple download URLs in tool_versions. (#1145)
The interface of `repository_ctx.download` and
`repository_ctx.download_and_extract` supports string lists as well as
strings as the value of the `url` argument. This is the ultimate
destination of the `url` attribute in the `tool_versions` dictionary, so
it makes sense for it to support lists as well.
It is often useful to provide multiple download URLs, e.g. when
vendoring deps through a mirror (to guard against issues like [git
archive checksums
changing](https://github.blog/changelog/2023-01-30-git-archive-checksums-may-change/)
while still keeping the canonical download URL) or in an airgapped
setting (to support internal URLs alongside external URLs). This is also
pretty common around Bazel repository rules that download things, e.g.
[http_archive](https://bazel.build/rules/lib/repo/http#http_archive-urls),
so it can be expected to work with `tool_versions` too.
diff --git a/python/repositories.bzl b/python/repositories.bzl
index f676610..2429d7e 100644
--- a/python/repositories.bzl
+++ b/python/repositories.bzl
@@ -99,12 +99,14 @@
def _python_repository_impl(rctx):
if rctx.attr.distutils and rctx.attr.distutils_content:
fail("Only one of (distutils, distutils_content) should be set.")
+ if bool(rctx.attr.url) == bool(rctx.attr.urls):
+ fail("Exactly one of (url, urls) must be set.")
platform = rctx.attr.platform
python_version = rctx.attr.python_version
python_short_version = python_version.rpartition(".")[0]
release_filename = rctx.attr.release_filename
- url = rctx.attr.url
+ url = rctx.attr.urls or [rctx.attr.url]
if release_filename.endswith(".zst"):
rctx.download(
@@ -428,8 +430,10 @@
doc = "A directory prefix to strip from the extracted files.",
),
"url": attr.string(
- doc = "The URL of the interpreter to download",
- mandatory = True,
+ doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.",
+ ),
+ "urls": attr.string_list(
+ doc = "The URL of the interpreter to download. Exactly one of url and urls must be set.",
),
"zstd_sha256": attr.string(
default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
@@ -506,7 +510,7 @@
if not sha256:
continue
- (release_filename, url, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions)
+ (release_filename, urls, strip_prefix, patches) = get_release_info(platform, python_version, base_url, tool_versions)
# allow passing in a tool version
coverage_tool = None
@@ -536,7 +540,7 @@
platform = platform,
python_version = python_version,
release_filename = release_filename,
- url = url,
+ urls = urls,
distutils = distutils,
distutils_content = distutils_content,
strip_prefix = strip_prefix,
diff --git a/python/versions.bzl b/python/versions.bzl
index 4feeeae..662f89d 100644
--- a/python/versions.bzl
+++ b/python/versions.bzl
@@ -41,6 +41,8 @@
# "strip_prefix": "python",
# },
#
+# It is possible to provide lists in "url".
+#
# buildifier: disable=unsorted-dict-items
TOOL_VERSIONS = {
"3.8.10": {
@@ -281,19 +283,28 @@
if type(url) == type({}):
url = url[platform]
+ if type(url) != type([]):
+ url = [url]
+
strip_prefix = tool_versions[python_version].get("strip_prefix", None)
if type(strip_prefix) == type({}):
strip_prefix = strip_prefix[platform]
- release_filename = url.format(
- platform = platform,
- python_version = python_version,
- build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only",
- )
- if "://" in release_filename: # is absolute url?
- url = release_filename
- else:
- url = "/".join([base_url, release_filename])
+ release_filename = None
+ rendered_urls = []
+ for u in url:
+ release_filename = u.format(
+ platform = platform,
+ python_version = python_version,
+ build = "shared-install_only" if (WINDOWS_NAME in platform) else "install_only",
+ )
+ if "://" in release_filename: # is absolute url?
+ rendered_urls.append(release_filename)
+ else:
+ rendered_urls.append("/".join([base_url, release_filename]))
+
+ if release_filename == None:
+ fail("release_filename should be set by now; were any download URLs given?")
patches = tool_versions[python_version].get("patches", [])
if type(patches) == type({}):
@@ -302,7 +313,7 @@
else:
patches = []
- return (release_filename, url, strip_prefix, patches)
+ return (release_filename, rendered_urls, strip_prefix, patches)
def print_toolchains_checksums(name):
native.genrule(
@@ -333,10 +344,11 @@
"echo \"{python_version}: {platform}: $$(curl --location --fail {release_url_sha256} 2>/dev/null || curl --location --fail {release_url} 2>/dev/null | shasum -a 256 | awk '{{ print $$1 }}')\"".format(
python_version = python_version,
platform = platform,
- release_url = get_release_info(platform, python_version)[1],
- release_url_sha256 = get_release_info(platform, python_version)[1] + ".sha256",
+ release_url = release_url,
+ release_url_sha256 = release_url + ".sha256",
)
for platform in TOOL_VERSIONS[python_version]["sha256"].keys()
+ for release_url in get_release_info(platform, python_version)[1]
])
def gen_python_config_settings(name = ""):
