feat(pypi): add pip.dep to declare abstract pypi dependencies (#3850) Introduce the pip.dep tag class to allow modules to declare abstract PyPI dependencies. These declarations ensure the target structure for a PyPI package is created, but provide a no-op implementation that passes analysis time, but fails at execution time. Along the way, create an agent rule for `*.bzl` files to help guide it in creating better bzl files. (cherry picked from commit 30a05831ff8cda98b7fe62fb48fdbf70a01faf09) Work towards #3867
diff --git a/.agents/rules/bzl.md b/.agents/rules/bzl.md new file mode 100644 index 0000000..10aca72 --- /dev/null +++ b/.agents/rules/bzl.md
@@ -0,0 +1,10 @@ +--- +trigger: glob +description: Starlark / Bazel .bzl file coding style rules +globs: *.bzl +--- + +# Starlark Rules + +* Use triple-quoted strings for multi-line rule doc args. +* Don't use backslash line continuation in rule doc args.
diff --git a/CHANGELOG.md b/CHANGELOG.md index 6506c8d..0e91608 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md
@@ -62,6 +62,10 @@ * Exposed {bzl:obj}`VenvSymlinkEntry` and {bzl:obj}`VenvSymlinkKind` in {bzl:target}`//python:py_info.bzl`. * (pypi) Added `@pypi` repo: a unified hub of `pip.parse` hubs. +* (pypi) Added a `dep` tag class to the `pip` bzlmod extension. This allows +modules to declare abstract PyPI dependencies, ensuring target structures +exist in the unified hub, while allowing other modules to provide the +concrete implementation via `pip.parse`. * (uv) Support for basic `uv.lock` generation via the `lock` rule and basic support for importing the `uv.lock` file itself. Since this may have bugs, please report this by creating new tickets. @@ -71,6 +75,7 @@ + {#v2-1-0} ## [2.1.0] - 2026-06-17
diff --git a/docs/pypi/download.md b/docs/pypi/download.md index 161a753..88c2dd2 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md
@@ -121,6 +121,41 @@ `@pypi//numpy`), and the dependency will automatically resolve to the correct wheel version from the active hub during the build. +### Declaring Abstract Dependencies (pip.dep) + +:::{versionadded} VERSION_NEXT_FEATURE +Declaring abstract PyPI dependencies via `pip.dep` tags. +::: + +Sometimes a shared library target or a ruleset needs to depend on a PyPI +package (e.g., `@pypi//numpy`), but does not want to force a specific package +version or a concrete `requirements.txt` lock file on its consumers. + +Instead of calling `pip.parse()`, the module can declare its dependency using +the `pip.dep` tag: + +```starlark +pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") + +# Declare an abstract dependency on 'numpy' and specify extra targets that +# are expected to be available in the package. +pip.dep( + name = "numpy", + extra_targets = ["extra-alias"], +) +``` + +This ensures that the target structure `@pypi//numpy` (and +`@pypi//numpy:extra-alias`) exists in the unified `@pypi` hub repository, so the +declaring module can compile and analyze successfully without needing any local +requirements file. + +The actual concrete implementation and version of the package must be provided +by a downstream module calling `pip.parse`. + +If a downstream module attempts to build a target that depends on an abstract +dependency, but has not provided a concrete implementation for it via any +`pip.parse` call, the build will fail at execution time. As with any repository rule or extension, if you would like to ensure that `pip_parse` is
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 02c5ad0..094cac7 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl
@@ -424,6 +424,15 @@ pip_attr = pip_attr, ) + # dict[str package, dict[str, None] extra_targets] + declared_deps = {} + for mod in module_ctx.modules: + for dep_attr in mod.tags.dep: + name = normalize_name(dep_attr.name) + targets = declared_deps.setdefault(name, {}) + for target in dep_attr.extra_targets: + targets[target] = None + # Keeps track of all the hub's whl repos across the different versions. # dict[hub, dict[whl, dict[version, str pip]]] # Where hub, whl, and pip are the repo names @@ -448,6 +457,7 @@ return struct( config = config, + declared_deps = declared_deps, default_hub = config.default_hub or renamed_default_hub, exposed_packages = exposed_packages, extra_aliases = extra_aliases, @@ -492,6 +502,15 @@ if hub_name not in extra_aliases[qual_alias]: extra_aliases[qual_alias].append(hub_name) + for norm_pkg, extra_targets in mods.declared_deps.items(): + if norm_pkg not in packages: + packages[norm_pkg] = [] + + for target_name in extra_targets: + qual_alias = "%s:%s" % (norm_pkg, target_name) + if qual_alias not in extra_aliases: + extra_aliases[qual_alias] = [] + unified_hub_repo( name = "pypi", default_hub = mods.default_hub or (hubs[0] if hubs else ""), @@ -1015,6 +1034,31 @@ other tags in this extension.""", ) +_dep_tag = tag_class( + attrs = { + "extra_targets": attr.string_list( + doc = """\ +A list of extra target names in the package that are expected to be available. +See {obj}`pip.parse.extra_hub_aliases`. +""", + default = [], + ), + "name": attr.string( + doc = "The name of a pypi package. Note that the name is normalized.", + mandatory = True, + ), + }, + doc = """\ +Declare an abstract PyPI dependency to ensure its target structure exists in the unified hub. + +This is useful for targets or rules that need to depend on a package (e.g., `@pypi//numpy`) +but do not want to force a specific version or concrete requirements lock file on their +consumers. The concrete version and implementation must be provided by downstreams calling +`pip.parse`. If they are not, the target will still be defined, but it will result in an +execution-phase error when built. +""", +) + pypi = module_extension( environ = ["RULES_PYTHON_PYPI_HUB_RESERVED"], doc = """\ @@ -1065,6 +1109,7 @@ ::: """, ), + "dep": _dep_tag, "override": _override_tag, "parse": tag_class( attrs = _pip_parse_ext_attrs(),
diff --git a/python/private/pypi/missing_package.bzl b/python/private/pypi/missing_package.bzl index c59f754..46823e8 100644 --- a/python/private/pypi/missing_package.bzl +++ b/python/private/pypi/missing_package.bzl
@@ -6,12 +6,19 @@ def _missing_package_error_impl(ctx): out = ctx.actions.declare_file(ctx.label.name + ".error") + if ctx.attr.hub_name: + hub_clause = ' when building under PyPI hub "{hub}". Try adding it to the requirements of this hub (e.g. requirements_lock or requirements_by_platform in pip.parse)'.format( + hub = ctx.attr.hub_name, + ) + else: + hub_clause = ' because no default PyPI hub was configured. Try designating a default hub via pip.default(default_hub = "...") or select a hub using --@rules_python//python/config_settings:venv' + # Register an action that fails when Bazel attempts to stage/build this file ctx.actions.run_shell( outputs = [out], command = "echo 'ERROR: PyPI package \"{pkg}\" is not available{hub_clause}.' >&2 && exit 1".format( pkg = ctx.attr.package_name, - hub_clause = (' when building under PyPI hub "%s"' % ctx.attr.hub_name) if ctx.attr.hub_name else " because no PyPI hub or default hub is requested", + hub_clause = hub_clause, ), )
diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index 8a37d4b..c0b9905 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel
@@ -46,3 +46,16 @@ }, deps = ["@pypi//six"], ) + +py_binary( + name = "bin_declared_only", + srcs = ["bin_declared_only.py"], + deps = ["@pypi//declared_only_pkg"], +) + +py_binary( + name = "bin_declared_only_alias", + srcs = ["bin_declared_only.py"], + main = "bin_declared_only.py", + deps = ["@pypi//declared_only_pkg:declared-only-alias"], +)
diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index 0d0f44f..6a4b87e 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel
@@ -45,4 +45,8 @@ use_repo(pip, "pypi_b") pip.default(default_hub = "pypi_b") +pip.dep( + name = "declared-only-pkg", + extra_targets = ["declared-only-alias"], +) use_repo(pip, "pypi")
diff --git a/tests/integration/unified_pypi/bin_declared_only.py b/tests/integration/unified_pypi/bin_declared_only.py new file mode 100644 index 0000000..0b03607 --- /dev/null +++ b/tests/integration/unified_pypi/bin_declared_only.py
@@ -0,0 +1,2 @@ +# Dummy file for integration test +print("declared_only")
diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index 707ab13..a19a9fc 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py
@@ -30,7 +30,7 @@ ) self.assert_result_matches( result, - 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_a".', + 'ERROR: PyPI package "six" is not available when building under PyPI hub "pypi_a"\\. Try adding it to the requirements of this hub', ) def test_sibling_extra_alias_cquery_succeeds_but_build_fails(self): @@ -43,7 +43,7 @@ ) self.assert_result_matches( result, - 'ERROR: PyPI package "colorama:my_colorama" is not available when building under PyPI hub "pypi_b".', + 'ERROR: PyPI package "colorama:my_colorama" is not available when building under PyPI hub "pypi_b"\\. Try adding it to the requirements of this hub', ) @contextlib.contextmanager @@ -74,6 +74,27 @@ "default_hub 'invalid_hub' is not a defined PyPI hub", ) + def test_unimplemented_declared_dep_fails_build(self): + # Even though cquery succeeds: + self.run_bazel("cquery", "//:bin_declared_only") + + # Build must fail because the package is not implemented by any concrete hub + result = self.run_bazel("build", "//:bin_declared_only", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg" is not available when building under PyPI hub "pypi_b"\\. Try adding it to the requirements of this hub', + ) + + def test_unimplemented_declared_dep_alias_fails_build(self): + # Build must fail for alias too + result = self.run_bazel("build", "//:bin_declared_only_alias", check=False) + self.assertNotEqual(result.exit_code, 0) + self.assert_result_matches( + result, + 'ERROR: PyPI package "declared_only_pkg:declared-only-alias" is not available when building under PyPI hub "pypi_b"\\. Try adding it to the requirements of this hub', + ) + if __name__ == "__main__": unittest.main()
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index bc4c0bc..c2e0c8b 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl
@@ -90,7 +90,13 @@ }.items() ] -def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], is_root = True): +def _dep(*, name, extra_targets = []): + return struct( + name = name, + extra_targets = extra_targets, + ) + +def _mod(*, name, default = _default_tags_default, parse = [], override = [], whl_mods = [], dep = [], is_root = True): return struct( name = name, tags = struct( @@ -98,6 +104,7 @@ override = override, whl_mods = whl_mods, default = default, + dep = dep, ), is_root = is_root, ) @@ -106,6 +113,7 @@ return env.expect.that_struct( parse_modules(**kwargs), attrs = dict( + declared_deps = subjects.dict, default_hub = subjects.str, exposed_packages = subjects.dict, hub_group_map = subjects.dict, @@ -435,6 +443,89 @@ _tests.append(_test_default_hub_precedence) +def _test_extension_dep(env): + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "my_module", + dep = [ + _dep( + name = "declared-pkg", + extra_targets = ["declared-alias"], + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + ), + available_interpreters = {}, + minor_mapping = {}, + ) + + pypi.declared_deps().contains_exactly({"declared_pkg": {"declared-alias": None}}) + pypi.exposed_packages().contains_exactly({}) + pypi.hub_group_map().contains_exactly({}) + pypi.hub_whl_map().contains_exactly({}) + pypi.whl_libraries().contains_exactly({}) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_extension_dep) + +def _test_extension_dep_coexists_with_concrete_hub(env): + pypi = _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + _mod( + name = "my_module", + parse = [ + _parse( + hub_name = "pypi_a", + python_version = "3.15", + simpleapi_skip = ["simple"], + requirements_lock = "requirements.txt", + ), + ], + dep = [ + _dep( + name = "simple", + extra_targets = ["extra-target"], + ), + ], + ), + os_name = "linux", + arch_name = "x86_64", + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + + pypi.declared_deps().contains_exactly({"simple": {"extra-target": None}}) + pypi.exposed_packages().contains_exactly({"pypi_a": ["simple"]}) + pypi.hub_group_map().contains_exactly({"pypi_a": {}}) + pypi.hub_whl_map().contains_exactly({"pypi_a": { + "simple": { + "pypi_a_315_simple": [ + whl_config_setting( + version = "3.15", + ), + ], + }, + }}) + pypi.whl_libraries().contains_exactly({ + "pypi_a_315_simple": { + "config_load": "@pypi_a//:config.bzl", + "dep_template": "@pypi_a//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", + }, + }) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_extension_dep_coexists_with_concrete_hub) + def extension_test_suite(name): """Create the test suite.