wip
Signed-off-by: Thulio Ferraz Assis <3149049+f0rmiga@users.noreply.github.com>
diff --git a/examples/pip_parse/.bazelversion b/examples/pip_parse/.bazelversion
new file mode 100644
index 0000000..af8c8ec
--- /dev/null
+++ b/examples/pip_parse/.bazelversion
@@ -0,0 +1 @@
+4.2.2
diff --git a/examples/pip_parse/BUILD b/examples/pip_parse/BUILD
index 92b59ae..4ae065a 100644
--- a/examples/pip_parse/BUILD
+++ b/examples/pip_parse/BUILD
@@ -4,6 +4,7 @@
"dist_info_requirement",
"entry_point",
)
+load("@rules_python//python/pip_install:pip_repository.bzl", "wheel")
load("@rules_python//python:defs.bzl", "py_binary", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
@@ -79,3 +80,56 @@
},
deps = ["@rules_python//python/runfiles"],
)
+
+wheel(
+ name = "numpy",
+ src = "@numpy_wheel_source_distribution//file",
+)
+
+# wheel(
+# name = "pybind11",
+# src = "@pybind11_wheel_source_distribution//file",
+# )
+
+# wheel(
+# name = "pythran",
+# src = "@pythran_wheel_source_distribution//file",
+# )
+
+# wheel(
+# name = "gast",
+# src = "@gast_wheel_source_distribution//file",
+# )
+
+# wheel(
+# name = "beniget",
+# src = "@beniget_wheel_source_distribution//file",
+# )
+
+# wheel(
+# name = "ply",
+# src = "@ply_wheel_source_distribution//file",
+# )
+
+# wheel(
+# name = "scipy",
+# src = "@scipy_wheel_source_distribution//file",
+# deps = [
+# ":numpy",
+# ":pybind11",
+# ":pythran",
+# ":gast",
+# ":beniget",
+# ":ply",
+# ],
+# )
+
+wheel(
+ name = "boto3",
+ src = "@boto3_wheel_source_distribution//file",
+)
+
+wheel(
+ name = "django",
+ src = "@django_wheel_source_distribution//file",
+)
diff --git a/examples/pip_parse/WORKSPACE b/examples/pip_parse/WORKSPACE
index e96db9f..b706fd6 100644
--- a/examples/pip_parse/WORKSPACE
+++ b/examples/pip_parse/WORKSPACE
@@ -1,18 +1,37 @@
workspace(name = "rules_python_pip_parse_example")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+
+http_archive(
+ name = "bazel_skylib",
+ sha256 = "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728",
+ urls = [
+ "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz",
+ "https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz",
+ ],
+)
+
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
+
+bazel_skylib_workspace()
+
local_repository(
name = "rules_python",
path = "../..",
)
+load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
+
+pip_install_dependencies()
+
load("@rules_python//python:repositories.bzl", "python_register_toolchains")
python_register_toolchains(
- name = "python39",
- python_version = "3.9",
+ name = "python310",
+ python_version = "3.10",
)
-load("@python39//:defs.bzl", "interpreter")
+load("@python310//:defs.bzl", "interpreter")
load("@rules_python//python:pip.bzl", "pip_parse")
pip_parse(
@@ -48,3 +67,66 @@
# Initialize repositories for all packages in requirements_lock.txt.
install_deps()
+
+http_file(
+ name = "numpy_wheel_source_distribution",
+ downloaded_file_path = "numpy-1.22.3.zip",
+ sha256 = "dbc7601a3b7472d559dc7b933b18b4b66f9aa7452c120e87dfb33d02008c8a18",
+ urls = ["https://files.pythonhosted.org/packages/64/4a/b008d1f8a7b9f5206ecf70a53f84e654707e7616a771d84c05151a4713e9/numpy-1.22.3.zip"],
+)
+
+http_file(
+ name = "pybind11_wheel_source_distribution",
+ downloaded_file_path = "pybind11-2.9.2.tar.gz",
+ sha256 = "e5541f8bccf9111d1a94f7897593b55c4cf1a28d5e8cfc8225a855651f011071",
+ urls = ["https://files.pythonhosted.org/packages/cf/6a/a7f2c40fdb43fcf59bc1eebb0a4c4206a99ccddee6391a1168fa6efebce9/pybind11-2.9.2.tar.gz"],
+)
+
+http_file(
+ name = "pythran_wheel_source_distribution",
+ downloaded_file_path = "pythran-0.11.0.tar.gz",
+ sha256 = "0b2cba712e09f7630879dff69f268460bfe34a6d6000451b47d598558a92a875",
+ urls = ["https://files.pythonhosted.org/packages/88/9f/161f08131abf7f23920cee29b691de27f10fd97ac09fb2f3532b3a7f9b96/pythran-0.11.0.tar.gz"],
+)
+
+http_file(
+ name = "gast_wheel_source_distribution",
+ downloaded_file_path = "gast-0.5.3.tar.gz",
+ sha256 = "cfbea25820e653af9c7d1807f659ce0a0a9c64f2439421a7bba4f0983f532dea",
+ urls = ["https://files.pythonhosted.org/packages/48/a3/0bd844c54ae8141642088b7ae09dd38fec2ec7faa9b7d25bb6a23c1f266f/gast-0.5.3.tar.gz"],
+)
+
+http_file(
+ name = "beniget_wheel_source_distribution",
+ downloaded_file_path = "beniget-0.4.1.tar.gz",
+ sha256 = "75554b3b8ad0553ce2f607627dad3d95c60c441189875b98e097528f8e23ac0c",
+ urls = ["https://files.pythonhosted.org/packages/14/e7/50cbac38f77eca8efd39516be6651fdb9f3c4c0fab8cf2cf05f612578737/beniget-0.4.1.tar.gz"],
+)
+
+http_file(
+ name = "ply_wheel_source_distribution",
+ downloaded_file_path = "ply-3.11.tar.gz",
+ sha256 = "00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3",
+ urls = ["https://files.pythonhosted.org/packages/e5/69/882ee5c9d017149285cab114ebeab373308ef0f874fcdac9beb90e0ac4da/ply-3.11.tar.gz"],
+)
+
+http_file(
+ name = "scipy_wheel_source_distribution",
+ downloaded_file_path = "scipy-1.8.0.tar.gz",
+ sha256 = "31d4f2d6b724bc9a98e527b5849b8a7e589bf1ea630c33aa563eda912c9ff0bd",
+ urls = ["https://files.pythonhosted.org/packages/b4/a2/4faa34bf0cdbefd5c706625f1234987795f368eb4e97bde9d6f46860843e/scipy-1.8.0.tar.gz"],
+)
+
+http_file(
+ name = "boto3_wheel_source_distribution",
+ downloaded_file_path = "boto3-1.21.38.tar.gz",
+ sha256 = "62dde36a57697b40b4693e0ad0d39013f1e187e5a3c52fdb50dbe710633061bb",
+ urls = ["https://files.pythonhosted.org/packages/df/89/6b20ad811c1502f9fbb29369556010ef49e830650ad4d6ac257d40134220/boto3-1.21.38.tar.gz"],
+)
+
+http_file(
+ name = "django_wheel_source_distribution",
+ downloaded_file_path = "Django-4.0.4.tar.gz",
+ sha256 = "4e8177858524417563cc0430f29ea249946d831eacb0068a1455686587df40b5",
+ urls = ["https://files.pythonhosted.org/packages/d2/95/93d1f75da95624bf89e373d079fa72debf77f9b10acc31998cc52a5ff3f9/Django-4.0.4.tar.gz"],
+)
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
index e587f34..d5e2a31 100644
--- a/python/pip_install/pip_repository.bzl
+++ b/python/pip_install/pip_repository.bzl
@@ -1,5 +1,15 @@
""
+load("@bazel_skylib//lib:paths.bzl", "paths")
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+load(
+ "@bazel_tools//tools/build_defs/cc:action_names.bzl",
+ "ASSEMBLE_ACTION_NAME",
+ "CPP_COMPILE_ACTION_NAME",
+ "CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME",
+ "CPP_LINK_STATIC_LIBRARY_ACTION_NAME",
+ "C_COMPILE_ACTION_NAME",
+)
load("//python/pip_install:repositories.bzl", "all_requirements")
load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
@@ -434,3 +444,209 @@
data_exclude_glob = data_exclude_glob,
srcs_exclude_glob = srcs_exclude_glob,
))
+
+def _wheel_impl(ctx):
+ tools = [
+ ("CC", C_COMPILE_ACTION_NAME),
+ ("CXX", CPP_COMPILE_ACTION_NAME),
+ ("AS", ASSEMBLE_ACTION_NAME),
+ ("LD", CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME),
+ ("AR", CPP_LINK_STATIC_LIBRARY_ACTION_NAME),
+ ]
+ flags = [
+ ("LDFLAGS", CPP_LINK_DYNAMIC_LIBRARY_ACTION_NAME, ctx.fragments.cpp.linkopts),
+ ("CFLAGS", C_COMPILE_ACTION_NAME, ctx.fragments.cpp.copts),
+ ("CXXFLAGS", CPP_COMPILE_ACTION_NAME, ctx.fragments.cpp.cxxopts),
+ ]
+ env = _environment_variables(
+ ctx,
+ tools,
+ flags,
+ )
+
+ python_toolchain = ctx.toolchains["@bazel_tools//tools/python:toolchain_type"]
+ runtime = python_toolchain.py3_runtime
+ if not runtime.interpreter:
+ fail("py3_runtime must be set")
+
+ build_deps = ctx.files._build_deps
+ deps = ctx.files.deps
+
+ build_deps_pythonpath = _install_deps(ctx, runtime, env, [], "build_deps", build_deps)
+ deps_pythonpath = _install_deps(ctx, runtime, env, [build_deps_pythonpath], "deps", deps)
+
+ output_name = paths.replace_extension(paths.basename(ctx.file.src.path), ".whl").replace(".tar", "")
+ whl = ctx.actions.declare_file(output_name)
+ progress_message = "Building {}".format(output_name)
+
+ ctx.actions.run_shell(
+ command = """\
+set -o errexit -o nounset -o pipefail
+
+export PYTHONPATH="{pythonpath}"
+
+'{interpreter}' -m pip wheel \
+ --disable-pip-version-check \
+ --no-cache-dir \
+ --no-index \
+ --no-deps \
+ --no-build-isolation \
+ --use-pep517 \
+ --wheel-dir output/ \
+ '{src}'
+
+mv output/*.whl '{whl}'
+""".format(
+ interpreter = runtime.interpreter.path,
+ pythonpath = _construct_pythonpath([build_deps_pythonpath, deps_pythonpath]),
+ src = ctx.file.src.path,
+ whl = whl.path,
+ ),
+ env = env,
+ execution_requirements = {
+ "block-network": "1",
+ },
+ inputs = build_deps + [
+ ctx.file.src,
+ build_deps_pythonpath,
+ deps_pythonpath,
+ ],
+ mnemonic = "BuildWheel",
+ outputs = [whl],
+ progress_message = progress_message,
+ tools = runtime.files,
+ )
+
+ return [DefaultInfo(
+ files = depset([whl]),
+ runfiles = ctx.runfiles([whl]),
+ )]
+
+def _install_deps(ctx, runtime, env, extra_pythonpath, dirname, deps):
+ pythonpath = ctx.actions.declare_directory("{}_{}".format(dirname, ctx.attr.name))
+ commands = [
+ _install_dep_command(runtime.interpreter.path, dep.path, pythonpath.path)
+ for dep in deps
+ ]
+ ctx.actions.run_shell(
+ command = """\
+set -o errexit -o nounset -o pipefail
+
+export PYTHONPATH="{pythonpath}"
+
+{commands}
+""".format(
+ pythonpath = _construct_pythonpath(extra_pythonpath + [pythonpath]),
+ commands = "\n".join(commands),
+ ),
+ env = env,
+ execution_requirements = {
+ "block-network": "1",
+ },
+ inputs = deps + extra_pythonpath,
+ mnemonic = "InstallDeps",
+ outputs = [pythonpath],
+ progress_message = "Installing dependencies",
+ tools = runtime.files,
+ )
+ return pythonpath
+
+def _construct_pythonpath(paths):
+ return ":".join([
+ "$(pwd)/{}".format(p.path)
+ for p in paths
+ ])
+
+def _install_dep_command(interpreter, dep, pythonpath):
+ return """\
+if [[ '{dep}' =~ \\.whl$ ]]; then
+ # '{interpreter}' -m wheel unpack \
+ # --dest-dir '{pythonpath}' \
+ # '{dep}'
+ '{interpreter}' <<'EOF'
+import zipfile
+with zipfile.ZipFile('{dep}', 'r') as zf:
+ zf.extractall('{pythonpath}')
+EOF
+else
+ '{interpreter}' -m pip install \
+ --upgrade \
+ --disable-pip-version-check \
+ --no-cache-dir \
+ --no-index \
+ --no-deps \
+ --no-build-isolation \
+ --use-pep517 \
+ --target '{pythonpath}' \
+ '{dep}'
+fi
+""".format(
+ dep = dep,
+ interpreter = interpreter,
+ pythonpath = pythonpath,
+ )
+
+def _environment_variables(ctx, tools, flags):
+ cc_toolchain = find_cpp_toolchain(ctx)
+ feature_configuration = cc_common.configure_features(
+ ctx = ctx,
+ cc_toolchain = cc_toolchain,
+ )
+ env = {}
+ for (var, action) in tools:
+ env[var] = cc_common.get_tool_for_action(
+ feature_configuration = feature_configuration,
+ action_name = action,
+ )
+ for (var, action, user_compile_flags) in flags:
+ compile_variables = cc_common.create_compile_variables(
+ feature_configuration = feature_configuration,
+ cc_toolchain = cc_toolchain,
+ user_compile_flags = user_compile_flags,
+ )
+ env[var] = " ".join(cc_common.get_memory_inefficient_command_line(
+ feature_configuration = feature_configuration,
+ action_name = action,
+ variables = compile_variables,
+ ))
+
+ # TODO(f0rmiga): some libraries will not respect the env vars above, so the
+ # only way to still get them to compile successfuly is to add a PATH.
+ # E.g. compiling numpy calls `as` hardcoded somewhere. Even setting -B in
+ # the CFLAGS, CXXFLAGS and LDFLAGS won't solve fully.
+ # An alternative is to get binutils hermetic and put in the PATH to be found
+ # first.
+ env["PATH"] = "/usr/bin"
+ return env
+
+wheel = rule(
+ _wheel_impl,
+ attrs = {
+ "src": attr.label(
+ allow_single_file = True,
+ doc = "The wheel source distribution file.",
+ mandatory = True,
+ ),
+ "deps": attr.label_list(
+ allow_files = True,
+ doc = "Wheel distributions to be installed before building src.",
+ mandatory = False,
+ ),
+ "_cc_toolchain": attr.label(
+ default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
+ ),
+ "_build_deps": attr.label_list(
+ allow_files = True,
+ default = [
+ Label("@pypi__setuptools_whl//file"),
+ Label("@pypi__wheel_whl//file"),
+ Label("@pypi__cython//file"),
+ ],
+ ),
+ },
+ fragments = ["cpp"],
+ toolchains = [
+ "@bazel_tools//tools/cpp:toolchain_type",
+ "@bazel_tools//tools/python:toolchain_type",
+ ],
+)
diff --git a/python/pip_install/repositories.bzl b/python/pip_install/repositories.bzl
index 352f663..049f3f1 100644
--- a/python/pip_install/repositories.bzl
+++ b/python/pip_install/repositories.bzl
@@ -1,6 +1,7 @@
""
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+load("@bazel_skylib//lib:paths.bzl", "paths")
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
# Avoid a load from @bazel_skylib repository as users don't necessarily have it installed
@@ -42,6 +43,11 @@
"https://files.pythonhosted.org/packages/27/d6/003e593296a85fd6ed616ed962795b2f87709c3eee2bca4f6d0fe55c6d00/wheel-0.37.1-py2.py3-none-any.whl",
"4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a",
),
+ (
+ "pypi__cython",
+ "https://files.pythonhosted.org/packages/cb/da/54a5d7a7d9afc90036d21f4b58229058270cc14b4c81a86d9b2c77fd072e/Cython-0.29.28.tar.gz",
+ "d6fac2342802c30e51426828fe084ff4deb1b3387367cf98976bb2e64b6f8e45",
+ ),
]
_GENERIC_WHEEL = """\
@@ -60,7 +66,7 @@
"""
# Collate all the repository names so they can be easily consumed
-all_requirements = [name for (name, _, _) in _RULE_DEPS]
+all_requirements = [name for (name, url, _) in _RULE_DEPS if url.endswith(".whl")]
def requirement(pkg):
return "@pypi__" + pkg + "//:lib"
@@ -79,11 +85,30 @@
versions.check("4.0.0")
for (name, url, sha256) in _RULE_DEPS:
- maybe(
- http_archive,
- name,
- url = url,
- sha256 = sha256,
- type = "zip",
- build_file_content = _GENERIC_WHEEL,
- )
+ filename = paths.basename(url)
+
+ if filename.endswith(".whl"):
+ maybe(
+ http_file,
+ name + "_whl",
+ urls = [url],
+ sha256 = sha256,
+ downloaded_file_path = filename,
+ )
+
+ maybe(
+ http_archive,
+ name,
+ urls = [url],
+ sha256 = sha256,
+ type = "zip",
+ build_file_content = _GENERIC_WHEEL,
+ )
+ else:
+ maybe(
+ http_file,
+ name,
+ urls = [url],
+ sha256 = sha256,
+ downloaded_file_path = filename,
+ )
