Expose package metadata from wheels in .par files (#49)
* Expose package metadata from wheels in .par files
* Update pkg_resources's default distribution metadata object.
`pkg_resources` creates a default WorkingSet as soon as it is
imported, so even though we add hooks for it to find the .dist-info
metadata inside .par files, it is too late. So we manually update the
default WorkingSet, trying not to disturb existing entries.
* Cut down unnecessary test data.
* Fix lint errors
diff --git a/WORKSPACE b/WORKSPACE
index 89cbb3c..09df09f 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -5,6 +5,14 @@
name = "test_workspace",
path = "tests/test_workspace",
)
+local_repository(
+ name = "pypi__portpicker_1_2_0",
+ path = "third_party/pypi__portpicker_1_2_0",
+)
+local_repository(
+ name = "pypi__yapf_0_19_0",
+ path = "third_party/pypi__yapf_0_19_0",
+)
# Not actually referenced anywhere, but must be marked as a separate
# repository so that things like "bazel test //..." don't get confused
diff --git a/runtime/support.py b/runtime/support.py
index d9e957a..0132e39 100644
--- a/runtime/support.py
+++ b/runtime/support.py
@@ -33,8 +33,10 @@
"""
import os
+import pkgutil
import sys
import warnings
+import zipimport
def _log(msg):
@@ -61,6 +63,97 @@
return archive_path
+def _setup_pkg_resources(pkg_resources_name):
+ """Setup hooks into the `pkg_resources` module
+
+ This enables the pkg_resources module to find metadata from wheels
+ that have been included in this .par file.
+
+ The functions and classes here are scoped to this function, since
+ we might have multitple pkg_resources modules, or none.
+ """
+
+ try:
+ __import__(pkg_resources_name)
+ pkg_resources = sys.modules.get(pkg_resources_name)
+ if pkg_resources is None:
+ return
+ except ImportError:
+ # Skip setup
+ return
+
+ class DistInfoMetadata(pkg_resources.EggMetadata):
+ """Metadata provider for zip files containing .dist-info
+
+ In find_dist_info_in_zip(), we call
+ metadata.resource_listdir(directory_name). However, it doesn't
+ work with EggMetadata, because _zipinfo_name() expects the
+ directory name to end with a /, but metadata._listdir() which
+ expects the directory to _not_ end with a /.
+
+ Therefore this class exists.
+ """
+
+ def _listdir(self, fspath):
+ """List of resource names in the directory (like ``os.listdir()``)
+
+ Overrides EggMetadata._listdir()
+ """
+
+ zipinfo_name = self._zipinfo_name(fspath)
+ while zipinfo_name.endswith('/'):
+ zipinfo_name = zipinfo_name[:-1]
+ result = self._index().get(zipinfo_name, ())
+ return list(result)
+
+ def find_dist_info_in_zip(importer, path_item, only=False):
+ """Find dist-info style metadata in zip files.
+
+ We ignore the `only` flag because it's not clear what it should
+ actually do in this case.
+ """
+ metadata = DistInfoMetadata(importer)
+ for subitem in metadata.resource_listdir('/'):
+ if subitem.lower().endswith('.dist-info'):
+ subpath = os.path.join(path_item, subitem)
+ submeta = pkg_resources.EggMetadata(
+ zipimport.zipimporter(subpath))
+ submeta.egg_info = subpath
+ dist = pkg_resources.Distribution.from_location(
+ path_item, subitem, submeta)
+ yield dist
+ return
+
+ def find_eggs_and_dist_info_in_zip(importer, path_item, only=False):
+ """Chain together our finder and the standard pkg_resources finder
+
+ For simplicity, and since pkg_resources doesn't provide a public
+ interface to do so, we hardcode the chaining (find_eggs_in_zip).
+ """
+ # Our finder
+ for dist in find_dist_info_in_zip(importer, path_item, only):
+ yield dist
+ # The standard pkg_resources finder
+ for dist in pkg_resources.find_eggs_in_zip(importer, path_item, only):
+ yield dist
+ return
+
+ # This overwrites the existing registered finder.
+ pkg_resources.register_finder(zipimport.zipimporter,
+ find_eggs_and_dist_info_in_zip)
+
+ # Note that the default WorkingSet has already been created, and
+ # there is no public interface to easily refresh/reload it that
+ # doesn't also have a "Don't use this" warning. So we manually
+ # add just the entries we know about to the existing WorkingSet.
+ for entry in sys.path:
+ importer = pkgutil.get_importer(entry)
+ if isinstance(importer, zipimport.zipimporter):
+ for dist in find_dist_info_in_zip(importer, entry, only=True):
+ pkg_resources.working_set.add(dist, entry, insert=False,
+ replace=False)
+
+
def setup(import_roots=None):
"""Initialize subpar run-time support"""
# Add third-party library entries to sys.path
@@ -75,3 +168,7 @@
new_path = os.path.join(archive_path, import_root)
_log('# adding %s to sys.path' % new_path)
sys.path.insert(1, new_path)
+
+ # Add hook for package metadata
+ _setup_pkg_resources('pkg_resources')
+ _setup_pkg_resources('pip._vendor.pkg_resources')
diff --git a/runtime/support_test.py b/runtime/support_test.py
index ab25bd6..3ec1063 100644
--- a/runtime/support_test.py
+++ b/runtime/support_test.py
@@ -21,7 +21,7 @@
class SupportTest(unittest.TestCase):
- def test_log(self):
+ def test__log(self):
old_stderr = sys.stderr
try:
mock_stderr = io.StringIO()
@@ -36,17 +36,35 @@
finally:
sys.stderr = old_stderr
- def test_find_archive(self):
+ def test__find_archive(self):
# pylint: disable=protected-access
path = support._find_archive()
self.assertNotEqual(path, None)
def test_setup(self):
- support.setup(import_roots=['some_root', 'another_root'])
- self.assertTrue(sys.path[1].endswith('subpar/runtime/some_root'),
- sys.path)
- self.assertTrue(sys.path[2].endswith('subpar/runtime/another_root'),
- sys.path)
+ # `import pip` can cause arbitrary sys.path changes,
+ # especially if using the Debian `python-pip` package or
+ # similar. Get that lunacy out of the way before starting
+ # test
+ try:
+ import pip # noqa
+ except ImportError:
+ pass
+
+ old_sys_path = sys.path
+ try:
+ mock_sys_path = list(sys.path)
+ sys.path = mock_sys_path
+ support.setup(import_roots=['some_root', 'another_root'])
+ finally:
+ sys.path = old_sys_path
+ self.assertTrue(mock_sys_path[1].endswith('subpar/runtime/some_root'),
+ mock_sys_path)
+ self.assertTrue(
+ mock_sys_path[2].endswith('subpar/runtime/another_root'),
+ mock_sys_path)
+ self.assertEqual(mock_sys_path[0], sys.path[0])
+ self.assertEqual(mock_sys_path[3:], sys.path[1:])
if __name__ == '__main__':
diff --git a/tests/BUILD b/tests/BUILD
index 0d35eb7..0d215bc 100644
--- a/tests/BUILD
+++ b/tests/BUILD
@@ -104,6 +104,19 @@
srcs_version = "PY2AND3",
)
+par_binary(
+ name = "package_pkg_resources/main",
+ srcs = [
+ "package_pkg_resources/main.py",
+ ],
+ data = [
+ "@pypi__portpicker_1_2_0//:files",
+ "@pypi__yapf_0_19_0//:files",
+ ],
+ main = "package_pkg_resources/main.py",
+ srcs_version = "PY2AND3",
+)
+
# Test targets
[(
# Run test without .par file as a control
@@ -154,6 +167,7 @@
),
("indirect_dependency", "//tests:package_c/c", "tests/package_c/c"),
("main_boilerplate", "//tests:package_g/g", "tests/package_g/g"),
+ ("pkg_resources", "//tests:package_pkg_resources/main", "tests/package_pkg_resources/main"),
("shadow", "//tests:package_shadow/main", "tests/package_shadow/main"),
("version", "//tests:package_f/f", "tests/package_f/f"),
]]
diff --git a/tests/package_pkg_resources/main.py b/tests/package_pkg_resources/main.py
new file mode 100644
index 0000000..dea801d
--- /dev/null
+++ b/tests/package_pkg_resources/main.py
@@ -0,0 +1,45 @@
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Integration test program for Subpar.
+
+Test that pkg_resources correctly identifies distribution packages
+inside a .par file.
+"""
+
+
+def main():
+ print('In pkg_resources test main()')
+ try:
+ import pkg_resources
+ except ImportError:
+ print('Skipping test, pkg_resources module is not available')
+ return
+
+ ws = pkg_resources.working_set
+
+ # Informational for debugging
+ distributions = list(ws)
+ print('Resources found: %s' % distributions)
+
+ # Check for the packages we provided metadata for. There will
+ # also be metadata for whatever other packages happen to be
+ # installed in the current Python interpreter.
+ for spec in ['portpicker==1.2.0', 'yapf==0.19.0']:
+ dist = ws.find(pkg_resources.Requirement.parse(spec))
+ assert dist, (spec, distributions)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/package_pkg_resources/main_PY2_filelist.txt b/tests/package_pkg_resources/main_PY2_filelist.txt
new file mode 100644
index 0000000..7b93f00
--- /dev/null
+++ b/tests/package_pkg_resources/main_PY2_filelist.txt
@@ -0,0 +1,12 @@
+__main__.py
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
+subpar/__init__.py
+subpar/runtime/__init__.py
+subpar/runtime/support.py
+subpar/tests/__init__.py
+subpar/tests/package_pkg_resources/__init__.py
+subpar/tests/package_pkg_resources/main
+subpar/tests/package_pkg_resources/main.py
diff --git a/tests/package_pkg_resources/main_PY3_filelist.txt b/tests/package_pkg_resources/main_PY3_filelist.txt
new file mode 100644
index 0000000..7b93f00
--- /dev/null
+++ b/tests/package_pkg_resources/main_PY3_filelist.txt
@@ -0,0 +1,12 @@
+__main__.py
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
+subpar/__init__.py
+subpar/runtime/__init__.py
+subpar/runtime/support.py
+subpar/tests/__init__.py
+subpar/tests/package_pkg_resources/__init__.py
+subpar/tests/package_pkg_resources/main
+subpar/tests/package_pkg_resources/main.py
diff --git a/third_party/README.md b/third_party/README.md
new file mode 100644
index 0000000..6cbf2d2
--- /dev/null
+++ b/third_party/README.md
@@ -0,0 +1,9 @@
+# Wheels
+
+This directory contains code, metadata files, and wheels, from other projects,
+used for testing. The projects were chosen to match the licence and ownership
+of this project, i.e. Apache License 2.0, copyright owned by Google, from the
+Google organization repository. Do not add any other type of thing here.
+
+- [python_portpicker](https://github.com/google/python_portpicker)
+- [yapf](https://github.com/google/yapf)
diff --git a/third_party/pypi__portpicker_1_2_0/BUILD b/third_party/pypi__portpicker_1_2_0/BUILD
new file mode 100644
index 0000000..6cff5d5
--- /dev/null
+++ b/third_party/pypi__portpicker_1_2_0/BUILD
@@ -0,0 +1,10 @@
+# Test package for Subpar
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+ name = "files",
+ srcs = [],
+ data = glob(["portpicker-1.2.0.dist-info/**"]),
+ imports = ["."],
+)
diff --git a/third_party/pypi__portpicker_1_2_0/WORKSPACE b/third_party/pypi__portpicker_1_2_0/WORKSPACE
new file mode 100644
index 0000000..57c31df
--- /dev/null
+++ b/third_party/pypi__portpicker_1_2_0/WORKSPACE
@@ -0,0 +1 @@
+workspace(name = "pypi__portpicker_1_2_0")
diff --git a/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
new file mode 100644
index 0000000..8e852c1
--- /dev/null
+++ b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
@@ -0,0 +1,29 @@
+Metadata-Version: 2.0
+Name: portpicker
+Version: 1.2.0
+Summary: A library to choose unique available network ports.
+Home-page: https://github.com/google/python_portpicker
+Author: Google
+Author-email: greg@krypto.org
+License: Apache 2.0
+Description-Content-Type: UNKNOWN
+Platform: POSIX
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: Jython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+
+Portpicker provides an API to find and return an available network
+port for an application to bind to. Ideally suited for use from
+unittests or for test harnesses that launch local servers.
+
diff --git a/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
new file mode 100644
index 0000000..6ec9c43
--- /dev/null
+++ b/third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
@@ -0,0 +1 @@
+{"classifiers": ["Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: Jython", "Programming Language :: Python :: Implementation :: PyPy"], "description_content_type": "UNKNOWN", "extensions": {"python.details": {"contacts": [{"email": "greg@krypto.org", "name": "Google", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/google/python_portpicker"}}}, "generator": "bdist_wheel (0.30.0)", "license": "Apache 2.0", "metadata_version": "2.0", "name": "portpicker", "platform": "POSIX", "summary": "A library to choose unique available network ports.", "version": "1.2.0"}
\ No newline at end of file
diff --git a/third_party/pypi__yapf_0_19_0/BUILD b/third_party/pypi__yapf_0_19_0/BUILD
new file mode 100644
index 0000000..cd12f69
--- /dev/null
+++ b/third_party/pypi__yapf_0_19_0/BUILD
@@ -0,0 +1,10 @@
+# Test package for Subpar
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+ name = "files",
+ srcs = [],
+ data = glob(["yapf-0.19.0.dist-info/**"]),
+ imports = ["."],
+)
diff --git a/third_party/pypi__yapf_0_19_0/WORKSPACE b/third_party/pypi__yapf_0_19_0/WORKSPACE
new file mode 100644
index 0000000..eae31db
--- /dev/null
+++ b/third_party/pypi__yapf_0_19_0/WORKSPACE
@@ -0,0 +1 @@
+workspace(name = "pypi__yapf_0_19_0")
diff --git a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
new file mode 100644
index 0000000..067f12a
--- /dev/null
+++ b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
@@ -0,0 +1,24 @@
+Metadata-Version: 2.0
+Name: yapf
+Version: 0.19.0
+Summary: A formatter for Python code.
+Home-page: UNKNOWN
+Author: Bill Wendling
+Author-email: morbo@google.com
+License: Apache License, Version 2.0
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Quality Assurance
+
+Text from original package has been elided.
diff --git a/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
new file mode 100644
index 0000000..71e9541
--- /dev/null
+++ b/third_party/pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
@@ -0,0 +1 @@
+{"classifiers": ["Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Quality Assurance"], "extensions": {"python.commands": {"wrap_console": {"yapf": "yapf:run_main"}}, "python.details": {"contacts": [{"email": "morbo@google.com", "name": "Bill Wendling", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}}, "python.exports": {"console_scripts": {"yapf": "yapf:run_main"}}}, "generator": "bdist_wheel (0.29.0)", "license": "Apache License, Version 2.0", "metadata_version": "2.0", "name": "yapf", "summary": "A formatter for Python code.", "version": "0.19.0"}
\ No newline at end of file
