Fixes to support Python 3.10, require 3.6+ (#24)
Switch from asyncio.coroutine to async def to support 3.10.
https://www.python.org/dev/peps/pep-0492/
Drop the use of the loop= parameter in portserver for 3.10.
Refactor portserver_test to launch a subprocess instead of mock.
This will wind up as 1.4.0.
Use tox for our testing.
Test on 3.10 in CI.
Require 3.6+.
Add a package.sh.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index ace9f1e..d50e439 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -12,7 +12,7 @@
strategy:
fail-fast: false
matrix:
- python-version: [3.6, 3.7, 3.8, 3.9]
+ python-version: [3.6, 3.7, 3.8, 3.9, '3.10.0-beta.1']
steps:
- uses: actions/checkout@v2
diff --git a/.travis.yml b/.travis.yml
index 6dc9230..5c5e2ad 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,9 +4,9 @@
- "3.7"
- "3.8"
- "3.9"
+ - "3.10-dev"
os: linux
arch:
- - amd64
- ppc64le
dist: focal
install:
diff --git a/ChangeLog.md b/ChangeLog.md
index 8430f45..b385a38 100644
--- a/ChangeLog.md
+++ b/ChangeLog.md
@@ -1,3 +1,11 @@
+## 1.4.0
+
+* Use `async def` instead of `@asyncio.coroutine` in order to support 3.10.
+* The portserver now checks for and rejects pid values that are out of range.
+* Declare a minimum Python version of 3.6 in the package config.
+* Rework `portserver_test.py` to launch an actual portserver process instead
+ of mocks.
+
## 1.3.9
* No portpicker or portserver code changes
diff --git a/MANIFEST.in b/MANIFEST.in
index 581e4de..a5db4a8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -6,3 +6,4 @@
include ChangeLog.md
include setup.py
include test.sh
+exclude package.sh
diff --git a/package.sh b/package.sh
new file mode 100755
index 0000000..7fb24ad
--- /dev/null
+++ b/package.sh
@@ -0,0 +1,11 @@
+#!/bin/sh -ex
+
+unset PYTHONPATH
+python3 -m venv build/venv
+. build/venv/bin/activate
+
+pip install --upgrade build twine
+python -m build
+twine check dist/*
+
+echo 'When ready, upload to PyPI using: build/venv/bin/twine upload dist/*'
diff --git a/pyproject.toml b/pyproject.toml
index fa2cd65..b1236df 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -17,7 +17,5 @@
commands =
check-manifest --ignore 'src/tests/**'
python -c 'from setuptools import setup; setup()' check -m -s
- pip install --upgrade build
- python -m build
- py.test {posargs}
+ py.test -s {posargs}
"""
diff --git a/setup.cfg b/setup.cfg
index 69d8982..aec9f46 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,7 +1,7 @@
# https://setuptools.readthedocs.io/en/latest/setuptools.html#configuring-setup-using-setup-cfg-files
[metadata]
name = portpicker
-version = 1.3.9
+version = 1.4.0
maintainer = Google LLC
maintainer_email = greg@krypto.org
license = Apache 2.0
@@ -21,12 +21,14 @@
Programming Language :: Python :: 3.7
Programming Language :: Python :: 3.8
Programming Language :: Python :: 3.9
+ Programming Language :: Python :: 3.10
Programming Language :: Python :: Implementation :: CPython
Programming Language :: Python :: Implementation :: PyPy
platforms = POSIX
requires =
[options]
+python_requires = >= 3.6
package_dir=
=src
py_modules = portpicker
diff --git a/src/portserver.py b/src/portserver.py
index 43f5567..58b7ecd 100644
--- a/src/portserver.py
+++ b/src/portserver.py
@@ -117,7 +117,7 @@
return False
try:
os.kill(pid, 0)
- except ProcessLookupError:
+ except (ProcessLookupError, OverflowError):
log.info('Not allocating a port to a non-existent process')
return False
return True
@@ -227,9 +227,8 @@
for port in ports_to_serve:
self._port_pool.add_port_to_free_pool(port)
- @asyncio.coroutine
- def handle_port_request(self, reader, writer):
- client_data = yield from reader.read(100)
+ async def handle_port_request(self, reader, writer):
+ client_data = await reader.read(100)
self._handle_port_request(client_data, writer)
writer.close()
@@ -241,6 +240,8 @@
writer: The asyncio Writer for the response to be written to.
"""
try:
+ if len(client_data) > 20:
+ raise ValueError('More than 20 characters in "pid".')
pid = int(client_data)
except ValueError as error:
self._client_request_errors += 1
@@ -349,10 +350,11 @@
event_loop = asyncio.get_event_loop()
event_loop.add_signal_handler(signal.SIGUSR1, request_handler.dump_stats)
+ old_py_loop = {'loop': event_loop} if sys.version_info < (3, 10) else {}
coro = asyncio.start_unix_server(
request_handler.handle_port_request,
path=config.portserver_unix_socket_address.replace('@', '\0', 1),
- loop=event_loop)
+ **old_py_loop)
server_address = config.portserver_unix_socket_address
server = event_loop.run_until_complete(coro)
diff --git a/src/tests/portserver_test.py b/src/tests/portserver_test.py
index c87ad82..394b1b5 100644
--- a/src/tests/portserver_test.py
+++ b/src/tests/portserver_test.py
@@ -16,11 +16,13 @@
#
"""Tests for the example portserver."""
-from __future__ import print_function
import asyncio
import os
+import signal
import socket
+import subprocess
import sys
+import time
import unittest
from unittest import mock
@@ -129,38 +131,108 @@
portserver._configure_logging(False)
portserver._configure_logging(True)
+
+ _test_socket_addr = f'@TST-{os.getpid()}'
+
@mock.patch.object(
sys, 'argv', ['PortserverFunctionsTest.test_main',
- '--portserver_unix_socket_address=@TST-%d' % os.getpid()]
+ f'--portserver_unix_socket_address={_test_socket_addr}']
)
@mock.patch.object(portserver, '_parse_port_ranges')
- @mock.patch.object(asyncio, 'get_event_loop')
- def test_main(self, *unused_mocks):
+ def test_main_no_ports(self, *unused_mocks):
portserver._parse_port_ranges.return_value = set()
with self.assertRaises(SystemExit):
portserver.main()
- # Give it at least one port and try again.
- portserver._parse_port_ranges.return_value = {self.port}
+ @unittest.skipUnless(sys.executable, 'Requires a stand alone interpreter')
+ @unittest.skipUnless(hasattr(socket, 'AF_UNIX'), 'AF_UNIX required')
+ def test_portserver_binary(self):
+ """Launch python portserver.py and test it."""
+ # Blindly assuming tree layout is src/tests/portserver_test.py
+ # with src/portserver.py.
+ portserver_py = os.path.join(
+ os.path.dirname(os.path.dirname(__file__)),
+ 'portserver.py')
+ anon_addr = self._test_socket_addr.replace('@', '\0')
- @asyncio.coroutine
- def mock_coroutine_template(*args, **kwargs):
- return mock.Mock()
+ conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+ with self.assertRaises(
+ ConnectionRefusedError,
+ msg=f'{self._test_socket_addr} should not listen yet.'):
+ conn.connect(anon_addr)
+ conn.close()
- mock_start_unix_server = mock.Mock(wraps=mock_coroutine_template)
+ server = subprocess.Popen(
+ [sys.executable, portserver_py,
+ f'--portserver_unix_socket_address={self._test_socket_addr}'],
+ stderr=subprocess.PIPE,
+ )
+ try:
+ # Wait a few seconds for the server to start listening.
+ start_time = time.monotonic()
+ while True:
+ time.sleep(0.05)
+ try:
+ conn.connect(anon_addr)
+ conn.close()
+ except ConnectionRefusedError:
+ delta = time.monotonic() - start_time
+ if delta < 4:
+ continue
+ else:
+ server.kill()
+ self.fail('Failed to connect to portserver '
+ f'{self._test_socket_addr} within '
+ f'{delta} seconds. STDERR:\n' +
+ server.stderr.read().decode('utf-8'))
+ else:
+ break
- with mock.patch.object(asyncio, 'start_unix_server',
- mock_start_unix_server):
- mock_event_loop = mock.Mock(spec=asyncio.base_events.BaseEventLoop)
- asyncio.get_event_loop.return_value = mock_event_loop
- mock_event_loop.run_forever.side_effect = KeyboardInterrupt
+ ports = set()
+ port = portpicker.get_port_from_port_server(
+ portserver_address=self._test_socket_addr)
+ ports.add(port)
+ port = portpicker.get_port_from_port_server(
+ portserver_address=self._test_socket_addr)
+ ports.add(port)
- portserver.main()
+ with subprocess.Popen('exit 0', shell=True) as quick_process:
+ quick_process.wait()
+ # This process doesn't exist so it should be a denied alloc.
+ # We use the pid from the above quick_process under the assumption
+ # that most OSes try to avoid rapid pid recycling.
+ denied_port = portpicker.get_port_from_port_server(
+ portserver_address=self._test_socket_addr,
+ pid=quick_process.pid) # A now unused pid.
+ self.assertIsNone(denied_port)
- mock_event_loop.run_until_complete.assert_any_call(
- mock.ANY)
- mock_event_loop.close.assert_called_once_with()
- # NOTE: This could be improved. Tests of main() are often gross.
+ self.assertEqual(len(ports), 2, msg=ports)
+
+ # Check statistics from portserver
+ server.send_signal(signal.SIGUSR1)
+ # TODO implement an I/O timeout
+ for line in server.stderr:
+ if b'denied-allocations ' in line:
+ denied_allocations = int(
+ line.split(b'denied-allocations ', 2)[1])
+ self.assertEqual(1, denied_allocations, msg=line)
+ elif b'total-allocations ' in line:
+ total_allocations = int(
+ line.split(b'total-allocations ', 2)[1])
+ self.assertEqual(2, total_allocations, msg=line)
+ break
+
+ rejected_port = portpicker.get_port_from_port_server(
+ portserver_address=self._test_socket_addr,
+ pid=99999999999999999999999999999999999) # Out of range.
+ self.assertIsNone(rejected_port)
+
+ # Done. shutdown gracefully.
+ server.send_signal(signal.SIGINT)
+ server.communicate(timeout=2)
+ finally:
+ server.kill()
+ server.wait()
class PortPoolTest(unittest.TestCase):
diff --git a/test.sh b/test.sh
index cbd0c69..9407f7c 100755
--- a/test.sh
+++ b/test.sh
@@ -1,27 +1,9 @@
#!/bin/sh -ex
-if which python3 >/dev/null ; then
- echo 'TESTING under Python 3'
- mkdir -p build/test_envs/python3
- python3 -m venv build/test_envs/python3
- # Without --upgrade pip won't copy local changes over to a new test install
- # unless you've updated the package version number.
- build/test_envs/python3/bin/pip install --upgrade pip
- build/test_envs/python3/bin/pip install --upgrade .
- build/test_envs/python3/bin/python3 src/tests/portpicker_test.py
+unset PYTHONPATH
+python3 -m venv build/venv
+. build/venv/bin/activate
- echo 'TESTING the portserver'
- PYTHONPATH=src build/test_envs/python3/bin/python3 src/tests/portserver_test.py
-fi
-
-if which python2.7 >/dev/null ; then
- echo 'TESTING under Python 2.7'
- mkdir -p build/test_envs/python2
- virtualenv --python=python2.7 build/test_envs/python2
- build/test_envs/python2/bin/pip install mock
- build/test_envs/python2/bin/pip install --upgrade pip
- build/test_envs/python2/bin/pip install --upgrade .
- build/test_envs/python2/bin/python2 src/tests/portpicker_test.py
-fi
-
-echo PASS
+pip install --upgrade pip
+pip install tox
+tox -e "py3$(python -c 'import sys; print(sys.version_info.minor)')"
