blob: a4da0de2fc9511980a9140f16df05e25f6fb43b2 [file]
# Copyright 2026 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Warms up the forkserver and runs one build action in each forked worker.
server.py calls warm_up() once, then forks a worker per request and calls
run() in it. The worker takes on the request's working directory check,
umask, environment, stdio, and argv, and runs SCRIPT the way
`python3 -S SCRIPT ARGS` would with the server's interpreter and startup
flags.
"""
import atexit
import fcntl
import gc
import importlib
import io
import os
import runpy
import signal
import socket
import sys
import traceback
import types
import warnings
from collections.abc import Mapping
from importlib.machinery import PathFinder, SourceFileLoader
from pathlib import Path
from typing import Any, NamedTuple, NoReturn
import client
_PREWARM_EXTRA_MODULES = (
"select",
"build.python.modules.serialization",
"build.scripts.service_profile",
)
_PROJECT_ROOT = Path(__file__).resolve().parents[3]
# Interpreter startup variables that each worker applies from the request's
# environment, as `python3 -S SCRIPT` would read them at startup.
ENV_PYTHONPATH = "PYTHONPATH"
ENV_DONT_WRITE_BYTECODE = "PYTHONDONTWRITEBYTECODE"
ENV_PYCACHE_PREFIX = "PYTHONPYCACHEPREFIX"
def _abspath(entry: str) -> str:
"""Returns a $PYTHONPATH entry made absolute like the interpreter does."""
path = os.path.normpath(entry) # "" -> "."
if path.startswith("/"):
return path
cwd = os.getcwd()
return cwd if path == "." else os.path.join(cwd, path)
def python_path(environ: Mapping[bytes, bytes]) -> list[str]:
"""Returns the sys.path entries that $PYTHONPATH in `environ` adds."""
value = environ.get(os.fsencode(ENV_PYTHONPATH))
if not value or sys.flags.ignore_environment:
return []
return [_abspath(os.fsdecode(entry)) for entry in value.split(b":")]
# The interpreter's own entries (the standard library), after sys.path[0] and
# the server's $PYTHONPATH entries. Every worker's sys.path ends with these.
_INTERPRETER_PATH = tuple(sys.path[1 + len(python_path(os.environb)) :])
class WarmState(NamedTuple):
"""What the server imported at warm-up, for deciding what a script reuses."""
# sys.path[1:] at warm-up: what a script whose $PYTHONPATH matches the
# server's gets after its own directory.
client_path: tuple[str, ...]
# sys.path[0] during warm-up, and the top-level modules loaded from it.
local_dir: str
local_modules: frozenset[str]
# Top-level modules found only through the checkout root that warm-up added.
root_modules: frozenset[str]
def is_path_module(module: Any) -> bool:
"""Returns True if `module` was found on sys.path (file or namespace pkg).
Built-in and frozen modules are found before sys.path is searched, so an
entry on sys.path can't shadow them.
"""
spec = getattr(module, "__spec__", None)
if spec is None:
return False
origin = spec.origin
return (isinstance(origin, str) and origin.startswith("/")) or (
origin is None and spec.submodule_search_locations is not None
)
def warm_up(cwd: bytes) -> WarmState:
"""Changes to `cwd` and pre-imports common modules."""
os.chdir(cwd)
# The server's module search path, plus the checkout root so
# `from build... import ...` works even when invoked outside
# `fx build` without PYTHONPATH.
client_path = tuple(sys.path[1:])
project_root = str(_PROJECT_ROOT)
added_project_root = project_root not in sys.path
if added_project_root:
sys.path.append(project_root)
# Served scripts get the server's warning filters, whatever imports set.
filters = warnings.filters[:]
for extra in _PREWARM_EXTRA_MODULES:
importlib.import_module(extra)
warnings.filters[:] = filters # type: ignore[index]
warnings._filters_mutated() # type: ignore[attr-defined]
local_mods: set[str] = set()
root_mods: set[str] = set()
for name, mod in sys.modules.items():
if "." in name or not is_path_module(mod):
continue
origin = getattr(mod.__spec__, "origin", None)
if isinstance(origin, str) and os.path.dirname(origin) == sys.path[0]:
local_mods.add(name)
elif (
added_project_root
and PathFinder.find_spec(name, client_path) is None
):
root_mods.add(name)
# Keep the imported objects out of the collector's way so that workers
# don't copy-on-write the whole heap when a GC pass runs.
gc.collect()
gc.freeze()
return WarmState(
client_path, sys.path[0], frozenset(local_mods), frozenset(root_mods)
)
def run(
conn: socket.socket,
request: client.Request,
fds: list[int],
warm: WarmState,
import_cwd: bytes,
) -> NoReturn:
"""Runs `request` in this forked worker, then exits with its status.
Args:
conn: The client's connection. Closed once the script starts.
request: What to run.
fds: The client's stdin, stdout, and stderr.
warm: What warm_up() imported.
import_cwd: The directory warm_up() ran in.
"""
started = False
interrupted = False
try:
os.setpgid(0, 0)
for sig in (signal.SIGCHLD, signal.SIGTERM, signal.SIGHUP):
signal.signal(sig, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.default_int_handler)
signal.pthread_sigmask(signal.SIG_SETMASK, set())
if request.cwd != import_cwd:
# Warm-up imports computed paths from import_cwd, so another
# directory (e.g. a nested build) can't be served.
os.write(
fds[2],
b"python_forkserver: this server runs actions from %s, not %s; "
b"unset FX_PYTHON_FORKSERVER_SOCKET to run them without it.\n"
% (import_cwd, request.cwd),
)
os._exit(1)
if _peer_closed(conn):
os._exit(1) # The client is gone, so nobody wants the result.
# Before apply_request(): if conn landed on fd 0-2, dup2() would
# replace it, and closing it afterwards would close the script's stdio.
conn.close()
apply_request(request, fds)
started = True
code = run_script(warm)
except KeyboardInterrupt as e:
if started:
# Like the interpreter: no frames from this file.
print_uncaught(e, __file__)
interrupted = True
code = 130 # In case dying from SIGINT below fails.
except BaseException:
traceback.print_exc()
code = 1
if started:
try:
atexit._run_exitfuncs()
except BaseException:
pass
for stream in (sys.stdout, sys.stderr):
try:
stream.flush()
except BaseException:
pass
if interrupted:
# Like CPython after an uncaught KeyboardInterrupt: once cleanup is
# done, die from SIGINT, so that callers (and Ninja) see the same.
signal.signal(signal.SIGINT, signal.SIG_DFL)
os.kill(os.getpid(), signal.SIGINT)
os._exit(code & 0xFF)
def _peer_closed(conn: socket.socket) -> bool:
"""Returns True if the client end of `conn` has already closed."""
try:
return conn.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) == b""
except BlockingIOError:
return False
except OSError:
return True
def apply_request(request: client.Request, fds: list[int]) -> None:
"""Applies a request's umask, environment, stdio fds, and `sys.argv`.
Takes ownership of `fds` (the client's stdin, stdout, and stderr).
"""
os.umask(request.umask)
environb = os.environb
environb.clear()
for entry in request.env:
key, sep, value = entry.partition(b"=")
if sep and key:
environb[key] = value
# Move any received fd that landed on 0-2 (if server stdio was closed)
# above 2 first so dup2() onto 0, 1, 2 never clobbers another slot.
for i, fd in enumerate(fds):
if fd <= 2:
fds[i] = fcntl.fcntl(fd, fcntl.F_DUPFD_CLOEXEC, 3)
for target, fd in enumerate(fds):
os.dup2(fd, target)
os.close(fd)
# sys.stdout chose line buffering for the server's own fd 1; choose again
# for the client's, the way the interpreter does at startup (stderr is
# always line-buffered, and under -u, stdout writes through instead).
if (
isinstance(sys.stdout, io.TextIOWrapper)
and not sys.stdout.write_through
):
sys.stdout.reconfigure(line_buffering=os.isatty(1))
sys.argv = [os.fsdecode(a) for a in request.argv]
def run_script(warm: WarmState) -> int:
"""Runs `python3 -S SCRIPT ARGS`, with SCRIPT ARGS in sys.argv.
Gives the script the module search path, bytecode settings, and fresh
__main__ that the server's interpreter would give it with the request's
environment. Startup flags and -W options stay the server's.
Returns:
The script's exit code.
"""
script = script_path(sys.argv[0])
if script.endswith(".pyz"):
path0 = script # A zip application's archive holds __main__.py.
else:
path0 = os.path.dirname(os.path.realpath(sys.argv[0]))
# The interpreter's own rule under -S: the script's directory, then
# $PYTHONPATH, then the standard library.
rest = (*python_path(os.environb), *_INTERPRETER_PATH)
forget_shadowed_modules(path0, rest, warm)
sys.path[:] = [path0, *rest]
if not sys.flags.ignore_environment:
sys.dont_write_bytecode = bool(os.environ.get(ENV_DONT_WRITE_BYTECODE))
sys.pycache_prefix = os.environ.get(ENV_PYCACHE_PREFIX) or None
main_module = new_main_module(script)
try:
if script.endswith(".pyz"):
# The same call `python3 app.pyz` makes (pymain_run_module() in
# CPython's Modules/main.c). runpy._run_module_as_main() is
# private, but the public runpy.run_path() differs: it runs the
# archive in a temporary __main__ module and changes sys.argv[0],
# then restores both.
runpy._run_module_as_main("__main__", False) # type: ignore[attr-defined]
else:
# Compile directly from source like `python3 -S SCRIPT` so
# __main__ never writes source-tree __pycache__/*.pyc or skips
# compile-time -W error::SyntaxWarning on cached bytecode.
with open(script, "rb") as f:
code = compile(f.read(), script, "exec", dont_inherit=True)
exec(code, main_module.__dict__)
result: Any = None
except SystemExit as e:
result = e.code
except KeyboardInterrupt:
raise
except BaseException as e:
print_uncaught(e, __file__)
return 1
return _exit_code(result)
def _exit_code(result: Any) -> int:
"""Converts a SystemExit code (or main()'s result) to an exit code."""
if result is None:
return 0
if isinstance(result, int):
return result
print(result, file=sys.stderr)
return 1
def script_path(script: str) -> str:
"""Returns `script` made absolute the way the interpreter does (no normpath)."""
return script if script.startswith("/") else os.getcwd() + "/" + script
def new_main_module(script: str) -> types.ModuleType:
"""Installs and returns an empty __main__ module for running `script`.
Args:
script: Absolute path of the script, as from script_path().
"""
main = types.ModuleType("__main__")
main.__builtins__ = sys.modules["builtins"] # type: ignore[attr-defined]
if not script.endswith(".pyz"):
main.__file__ = script
main.__cached__ = None # type: ignore[attr-defined]
main.__loader__ = SourceFileLoader("__main__", script)
sys.modules["__main__"] = main
return main
def print_uncaught(exc: BaseException, skip_file: str) -> None:
"""Prints `exc` the way the interpreter prints an uncaught exception.
Leaves out the leading frames from `skip_file` (the code that started the
script), as the interpreter would.
"""
tb = exc.__traceback__
while tb is not None and tb.tb_frame.f_code.co_filename == skip_file:
tb = tb.tb_next
# sys.excepthook prints the exception's own __traceback__, if it has one.
sys.excepthook(type(exc), exc.with_traceback(tb), tb)
def forget_shadowed_modules(
path0: str, rest: tuple[str, ...], warm: WarmState
) -> None:
"""Drops pre-imported modules that a script would not import itself.
A fresh interpreter resolves top-level imports on [path0, *rest].
Modules warm-up loaded outside rest, or that a same-named module or
package in path0 (directory or .pyz archive) would shadow, would
resolve differently, so they're removed from sys.modules.
"""
if rest == warm.client_path and os.path.isdir(path0):
if path0 == warm.local_dir:
stale = set(warm.root_modules)
else:
stale = set(warm.local_modules) | set(warm.root_modules)
try:
entries = os.listdir(path0)
except OSError:
entries = []
for entry in entries:
stem = entry.partition(".")[0]
if is_path_module(sys.modules.get(stem)):
stale.add(stem)
else:
stale = set()
for name, mod in list(sys.modules.items()):
if "." in name or not is_path_module(mod):
continue
mod_spec = mod.__spec__
if mod_spec is None:
continue
spec = PathFinder.find_spec(name, [path0, *rest])
if (
spec is None
or spec.origin != mod_spec.origin
or list(spec.submodule_search_locations or ())
!= list(mod_spec.submodule_search_locations or ())
):
stale.add(name)
if not stale:
return
for name in list(sys.modules):
if name.partition(".")[0] in stale:
del sys.modules[name]