diff --git a/CHANGES.md b/CHANGES.md index f04414489..c61c86adb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,35 @@ # Release Notes +## 2.2.0 + +This release adds tools to interact with Pex's new embedded offline +documentation. You can browse those docs with `pex --docs` or, more +flexibly, with `pex3 docs`. See `pex3 docs --help` for all the options +available. + +This release also returns to [SemVer](https://semver.org/) versioning +practices. Simply, you can expect 3 things from Pex version numbers: + ++ The first component (the major version) will remain 2 as long as + possible. Pex tries very hard to never break existing users and to + allow them to upgrade without fear of breaking. This includes not + breaking Python compatibility. In Pex 2, Python 2.7 is supported as + well as Python 3.5+ for both CPython and PyPy. Pex will only continue + to add support for new CPython and PyPy releases and never remove + support for already supported Python versions while the major version + remains 2. ++ The second component (the minor version) will be incremented whenever + a release adds a feature. Since Pex is a command line tool only (not + a library), this means you can expect a new subcommand, a new option, + or a new allowable option value was added. Bugs might also have been + fixed. ++ The third component (the patch version) indicates only bugs were + fixed. + +You can expect the minor version to get pretty big going forward! + +* Add `pex --docs` and several `pex3 docs` options. (#2365) + ## 2.1.164 This release moves Pex documentation from https://pex.readthedocs.io to diff --git a/pex/bin/pex.py b/pex/bin/pex.py index df1336fce..8ca249286 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -25,6 +25,7 @@ ) from pex.common import die, is_pyc_dir, is_pyc_file, safe_mkdtemp from pex.dependency_manager import DependencyManager +from pex.docs.command import serve_html_docs from pex.enum import Enum from pex.inherit_path import InheritPath from pex.interpreter_constraints import InterpreterConstraints @@ -50,7 +51,7 @@ if TYPE_CHECKING: from argparse import Namespace - from typing import Dict, Iterable, Iterator, List, Optional, Set, Text, Tuple + from typing import Dict, Iterable, Iterator, List, NoReturn, Optional, Set, Text, Tuple import attr # vendor:skip @@ -90,6 +91,13 @@ def __call__(self, parser, namespace, values, option_str=None): sys.exit(0) +class OpenHtmlDocsAction(Action): + def __call__(self, *args, **kwargs): + # type: (...) -> NoReturn + try_(serve_html_docs(open_browser=True)) + sys.exit(0) + + def configure_clp_pex_resolution(parser): # type: (ArgumentParser) -> None group = parser.add_argument_group( @@ -762,8 +770,22 @@ def configure_clp(): "--help-variables", action=PrintVariableHelpAction, nargs=0, - help="Print out help about the various environment variables used to change the behavior of " - "a running PEX file.", + help=( + "Print out help about the various environment variables used to change the behavior " + "of a running PEX file." + ), + ) + parser.add_argument( + "--docs", + "--help-html", + dest="open_html_docs", + action=OpenHtmlDocsAction, + nargs=0, + help=( + "Open a browser to view the embedded documentation for this Pex installation. For " + "more flexible interaction with the embedded documentation, you can use this Pex " + "installation's `pex3` script. Try `pex3 docs --help` to get started." + ), ) return parser diff --git a/pex/cli/commands/docs.py b/pex/cli/commands/docs.py index 89b9e1df0..a22d171c8 100644 --- a/pex/cli/commands/docs.py +++ b/pex/cli/commands/docs.py @@ -1,220 +1,76 @@ # Copyright 2024 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -import errno -import json -import logging -import os -import re -import signal -import subprocess -import sys -import time -from textwrap import dedent - -from pex import docs from pex.cli.command import BuildTimeCommand -from pex.commands.command import try_open_file -from pex.common import safe_open -from pex.result import Error, Result -from pex.typing import TYPE_CHECKING -from pex.variables import ENV -from pex.version import __version__ - -if TYPE_CHECKING: - from typing import Optional, Union - - import attr # vendor:skip -else: - from pex.third_party import attr - - -logger = logging.getLogger(__name__) - - -SERVER_NAME = "Pex v{version} docs HTTP server".format(version=__version__) -SERVER_DIR = os.path.join(ENV.PEX_ROOT, "docs", "server", __version__) - - -@attr.s(frozen=True) -class Pidfile(object): - _PIDFILE = os.path.join(SERVER_DIR, "pidfile") - - @classmethod - def load(cls): - # type: () -> Optional[Pidfile] - try: - with open(cls._PIDFILE) as fp: - data = json.load(fp) - return cls(url=data["url"], pid=data["pid"]) - except (OSError, IOError, ValueError, KeyError) as e: - logger.warning( - "Failed to load {server} pid file from {path}: {err}".format( - server=SERVER_NAME, path=cls._PIDFILE, err=e - ) - ) - return None +from pex.docs.command import HtmlDocsConfig, register_open_options, serve_html_docs +from pex.docs.server import SERVER_NAME, Pidfile +from pex.docs.server import shutdown as shutdown_docs_server +from pex.result import Ok, Result, try_ - @staticmethod - def _read_url( - server_log, # type: str - timeout, # type: float - ): - # type: (...) -> Optional[str] - # The permutations of Python versions, simple http server module and the output it provides: - # 2.7: Serving HTTP on 0.0.0.0 port 46399 ... -mSimpleHttpServer - # 3.5: Serving HTTP on 0.0.0.0 port 45577 ... -mhttp.server - # 3.6+: Serving HTTP on 0.0.0.0 port 33539 (http://0.0.0.0:33539/) ... -mhttp.server +class Docs(BuildTimeCommand): + """Interact with the Pex documentation. - start = time.time() - while time.time() - start < timeout: - with open(server_log) as fp: - for line in fp: - if line.endswith(os.linesep): - match = re.search(r"Serving HTTP on 0.0.0.0 port (?P\d+)", line) - if match: - port = match.group("port") - return "http://localhost:{port}".format(port=port) - return None + With no arguments, ensures a local documentation server is running and then opens a browser + to view the local docs. + """ @classmethod - def record( - cls, - server_log, # type: str - pid, # type: int - timeout=5.0, # type: float - ): - # type: (...) -> Optional[Pidfile] - url = cls._read_url(server_log, timeout) - if not url: - return None - - with safe_open(cls._PIDFILE, "w") as fp: - json.dump(dict(url=url, pid=pid), fp, indent=2, sort_keys=True) - return cls(url=url, pid=pid) - - url = attr.ib() # type: str - pid = attr.ib() # type: int - - def alive(self): - # type: () -> bool - # TODO(John Sirois): Handle pid rollover - try: - os.kill(self.pid, 0) - return True - except OSError as e: - if e.errno == errno.ESRCH: # No such process. - return False - raise - - def kill(self): - # type: () -> None - os.kill(self.pid, signal.SIGTERM) - - -@attr.s(frozen=True) -class LaunchResult(object): - url = attr.ib() # type: str - already_running = attr.ib() # type: bool - - -def launch_docs_server( - document_root, # type: str - port, # type: int - timeout=5.0, # type: float -): - # type: (...) -> Union[str, LaunchResult] - - pidfile = Pidfile.load() - if pidfile and pidfile.alive(): - return LaunchResult(url=pidfile.url, already_running=True) - - # Not proper daemonization, but good enough. - log = os.path.join(SERVER_DIR, "log.txt") - http_server_module = "http.server" if sys.version_info[0] == 3 else "SimpleHttpServer" - env = os.environ.copy() - # N.B.: We set up line buffering for the process pipes as well as the underlying Python running - # the http server to ensure we can observe the `Serving HTTP on ...` line we need to grab the - # ephemeral port chosen. - env.update(PYTHONUNBUFFERED="1") - with safe_open(log, "w") as fp: - process = subprocess.Popen( - args=[sys.executable, "-m", http_server_module, str(port)], - env=env, - cwd=document_root, - preexec_fn=os.setsid, - bufsize=1, - stdout=fp.fileno(), - stderr=subprocess.STDOUT, + def add_extra_arguments(cls, parser): + register_open_options(parser) + parser.add_argument( + "--no-open", + dest="open", + default=True, + action="store_false", + help="Don't open the docs; just ensure the docs server is running and print its info.", ) - - pidfile = Pidfile.record(server_log=log, pid=process.pid, timeout=timeout) - if not pidfile: - try: - os.kill(process.pid, signal.SIGKILL) - except OSError as e: - if e.errno != errno.ESRCH: # No such process. - raise - return log - - return LaunchResult(url=pidfile.url, already_running=False) - - -def shutdown_docs_server(): - # type: () -> bool - - pidfile = Pidfile.load() - if not pidfile: - return False - - logger.info( - "Killing {server} {url} @ {pid}".format( - server=SERVER_NAME, url=pidfile.url, pid=pidfile.pid + kill_or_info = parser.add_mutually_exclusive_group() + kill_or_info.add_argument( + "-k", + "--kill-server", + dest="kill_server", + default=False, + action="store_true", + help="Shut down the {server} if it is running.".format(server=SERVER_NAME), + ) + kill_or_info.add_argument( + "--server-info", + dest="server_info", + default=False, + action="store_true", + help="Print information about the status of the {server}.".format(server=SERVER_NAME), ) - ) - pidfile.kill() - return True - - -class Docs(BuildTimeCommand): - """Interact with the Pex documentation.""" def run(self): # type: () -> Result - html_docs = docs.root(doc_type="html") - if not html_docs: - # TODO(John Sirois): Evaluate if this should fall back to opening latest html docs instead of just - # displaying links. - return Error( - dedent( - """\ - This Pex distribution does not include embedded docs. - You can find the latest docs here: - HTML: https://docs.pex-tool.org - PDF: https://github.com/pex-tool/pex/releases/latest/download/pex.pdf - """ - ).rstrip() - ) + if self.options.server_info: + pidfile = Pidfile.load() + if pidfile and pidfile.alive(): + return Ok( + "{server} serving {info}".format(server=SERVER_NAME, info=pidfile.server_info) + ) + return Ok("No {server} is running.".format(server=SERVER_NAME)) + + if self.options.kill_server: + server_info = shutdown_docs_server() + if server_info: + return Ok("Shut down {server} {info}".format(server=SERVER_NAME, info=server_info)) + return Ok("No {server} was running.".format(server=SERVER_NAME)) - # TODO(John Sirois): Consider trying a standard pex docs port, then fall back to ephemeral if: - # 2.7: socket.error: [Errno 98] Address already in use - # 3.x: OSError: [Errno 98] Address already in use - # This would allow for cookie stickiness for light / dark mode although the furo theme default of detecting - # the system default mode works well in practice to get you what you probably wanted anyhow. - result = launch_docs_server(html_docs, port=0) - if isinstance(result, str): - with open(result) as fp: - for line in fp: - logger.log(logging.ERROR, line.rstrip()) - return Error("Failed to launch {server}.".format(server=SERVER_NAME)) + launch_result = try_( + serve_html_docs( + open_browser=self.options.open, config=HtmlDocsConfig.from_options(self.options) + ) + ) + if self.options.open: + return Ok() - logger.info( + return Ok( ( - "{server} already running at {url}" - if result.already_running - else "Launched {server} at {url}" - ).format(server=SERVER_NAME, url=result.url) + "{server} already running {info}" + if launch_result.already_running + else "Launched {server} {info}" + ).format(server=SERVER_NAME, info=launch_result.server_info) ) - return try_open_file(result.url) diff --git a/pex/commands/command.py b/pex/commands/command.py index c78fdb075..9f6004725 100644 --- a/pex/commands/command.py +++ b/pex/commands/command.py @@ -12,10 +12,12 @@ import tempfile from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace, _ActionsContainer from contextlib import contextmanager +from subprocess import CalledProcessError from pex import pex_warnings from pex.argparse import HandleBoolAction from pex.common import safe_mkdtemp, safe_open +from pex.compatibility import shlex_quote from pex.result import Error, Ok, Result from pex.typing import TYPE_CHECKING, Generic, cast from pex.variables import ENV, Variables @@ -48,11 +50,23 @@ def try_run_program( args, # type: Iterable[str] url=None, # type: Optional[str] error=None, # type: Optional[str] + disown=False, # type: bool **kwargs # type: Any ): # type: (...) -> Result + cmd = [program] + list(args) try: - subprocess.check_call([program] + list(args), **kwargs) + process = subprocess.Popen(cmd, preexec_fn=os.setsid if disown else None, **kwargs) + if not disown and process.wait() != 0: + return Error( + str( + CalledProcessError( + returncode=process.returncode, + cmd=" ".join(shlex_quote(arg) for arg in cmd), + ) + ), + exit_code=process.returncode, + ) return Ok() except OSError as e: msg = [error] if error else [] @@ -62,22 +76,35 @@ def try_run_program( "Find more information on `{program}` at {url}.".format(program=program, url=url) ) return Error("\n".join(msg)) - except subprocess.CalledProcessError as e: - return Error(str(e), exit_code=e.returncode) def try_open_file( path, # type: str + open_program=None, # type: Optional[str] error=None, # type: Optional[str] + suppress_stderr=False, # type: bool ): # type: (...) -> Result - opener, url = ( - ("xdg-open", "https://www.freedesktop.org/wiki/Software/xdg-utils/") - if "Linux" == os.uname()[0] - else ("open", None) - ) + + url = None # type: Optional[str] + if open_program: + opener = open_program + elif "Linux" == os.uname()[0]: + opener = "xdg-open" + url = "https://www.freedesktop.org/wiki/Software/xdg-utils/" + else: + opener = "open" + with open(os.devnull, "wb") as devnull: - return try_run_program(opener, [path], url=url, error=error, stdout=devnull) + return try_run_program( + opener, + [path], + url=url, + error=error, + disown=True, + stdout=devnull, + stderr=devnull if suppress_stderr else None, + ) @attr.s(frozen=True) diff --git a/pex/docs/command.py b/pex/docs/command.py new file mode 100644 index 000000000..9a7609edb --- /dev/null +++ b/pex/docs/command.py @@ -0,0 +1,98 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import + +import logging +from argparse import Namespace, _ActionsContainer +from textwrap import dedent + +from pex import docs +from pex.commands.command import try_open_file +from pex.docs.server import SERVER_NAME, LaunchError, LaunchResult +from pex.docs.server import launch as launch_docs_server +from pex.result import Error, try_ +from pex.typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Optional, Union + + import attr # vendor:skip +else: + from pex.third_party import attr + + +logger = logging.getLogger(__name__) + + +def register_open_options(parser): + # type: (_ActionsContainer) -> None + parser.add_argument( + "--browser", + dest="browser", + default=None, + help="The browser to use to open docs with. Defaults to the system default opener.", + ) + + +@attr.s(frozen=True) +class HtmlDocsConfig(object): + @classmethod + def from_options( + cls, + options, # type: Namespace + fallback_url=None, # type: Optional[str] + ): + # type: (...) -> HtmlDocsConfig + return cls(browser=options.browser, fallback_url=fallback_url) + + browser = attr.ib(default=None) # type: Optional[str] + fallback_url = attr.ib(default=None) # type: Optional[str] + + +# PEX in ascii-hex rotated left 1 character and trailing 0 dropped: +# E X P +# 0x45 0x58 0x50 +# +# This is a currently unassigned port in the 1024-49151 user port registration range. +# This value plays by the loose rules and satisfies the criteria of being unlikely to be in use. +# We have no intention of registering with IANA tough! +STANDARD_PORT = 45585 + + +def serve_html_docs( + open_browser=False, # type: bool + config=HtmlDocsConfig(), # type: HtmlDocsConfig +): + # type: (...) -> Union[LaunchResult, Error] + html_docs = docs.root(doc_type="html") + if not html_docs: + return Error( + dedent( + """\ + This Pex distribution does not include embedded docs. + + You can find the latest docs here: + HTML: https://docs.pex-tool.org + PDF: https://github.com/pex-tool/pex/releases/latest/download/pex.pdf + """ + ).rstrip() + ) + + try: + result = launch_docs_server(html_docs, port=STANDARD_PORT) + except LaunchError: + try: + result = launch_docs_server(html_docs, port=0) + except LaunchError as e: + with open(e.log) as fp: + for line in fp: + logger.log(logging.ERROR, line.rstrip()) + return Error("Failed to launch {server}.".format(server=SERVER_NAME)) + + if open_browser: + try_( + try_open_file(result.server_info.url, open_program=config.browser, suppress_stderr=True) + ) + + return result diff --git a/pex/docs/server.py b/pex/docs/server.py new file mode 100644 index 000000000..7f32a0489 --- /dev/null +++ b/pex/docs/server.py @@ -0,0 +1,200 @@ +# Copyright 2024 Pex project contributors. +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +import errno +import json +import logging +import os +import re +import signal +import subprocess +import sys +import time + +from pex.common import safe_open +from pex.typing import TYPE_CHECKING +from pex.variables import ENV +from pex.version import __version__ + +if TYPE_CHECKING: + from typing import Optional + + import attr # vendor:skip +else: + from pex.third_party import attr + + +logger = logging.getLogger(__name__) + +SERVER_NAME = "Pex v{version} docs HTTP server".format(version=__version__) + +_SERVER_DIR = os.path.join(ENV.PEX_ROOT, "docs", "server", __version__) + + +@attr.s(frozen=True) +class ServerInfo(object): + url = attr.ib() # type: str + pid = attr.ib() # type: int + + def __str__(self): + # type: () -> str + return "{url} @ {pid}".format(url=self.url, pid=self.pid) + + +@attr.s(frozen=True) +class Pidfile(object): + _PIDFILE = os.path.join(_SERVER_DIR, "pidfile") + + @classmethod + def load(cls): + # type: () -> Optional[Pidfile] + try: + with open(cls._PIDFILE) as fp: + data = json.load(fp) + return cls(ServerInfo(url=data["url"], pid=data["pid"])) + except (OSError, IOError, ValueError, KeyError) as e: + logger.debug( + "Failed to load {server} pid file from {path}: {err}".format( + server=SERVER_NAME, path=cls._PIDFILE, err=e + ) + ) + return None + + @staticmethod + def _read_url( + server_log, # type: str + timeout, # type: float + ): + # type: (...) -> Optional[str] + + # The permutations of Python versions, simple http server module and the output it provides: + # 2.7: Serving HTTP on 0.0.0.0 port 46399 ... -mSimpleHttpServer + # 3.5: Serving HTTP on 0.0.0.0 port 45577 ... -mhttp.server + # 3.6+: Serving HTTP on 0.0.0.0 port 33539 (http://0.0.0.0:33539/) ... -mhttp.server + + start = time.time() + while time.time() - start < timeout: + with open(server_log) as fp: + for line in fp: + if line.endswith(os.linesep): + match = re.search(r"Serving HTTP on 0\.0\.0\.0 port (?P\d+)", line) + if match: + port = match.group("port") + return "http://localhost:{port}".format(port=port) + return None + + @classmethod + def record( + cls, + server_log, # type: str + pid, # type: int + timeout=5.0, # type: float + ): + # type: (...) -> Optional[Pidfile] + url = cls._read_url(server_log, timeout) + if not url: + return None + + with safe_open(cls._PIDFILE, "w") as fp: + json.dump(dict(url=url, pid=pid), fp, indent=2, sort_keys=True) + return cls(ServerInfo(url=url, pid=pid)) + + server_info = attr.ib() # type: ServerInfo + + def alive(self): + # type: () -> bool + # TODO(John Sirois): Handle pid rollover + try: + os.kill(self.server_info.pid, 0) + return True + except OSError as e: + if e.errno == errno.ESRCH: # No such process. + return False + raise + + def kill(self): + # type: () -> None + os.kill(self.server_info.pid, signal.SIGTERM) + + +@attr.s(frozen=True) +class LaunchResult(object): + server_info = attr.ib() # type: ServerInfo + already_running = attr.ib() # type: bool + + +# Frozen exception types don't work under 3.11+ where the `__traceback__` attribute can be set +# after construction in some cases. +@attr.s +class LaunchError(Exception): + """Indicates an error launching the docs server.""" + + log = attr.ib() # type: str + additional_msg = attr.ib(default=None) # type: Optional[str] + + def __str__(self): + # type: () -> str + lines = ["Error launching docs server."] + if self.additional_msg: + lines.append(self.additional_msg) + lines.append("See the log at {log} for more details.".format(log=self.log)) + return os.linesep.join(lines) + + +def launch( + document_root, # type: str + port, # type: int + timeout=5.0, # type: float +): + # type: (...) -> LaunchResult + + pidfile = Pidfile.load() + if pidfile and pidfile.alive(): + return LaunchResult(server_info=pidfile.server_info, already_running=True) + + # Not proper daemonization, but good enough. + log = os.path.join(_SERVER_DIR, "log.txt") + http_server_module = "http.server" if sys.version_info[0] == 3 else "SimpleHttpServer" + env = os.environ.copy() + # N.B.: We set up line buffering for the process pipes as well as the underlying Python running + # the http server to ensure we can observe the `Serving HTTP on ...` line we need to grab the + # ephemeral port chosen. + env.update(PYTHONUNBUFFERED="1") + with safe_open(log, "w") as fp: + process = subprocess.Popen( + args=[sys.executable, "-m", http_server_module, str(port)], + env=env, + cwd=document_root, + preexec_fn=os.setsid, + bufsize=1, + stdout=fp.fileno(), + stderr=subprocess.STDOUT, + ) + + pidfile = Pidfile.record(server_log=log, pid=process.pid, timeout=timeout) + if not pidfile: + try: + os.kill(process.pid, signal.SIGKILL) + except OSError as e: + if e.errno != errno.ESRCH: # No such process. + raise LaunchError( + log, + additional_msg=( + "Also failed to kill the partially launched server at pid {pid}: " + "{err}".format(pid=process.pid, err=e) + ), + ) + raise LaunchError(log) + return LaunchResult(server_info=pidfile.server_info, already_running=False) + + +def shutdown(): + # type: () -> Optional[ServerInfo] + + pidfile = Pidfile.load() + if not pidfile or not pidfile.alive(): + return None + + logger.debug("Killing {server} {info}".format(server=SERVER_NAME, info=pidfile.server_info)) + pidfile.kill() + return pidfile.server_info diff --git a/pex/version.py b/pex/version.py index 2ae99045d..7e51a5d59 100644 --- a/pex/version.py +++ b/pex/version.py @@ -1,4 +1,4 @@ # Copyright 2015 Pex project contributors. # Licensed under the Apache License, Version 2.0 (see LICENSE). -__version__ = "2.1.164" +__version__ = "2.2.0"