Merge branch 'main' into pl-20103-subclass-default-none

.circleci/config.yml

@@ -44,6 +44,10 @@ jobs:
  root: .
  paths:
  - ./coverage_*.xml
+ # test-py313:
+ # <<: *test-py38
+ # docker:
+ # - image: cimg/python:3.13
  test-py312:
  <<: *test-py38
  docker:
@@ -101,7 +105,7 @@ jobs:
  command: |
  curl -Os https://uploader.codecov.io/latest/linux/codecov
  chmod +x codecov
- for py in 3.7 3.8 3.9 3.10 3.11 3.12; do
+ for py in 3.7 3.8 3.9 3.10 3.11 3.12 3.13; do
  for suffix in "" _all _types _pydantic1 _pydantic2; do
  if [ -f coverage_py${py}${suffix}.xml ]; then
  ./codecov \
@@ -152,6 +156,8 @@ workflows:
  only: /^v\d+\.\d+\.\d+.*$/
  - test-py38:
  <<: *buildreq
+ # - test-py313:
+ # <<: *buildreq
  - test-py312:
  <<: *buildreq
  - test-py311:
@@ -166,6 +172,7 @@ workflows:
  <<: *buildreq
  - codecov:
  requires:
+ # - test-py313
  - test-py312
  - test-py311
  - test-py310

.github/workflows/tests.yml

@@ -12,7 +12,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python: [3.7, 3.8, 3.9, "3.10", 3.11, 3.12]
+ python: [3.7, 3.8, 3.9, "3.10", 3.11, 3.12, 3.13]
  steps:
  - uses: actions/checkout@v4
  - uses: actions/setup-python@v5

CHANGELOG.rst

@@ -15,21 +15,29 @@ paths are considered internals and can change in minor and patch releases.
 v4.33.0 (2024-09-??)
 --------------------
 
-Changed
-^^^^^^^
-- For consistency ``add_subclass_arguments`` now sets default ``None`` instead
- of ``SUPPRESS`` (`lightning#20103
- <https://github.com/Lightning-AI/pytorch-lightning/issue/20103>`__).
-
-
-v4.32.2 (2024-09-??)
---------------------
+Added
+^^^^^
+- Support for Python 3.13 (`#554
+ <https://github.com/omni-us/jsonargparse/pull/554>`__).
+- Support for `NotRequired` and `Required` annotations for `TypedDict` keys
+ (`#571 <https://github.com/omni-us/jsonargparse/pull/571>`__).
 
 Fixed
 ^^^^^
 - Callable type with subclass return not showing the ``--*.help`` option (`#567
  <https://github.com/omni-us/jsonargparse/pull/567>`__).
 
+- Forward referenced types not compatible with `Type` typehint (`#576
+ <https://github.com/omni-us/jsonargparse/pull/576/>`__)
+
+Changed
+^^^^^^^
+- Removed shtab experimental warning (`#561
+ <https://github.com/omni-us/jsonargparse/pull/561>`__).
+- For consistency ``add_subclass_arguments`` now sets default ``None`` instead
+ of ``SUPPRESS`` (`lightning#20103
+ <https://github.com/Lightning-AI/pytorch-lightning/issue/20103>`__).
+
 
 v4.32.1 (2024-08-23)
 --------------------

DOCUMENTATION.rst

@@ -401,8 +401,10 @@ Some notes about this support are:
  :ref:`parsing-paths` and :ref:`parsing-urls`.
 
 - ``Dict``, ``Mapping``, ``MutableMapping``, ``MappingProxyType``,
- ``OrderedDict`` and ``TypedDict`` are supported but only with ``str`` or
- ``int`` keys. For more details see :ref:`dict-items`.
+ ``OrderedDict``, and ``TypedDict`` are supported but only with ``str`` or
+ ``int`` keys. ``Required`` and ``NotRequired`` are also supported for
+ fine-grained specification of required/optional ``TypedDict`` keys.
+ For more details see :ref:`dict-items`.
 
 - ``Tuple``, ``Set`` and ``MutableSet`` are supported even though they can't be
  represented in json distinguishable from a list. Each ``Tuple`` element

jsonargparse/_completions.py

@@ -3,7 +3,6 @@
 import locale
 import os
 import re
-import warnings
 from collections import defaultdict
 from contextlib import contextmanager, suppress
 from contextvars import ContextVar
@@ -97,7 +96,6 @@ def __init__(
  def __call__(self, parser, namespace, shell, option_string=None):
  import shtab
 
- warnings.warn("Automatic shtab support is experimental and subject to change.", UserWarning)
  prog = norm_name(parser.prog)
  assert prog
  preambles = []

jsonargparse/_postponed_annotations.py

@@ -229,6 +229,8 @@ def resolve_subtypes_forward_refs(typehint):
  typehint_origin = Tuple
  elif typehint_origin in mapping_origin_types:
  typehint_origin = Dict
+ elif typehint_origin == type:
+ typehint_origin = Type
  typehint = typehint_origin[tuple(subtypes)]
  except Exception as ex:
  if logger:
@@ -240,6 +242,9 @@ def resolve_subtypes_forward_refs(typehint):
 
 def has_subtypes(typehint):
  typehint_origin = get_typehint_origin(typehint)
+ if typehint_origin is type and hasattr(typehint, "__args__"):
+ return True
+
  return (
  typehint_origin == Union
  or typehint_origin in sequence_origin_types
@@ -260,7 +265,6 @@ def get_types(obj: Any, logger: Optional[logging.Logger] = None) -> dict:
  types = get_type_hints(obj, global_vars)
  except Exception as ex1:
  types = ex1 # type: ignore[assignment]
-
  if isinstance(types, dict) and all(not type_requires_eval(t) for t in types.values()):
  return types
 

jsonargparse/_typehints.py

@@ -3,6 +3,7 @@
 import inspect
 import os
 import re
+import sys
 from argparse import ArgumentError
 from collections import OrderedDict, abc, defaultdict
 from contextlib import contextmanager, suppress
@@ -88,6 +89,21 @@
 
 
 Literal = typing_extensions_import("Literal")
+NotRequired = typing_extensions_import("NotRequired")
+Required = typing_extensions_import("Required")
+TypedDict = typing_extensions_import("TypedDict")
+_TypedDictMeta = typing_extensions_import("_TypedDictMeta")
+
+
+def _capture_typing_extension_shadows(name: str, *collections) -> None:
+ """
+ Ensure different origins for types in typing_extensions are captured.
+ """
+ current_module = sys.modules[__name__]
+ typehint = getattr(current_module, name)
+ if getattr(typehint, "__module__", None) == "typing_extensions" and hasattr(__import__("typing"), name):
+ for collection in collections:
+ collection.add(getattr(__import__("typing"), name))
 
 
 root_types = {
@@ -124,6 +140,8 @@
  OrderedDict,
  Callable,
  abc.Callable,
+ NotRequired,
+ Required,
 }
 
 leaf_types = {
@@ -160,9 +178,20 @@
 callable_origin_types = {Callable, abc.Callable}
 
 literal_types = {Literal}
-if getattr(Literal, "__module__", None) == "typing_extensions" and hasattr(__import__("typing"), "Literal"):
- root_types.add(__import__("typing").Literal)
- literal_types.add(__import__("typing").Literal)
+_capture_typing_extension_shadows("Literal", root_types, literal_types)
+
+not_required_types = {NotRequired}
+_capture_typing_extension_shadows("NotRequired", root_types, not_required_types)
+
+required_types = {Required}
+_capture_typing_extension_shadows("Required", root_types, required_types)
+not_required_required_types = not_required_types.union(required_types)
+
+typed_dict_types = {TypedDict}
+_capture_typing_extension_shadows("TypedDict", typed_dict_types)
+
+typed_dict_meta_types = {_TypedDictMeta}
+_capture_typing_extension_shadows("_TypedDictMeta", typed_dict_meta_types)
 
 subclass_arg_parser: ContextVar = ContextVar("subclass_arg_parser")
 allow_default_instance: ContextVar = ContextVar("allow_default_instance", default=False)
@@ -889,11 +918,18 @@ def adapt_typehints(
  else:
  kwargs["prev_val"] = None
  val[k] = adapt_typehints(v, subtypehints[1], **kwargs)
- if get_import_path(typehint.__class__) == "typing._TypedDictMeta":
+ if type(typehint) in typed_dict_meta_types:
  if typehint.__total__:
- missing_keys = typehint.__annotations__.keys() - val.keys()
- if missing_keys:
- raise_unexpected_value(f"Missing required keys: {missing_keys}", val)
+ required_keys = {
+ k for k, v in typehint.__annotations__.items() if get_typehint_origin(v) not in not_required_types
+ }
+ else:
+ required_keys = {
+ k for k, v in typehint.__annotations__.items() if get_typehint_origin(v) in required_types
+ }
+ missing_keys = required_keys - val.keys()
+ if missing_keys:
+ raise_unexpected_value(f"Missing required keys: {missing_keys}", val)
  extra_keys = val.keys() - typehint.__annotations__.keys()
  if extra_keys:
  raise_unexpected_value(f"Unexpected keys: {extra_keys}", val)
@@ -904,6 +940,11 @@ def adapt_typehints(
  elif typehint_origin is OrderedDict:
  val = dict(val) if serialize else OrderedDict(val)
 
+ # TypedDict NotRequired and Required
+ elif typehint_origin in not_required_required_types:
+ assert len(subtypehints) == 1, "(Not)Required requires a single type argument"
+ val = adapt_typehints(val, subtypehints[0], **adapt_kwargs)
+
  # Callable
  elif typehint_origin in callable_origin_types or typehint in callable_origin_types:
  if serialize:

jsonargparse/_util.py

@@ -273,7 +273,7 @@ def get_typehint_origin(typehint):
  typehint_class = get_import_path(typehint.__class__)
  if typehint_class == "types.UnionType":
  return Union
- if typehint_class == "typing._TypedDictMeta":
+ if typehint_class in {"typing._TypedDictMeta", "typing_extensions._TypedDictMeta"}:
  return dict
  return getattr(typehint, "__origin__", None)
 

jsonargparse_tests/test_formatters.py

@@ -26,7 +26,7 @@ def test_help_action_config_file(parser):
  parser.add_argument("-c", "--cfg", help="Config in yaml/json.", action="config")
  help_str = get_parser_help(parser)
  assert "ARG: --print_config" in help_str
- assert "ARG: -c CFG, --cfg CFG" in help_str
+ assert "ARG: -c CFG, --cfg CFG" in help_str or "ARG: -c, --cfg CFG" in help_str
  assert "ENV: APP_CFG" in help_str
  assert "Config in yaml/json." in help_str
  assert "APP_PRINT_CONFIG" not in help_str

jsonargparse_tests/test_postponed_annotations.py

@@ -10,7 +10,11 @@
 
 from jsonargparse import Namespace
 from jsonargparse._parameter_resolvers import get_signature_parameters as get_params
-from jsonargparse._postponed_annotations import TypeCheckingVisitor, evaluate_postponed_annotations, get_types
+from jsonargparse._postponed_annotations import (
+ TypeCheckingVisitor,
+ evaluate_postponed_annotations,
+ get_types,
+)
 from jsonargparse.typing import Path_drw
 from jsonargparse_tests.conftest import capture_logs, source_unavailable
 
@@ -267,6 +271,17 @@ def test_get_types_type_checking_tuple():
  assert str(types["p1"]) == f"{tpl}[{__name__}.TypeCheckingClass1, {__name__}.TypeCheckingClass2]"
 
 
+def function_type_checking_type(p1: Type["TypeCheckingClass2"]):
+ return p1
+
+
+def test_get_types_type_checking_type():
+ types = get_types(function_type_checking_type)
+ assert list(types.keys()) == ["p1"]
+ tpl = "typing.Type" if sys.version_info < (3, 10) else "type"
+ assert str(types["p1"]) == f"{tpl}[{__name__}.TypeCheckingClass2]"
+
+
 def function_type_checking_dict(p1: Dict[str, Union[TypeCheckingClass1, "TypeCheckingClass2"]]):
  return p1
 

jsonargparse_tests/test_shtab.py

@@ -7,7 +7,6 @@
 from pathlib import Path
 from typing import Any, Callable, Optional, Union
 from unittest.mock import patch
-from warnings import catch_warnings
 
 import pytest
 
@@ -45,10 +44,7 @@ def parser() -> ArgumentParser:
 
 
 def get_shtab_script(parser, shell):
- with catch_warnings(record=True) as w:
- shtab_script = get_parse_args_stdout(parser, [f"--print_shtab={shell}"])
- assert "support is experimental" in str(w[0].message)
- return shtab_script
+ return get_parse_args_stdout(parser, [f"--print_shtab={shell}"])
 
 
 def assert_bash_typehint_completions(subtests, shtab_script, completions):

jsonargparse_tests/test_stubs_resolver.py

@@ -150,7 +150,7 @@ def test_get_params_classmethod():
  expected = expected[:4] + ["compresslevel"] + expected[4:]
  assert expected == get_param_names(params)[: len(expected)]
  if sys.version_info >= (3, 10):
- assert all(p.annotation is not inspect._empty for p in params if p.name != "compresslevel")
+ assert all(p.annotation is not inspect._empty for p in params if p.name not in {"compresslevel", "stream"})
  with mock_typeshed_client_unavailable():
  params = get_params(TarFile, "open")
  assert expected == get_param_names(params)[: len(expected)]