Merge branch 'main' into default_settings_source

Author: kschwab

.github/workflows/ci.yml

@@ -13,10 +13,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: set up python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.10'
 
@@ -31,20 +31,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-13, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest, windows-latest]
         python: ['3.8', '3.9', '3.10', '3.11', '3.12']
-        exclude:
-          # Python 3.8 and 3.9 are not available on macOS 14
-          - os: macos-13
-            python: '3.10'
-          - os: macos-13
-            python: '3.11'
-          - os: macos-13
-            python: '3.12'
-          - os: macos-latest
-            python: '3.8'
-          - os: macos-latest
-            python: '3.9'
 
     env:
       PYTHON: ${{ matrix.python }}
@@ -53,10 +41,10 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: set up python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python }}
 
@@ -80,7 +68,7 @@ jobs:
     - run: coverage combine
     - run: coverage xml
 
-    - uses: codecov/codecov-action@v3
+    - uses: codecov/codecov-action@v4
       with:
         file: ./coverage.xml
         env_vars: PYTHON,OS
@@ -108,10 +96,10 @@ jobs:
       id-token: write
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: set up python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.10'
 

docs/index.md

@@ -1120,7 +1120,7 @@ parser methods that can be customised, along with their argparse counterparts (t
 * `add_argument_group_method` - (`argparse.ArgumentParser.add_argument_group`)
 * `add_parser_method` - (`argparse._SubParsersAction.add_parser`)
 * `add_subparsers_method` - (`argparse.ArgumentParser.add_subparsers`)
-* `formatter_class` - (`argparse.HelpFormatter`)
+* `formatter_class` - (`argparse.RawDescriptionHelpFormatter`)
 
 For a non-argparse parser the parser methods can be set to `None` if not supported. The CLI settings will only raise an
 error when connecting to the root parser if a parser method is necessary but set to `None`.

pydantic_settings/sources.py

@@ -8,11 +8,12 @@
 import typing
 import warnings
 from abc import ABC, abstractmethod
-from argparse import SUPPRESS, ArgumentParser, HelpFormatter, Namespace, _SubParsersAction
+from argparse import SUPPRESS, ArgumentParser, Namespace, RawDescriptionHelpFormatter, _SubParsersAction
 from collections import deque
 from dataclasses import asdict, is_dataclass
 from enum import Enum
 from pathlib import Path
+from textwrap import dedent
 from types import FunctionType
 from typing import (
     TYPE_CHECKING,
@@ -424,7 +425,7 @@ class Settings(BaseSettings):
             args = get_args(annotation)
             if origin_is_union(get_origin(field.annotation)) and len(args) == 2 and type(None) in args:
                 for arg in args:
-                    if arg != type(None):
+                    if arg is not None:
                         annotation = arg
                         break
 
@@ -748,7 +749,7 @@ class Cfg(BaseSettings):
         elif is_model_class(annotation) or is_pydantic_dataclass(annotation):
             fields = (
                 annotation.__pydantic_fields__
-                if is_pydantic_dataclass(annotation)
+                if is_pydantic_dataclass(annotation) and hasattr(annotation, '__pydantic_fields__')
                 else cast(BaseModel, annotation).model_fields
             )
             # `case_sensitive is None` is here to be compatible with the old behavior.
@@ -812,7 +813,7 @@ def explode_env_vars(self, field_name: str, field: FieldInfo, env_vars: Mapping[
                         if not allow_json_failure:
                             raise e
             if isinstance(env_var, dict):
-                if last_key not in env_var or not isinstance(env_val, EnvNoneType) or env_var[last_key] is {}:
+                if last_key not in env_var or not isinstance(env_val, EnvNoneType) or env_var[last_key] == {}:
                     env_var[last_key] = env_val
 
         return result
@@ -966,7 +967,7 @@ class CliSettingsSource(EnvSettingsSource, Generic[T]):
             Defaults to `argparse._SubParsersAction.add_parser`.
         add_subparsers_method: The root parser add subparsers (sub-commands) method.
             Defaults to `argparse.ArgumentParser.add_subparsers`.
-        formatter_class: A class for customizing the root parser help text. Defaults to `argparse.HelpFormatter`.
+        formatter_class: A class for customizing the root parser help text. Defaults to `argparse.RawDescriptionHelpFormatter`.
     """
 
     def __init__(
@@ -988,7 +989,7 @@ def __init__(
         add_argument_group_method: Callable[..., Any] | None = ArgumentParser.add_argument_group,
         add_parser_method: Callable[..., Any] | None = _SubParsersAction.add_parser,
         add_subparsers_method: Callable[..., Any] | None = ArgumentParser.add_subparsers,
-        formatter_class: Any = HelpFormatter,
+        formatter_class: Any = RawDescriptionHelpFormatter,
     ) -> None:
         self.cli_prog_name = (
             cli_prog_name if cli_prog_name is not None else settings_cls.model_config.get('cli_prog_name', sys.argv[0])
@@ -1040,7 +1041,10 @@ def __init__(
 
         root_parser = (
             _CliInternalArgParser(
-                cli_exit_on_error=self.cli_exit_on_error, prog=self.cli_prog_name, description=settings_cls.__doc__
+                cli_exit_on_error=self.cli_exit_on_error,
+                prog=self.cli_prog_name,
+                description=None if settings_cls.__doc__ is None else dedent(settings_cls.__doc__),
+                formatter_class=formatter_class,
             )
             if root_parser is None
             else root_parser
@@ -1329,7 +1333,11 @@ def _get_resolved_names(
 
     def _sort_arg_fields(self, model: type[BaseModel]) -> list[tuple[str, FieldInfo]]:
         positional_args, subcommand_args, optional_args = [], [], []
-        fields = model.__pydantic_fields__ if is_pydantic_dataclass(model) else model.model_fields
+        fields = (
+            model.__pydantic_fields__
+            if hasattr(model, '__pydantic_fields__') and is_pydantic_dataclass(model)
+            else model.model_fields
+        )
         for field_name, field_info in fields.items():
             if _CliSubCommand in field_info.metadata:
                 if not field_info.is_required():
@@ -1405,7 +1413,7 @@ def _connect_root_parser(
         add_argument_group_method: Callable[..., Any] | None = ArgumentParser.add_argument_group,
         add_parser_method: Callable[..., Any] | None = _SubParsersAction.add_parser,
         add_subparsers_method: Callable[..., Any] | None = ArgumentParser.add_subparsers,
-        formatter_class: Any = HelpFormatter,
+        formatter_class: Any = RawDescriptionHelpFormatter,
     ) -> None:
         self._root_parser = root_parser
         self._parse_args = self._connect_parser_method(parse_args_method, 'parsed_args_method')
@@ -1424,6 +1432,7 @@ def _connect_root_parser(
             subcommand_prefix=self.env_prefix,
             group=None,
             alias_prefixes=[],
+            model_default=PydanticUndefined,
         )
 
     def _add_parser_args(
@@ -1435,6 +1444,7 @@ def _add_parser_args(
         subcommand_prefix: str,
         group: Any,
         alias_prefixes: list[str],
+        model_default: Any,
     ) -> ArgumentParser:
         subparsers: Any = None
         alias_path_args: dict[str, str] = {}
@@ -1459,24 +1469,27 @@ def _add_parser_args(
                         field_name,
                         help=field_info.description,
                         formatter_class=self._formatter_class,
-                        description=model.__doc__,
+                        description=None if model.__doc__ is None else dedent(model.__doc__),
                     ),
                     model=model,
                     added_args=[],
                     arg_prefix=f'{arg_prefix}{field_name}.',
                     subcommand_prefix=f'{subcommand_prefix}{field_name}.',
                     group=None,
                     alias_prefixes=[],
+                    model_default=PydanticUndefined,
                 )
             else:
                 resolved_names, is_alias_path_only = self._get_resolved_names(field_name, field_info, alias_path_args)
                 arg_flag: str = '--'
                 kwargs: dict[str, Any] = {}
                 kwargs['default'] = SUPPRESS
-                kwargs['help'] = self._help_format(field_info)
+                kwargs['help'] = self._help_format(field_name, field_info, model_default)
                 kwargs['dest'] = f'{arg_prefix}{resolved_names[0]}'
                 kwargs['metavar'] = self._metavar_format(field_info.annotation)
-                kwargs['required'] = self.cli_enforce_required and field_info.is_required()
+                kwargs['required'] = (
+                    self.cli_enforce_required and field_info.is_required() and model_default is PydanticUndefined
+                )
                 if kwargs['dest'] in added_args:
                     continue
                 if _annotation_contains_types(
@@ -1504,8 +1517,10 @@ def _add_parser_args(
                         arg_flag,
                         arg_names,
                         kwargs,
+                        field_name,
                         field_info,
                         resolved_names,
+                        model_default=model_default,
                     )
                 elif is_alias_path_only:
                     continue
@@ -1544,17 +1559,33 @@ def _add_parser_submodels(
         arg_flag: str,
         arg_names: list[str],
         kwargs: dict[str, Any],
+        field_name: str,
         field_info: FieldInfo,
         resolved_names: tuple[str, ...],
+        model_default: Any,
     ) -> None:
         model_group: Any = None
         model_group_kwargs: dict[str, Any] = {}
         model_group_kwargs['title'] = f'{arg_names[0]} options'
-        model_group_kwargs['description'] = (
-            sub_models[0].__doc__
-            if self.cli_use_class_docs_for_groups and len(sub_models) == 1
-            else field_info.description
-        )
+        model_group_kwargs['description'] = field_info.description
+        if self.cli_use_class_docs_for_groups and len(sub_models) == 1:
+            model_group_kwargs['description'] = None if sub_models[0].__doc__ is None else dedent(sub_models[0].__doc__)
+
+        if model_default not in (PydanticUndefined, None):
+            if is_model_class(type(model_default)) or is_pydantic_dataclass(type(model_default)):
+                model_default = getattr(model_default, field_name)
+        else:
+            if field_info.default is not PydanticUndefined:
+                model_default = field_info.default
+            elif field_info.default_factory is not None:
+                model_default = field_info.default_factory
+        if model_default is None:
+            desc_header = f'default: {self.cli_parse_none_str} (undefined)'
+            if model_group_kwargs['description'] is not None:
+                model_group_kwargs['description'] = dedent(f'{desc_header}\n{model_group_kwargs["description"]}')
+            else:
+                model_group_kwargs['description'] = desc_header
+
         if not self.cli_avoid_json:
             added_args.append(arg_names[0])
             kwargs['help'] = f'set {arg_names[0]} from JSON string'
@@ -1569,6 +1600,7 @@ def _add_parser_submodels(
                 subcommand_prefix=subcommand_prefix,
                 group=model_group if model_group else model_group_kwargs,
                 alias_prefixes=[f'{arg_prefix}{name}.' for name in resolved_names[1:]],
+                model_default=model_default,
             )
 
     def _add_parser_alias_paths(
@@ -1658,14 +1690,19 @@ def _metavar_format_recurse(self, obj: Any) -> str:
     def _metavar_format(self, obj: Any) -> str:
         return self._metavar_format_recurse(obj).replace(', ', ',')
 
-    def _help_format(self, field_info: FieldInfo) -> str:
+    def _help_format(self, field_name: str, field_info: FieldInfo, model_default: Any) -> str:
         _help = field_info.description if field_info.description else ''
-        if field_info.is_required():
+        if field_info.is_required() and model_default in (PydanticUndefined, None):
             if _CliPositionalArg not in field_info.metadata:
-                _help += ' (required)' if _help else '(required)'
+                ifdef = 'ifdef: ' if model_default is None else ''
+                _help += f' ({ifdef}required)' if _help else f'({ifdef}required)'
         else:
             default = f'(default: {self.cli_parse_none_str})'
-            if field_info.default not in (PydanticUndefined, None):
+            if is_model_class(type(model_default)) or is_pydantic_dataclass(type(model_default)):
+                default = f'(default: {getattr(model_default, field_name)})'
+            elif model_default not in (PydanticUndefined, None) and callable(model_default):
+                default = f'(default factory: {self._metavar_format(model_default)})'
+            elif field_info.default not in (PydanticUndefined, None):
                 default = f'(default: {field_info.default})'
             elif field_info.default_factory is not None:
                 default = f'(default: {field_info.default_factory})'

pydantic_settings/version.py

@@ -1 +1 @@
-VERSION = '2.3.4'
+VERSION = '2.4.0'

requirements/linting.txt

@@ -12,11 +12,11 @@ click==8.1.7
     # via black
 distlib==0.3.8
     # via virtualenv
-filelock==3.15.3
+filelock==3.15.4
     # via virtualenv
-identify==2.5.36
+identify==2.6.0
     # via pre-commit
-mypy==1.10.0
+mypy==1.11.1
     # via -r requirements/linting.in
 mypy-extensions==1.0.0
     # via
@@ -40,19 +40,19 @@ pyyaml==6.0.1
     # via
     #   -r requirements/linting.in
     #   pre-commit
-ruff==0.4.10
+ruff==0.5.5
     # via -r requirements/linting.in
 tokenize-rt==5.2.0
     # via pyupgrade
 tomli==2.0.1
     # via
     #   black
     #   mypy
-types-pyyaml==6.0.12.20240311
+types-pyyaml==6.0.12.20240724
     # via -r requirements/linting.in
 typing-extensions==4.12.2
     # via
     #   black
     #   mypy
-virtualenv==20.26.2
+virtualenv==20.26.3
     # via pre-commit

requirements/pyproject.txt

@@ -10,17 +10,17 @@ azure-core==1.30.2
     # via
     #   azure-identity
     #   azure-keyvault-secrets
-azure-identity==1.17.0
+azure-identity==1.17.1
     # via pydantic-settings (pyproject.toml)
 azure-keyvault-secrets==4.8.0
     # via pydantic-settings (pyproject.toml)
-certifi==2024.6.2
+certifi==2024.7.4
     # via requests
 cffi==1.16.0
     # via cryptography
 charset-normalizer==3.3.2
     # via requests
-cryptography==42.0.8
+cryptography==43.0.0
     # via
     #   azure-identity
     #   msal
@@ -29,21 +29,19 @@ idna==3.7
     # via requests
 isodate==0.6.1
     # via azure-keyvault-secrets
-msal==1.28.1
+msal==1.30.0
     # via
     #   azure-identity
     #   msal-extensions
-msal-extensions==1.1.0
+msal-extensions==1.2.0
     # via azure-identity
-packaging==24.1
-    # via msal-extensions
-portalocker==2.8.2
+portalocker==2.10.1
     # via msal-extensions
 pycparser==2.22
     # via cffi
-pydantic==2.7.4
+pydantic==2.8.2
     # via pydantic-settings (pyproject.toml)
-pydantic-core==2.18.4
+pydantic-core==2.20.1
     # via pydantic
 pyjwt[crypto]==2.8.0
     # via