fix(config): produce annotated config.toml during legacy migration and

fix config source override order

Author: yaugenst-flex

tests/config/test_legacy.py

@@ -2,6 +2,8 @@
 
 import importlib
 
+import pytest
+
 from tidy3d.config.__init__ import get_manager, reload_config
 
 
@@ -14,9 +16,11 @@ def test_legacy_logging_level(config_manager):
 
 def test_env_switch(config_manager):
     config_module = importlib.import_module("tidy3d.config.__init__")
-    config_module.Env.dev.active()
+    with pytest.warns(DeprecationWarning, match="tidy3d.config.Env"):
+        config_module.Env.dev.active()
     assert get_manager().profile == "dev"
-    config_module.Env.set_current(config_module.Env.prod)
+    with pytest.warns(DeprecationWarning, match="tidy3d.config.Env"):
+        config_module.Env.set_current(config_module.Env.prod)
     assert get_manager().profile == "prod"
 
 

tests/config/test_loader.py

@@ -6,7 +6,10 @@
 from pydantic import Field
 
 from tidy3d.config import get_manager, reload_config
+from tidy3d.config import loader as config_loader
 from tidy3d.config import registry as config_registry
+from tidy3d.config.legacy import finalize_legacy_migration
+from tidy3d.config.loader import migrate_legacy_config
 from tidy3d.config.sections import ConfigSection
 from tidy3d.web.cli.app import tidy3d_cli
 
@@ -32,7 +35,7 @@ def test_save_includes_descriptions(config_manager, mock_config_dir):
     manager.save(include_defaults=True)
 
     content = _config_path(mock_config_dir).read_text(encoding="utf-8")
-    assert "# Web/HTTP configuration." in content
+    assert "Lowest logging level that will be emitted." in content
 
 
 def test_preserves_user_comments(config_manager, mock_config_dir):
@@ -41,10 +44,7 @@ def test_preserves_user_comments(config_manager, mock_config_dir):
 
     config_path = _config_path(mock_config_dir)
     text = config_path.read_text(encoding="utf-8")
-    text = text.replace(
-        "Web/HTTP configuration.",
-        "user-modified comment",
-    )
+    text = text.replace("Lowest logging level that will be emitted.", "user-modified comment")
     config_path.write_text(text, encoding="utf-8")
 
     reload_config(profile="default")
@@ -53,7 +53,7 @@ def test_preserves_user_comments(config_manager, mock_config_dir):
 
     updated = config_path.read_text(encoding="utf-8")
     assert "user-modified comment" in updated
-    assert "Web/HTTP configuration." not in updated
+    assert "Lowest logging level that will be emitted." not in updated
 
 
 def test_profile_preserves_comments(config_manager, mock_config_dir):
@@ -118,7 +118,7 @@ class CLIPlugin(ConfigSection):
         assert result.exit_code == 0, result.output
 
         config_text = _config_path(mock_config_dir).read_text(encoding="utf-8")
-        assert "Web/HTTP configuration." in config_text
+        assert "Lowest logging level that will be emitted." in config_text
         assert "[web]" in config_text
         assert "secret" not in config_text
         assert not profiles_dir.exists()
@@ -143,8 +143,58 @@ class CommentPlugin(ConfigSection):
         manager = get_manager()
         manager.save(include_defaults=True)
         content = _config_path(mock_config_dir).read_text(encoding="utf-8")
-        assert "Comment plugin configuration." in content
         assert "Plugin knob description." in content
     finally:
         config_registry._SECTIONS.pop("plugins.comment_test", None)
         reload_config(profile="default")
+
+
+def test_finalize_legacy_migration_promotes_flat_file(tmp_path):
+    canonical_dir = tmp_path / "canonical"
+    canonical_dir.mkdir()
+    legacy_file = canonical_dir / "config"
+    legacy_file.write_text('apikey = "legacy-key"\n', encoding="utf-8")
+    extra_file = canonical_dir / "extra.txt"
+    extra_file.write_text("keep", encoding="utf-8")
+
+    finalize_legacy_migration(canonical_dir)
+
+    config_toml = canonical_dir / "config.toml"
+    assert config_toml.exists()
+    content = config_toml.read_text(encoding="utf-8")
+    assert "[web]" in content
+    assert "[logging]" in content
+    assert "Lowest logging level that will be emitted." in content
+    assert "legacy-key" in content
+    assert not legacy_file.exists()
+    assert extra_file.exists()
+    assert extra_file.read_text(encoding="utf-8") == "keep"
+
+
+def test_migrate_legacy_config_promotes_structured_config(tmp_path, monkeypatch):
+    legacy_dir = tmp_path / "legacy"
+    legacy_dir.mkdir()
+    legacy_file = legacy_dir / "config"
+    legacy_file.write_text('apikey = "legacy-key"\n', encoding="utf-8")
+    (legacy_dir / "extra.txt").write_text("keep", encoding="utf-8")
+
+    canonical_dir = tmp_path / "canonical"
+
+    monkeypatch.setattr(config_loader, "legacy_config_directory", lambda: legacy_dir)
+    monkeypatch.setattr(config_loader, "canonical_config_directory", lambda: canonical_dir)
+
+    destination = migrate_legacy_config()
+
+    assert destination == canonical_dir
+    config_toml = canonical_dir / "config.toml"
+    assert config_toml.exists()
+    content = config_toml.read_text(encoding="utf-8")
+    assert "[web]" in content
+    assert "[logging]" in content
+    assert "Lowest logging level that will be emitted." in content
+    assert "legacy-key" in content
+    assert not (canonical_dir / "config").exists()
+    extra_file = canonical_dir / "extra.txt"
+    assert extra_file.exists()
+    assert extra_file.read_text(encoding="utf-8") == "keep"
+    assert legacy_dir.exists()

tests/config/test_manager.py

@@ -28,12 +28,19 @@ def test_runtime_isolated_per_profile(config_manager):
     assert config_manager.get_section("web").timeout == 45
 
 
-def test_environment_variable_precedence(monkeypatch, config_manager):
+def test_runtime_overrides_env(monkeypatch, config_manager):
     monkeypatch.setenv("TIDY3D_LOGGING__LEVEL", "WARNING")
     config_manager.switch_profile(config_manager.profile)
     config_manager.update_section("logging", level="DEBUG")
     logging_section = config_manager.get_section("logging")
-    # env var should still take precedence
+    # runtime change should override the environment variable
+    assert logging_section.level == "DEBUG"
+
+
+def test_env_applies_without_runtime_override(monkeypatch, config_manager):
+    monkeypatch.setenv("TIDY3D_LOGGING__LEVEL", "WARNING")
+    config_manager.switch_profile(config_manager.profile)
+    logging_section = config_manager.get_section("logging")
     assert logging_section.level == "WARNING"
 
 

tidy3d/config/legacy.py

@@ -8,6 +8,7 @@
 
 import os
 import ssl
+import warnings
 from pathlib import Path
 from typing import Any, Optional
 
@@ -19,6 +20,12 @@
 from .profiles import BUILTIN_PROFILES
 
 
+def _warn_env_deprecated() -> None:
+    message = "'tidy3d.config.Env' is deprecated; use 'config.switch_profile(...)' instead."
+    warnings.warn(message, DeprecationWarning, stacklevel=3)
+    log.warning(message, log_once=True)
+
+
 class LegacyConfigWrapper:
     """Provide attribute-level compatibility with the legacy config module."""
 
@@ -169,6 +176,7 @@ def manager(self) -> Optional[ConfigManager]:
         return self._manager
 
     def active(self) -> None:
+        _warn_env_deprecated()
         if self._manager is not None and self._manager.profile != self._name:
             self._manager.switch_profile(self._name)
 
@@ -296,6 +304,7 @@ def current(self) -> LegacyEnvironmentConfig:
         return self._current
 
     def set_current(self, env_config: LegacyEnvironmentConfig) -> None:
+        _warn_env_deprecated()
         key = normalize_profile_name(env_config.name)
         if env_config.manager is self._manager:
             if self._manager.profile != key:
@@ -377,5 +386,42 @@ def load_legacy_flat_config(config_dir: Path) -> dict[str, Any]:
     "LegacyConfigWrapper",
     "LegacyEnvironment",
     "LegacyEnvironmentConfig",
+    "finalize_legacy_migration",
     "load_legacy_flat_config",
 ]
+
+
+def finalize_legacy_migration(config_dir: Path) -> None:
+    """Promote a copied legacy configuration tree into the structured format.
+
+    Parameters
+    ----------
+    config_dir : Path
+        Destination directory (typically the canonical config location).
+    """
+
+    legacy_data = load_legacy_flat_config(config_dir)
+
+    from .manager import ConfigManager  # local import to avoid circular dependency
+
+    manager = ConfigManager(profile="default", config_dir=config_dir)
+    config_path = config_dir / "config.toml"
+    for section, values in legacy_data.items():
+        if isinstance(values, dict):
+            manager.update_section(section, **values)
+    try:
+        manager.save(include_defaults=True)
+    except Exception:
+        if config_path.exists():
+            try:
+                config_path.unlink()
+            except Exception:
+                pass
+        raise
+
+    legacy_flat_path = config_dir / "config"
+    if legacy_flat_path.exists():
+        try:
+            legacy_flat_path.unlink()
+        except Exception as exc:
+            log.warning(f"Failed to remove legacy configuration file '{legacy_flat_path}': {exc}")

tidy3d/config/loader.py

@@ -339,6 +339,10 @@ def migrate_legacy_config(*, overwrite: bool = False, remove_legacy: bool = Fals
     canonical_dir.parent.mkdir(parents=True, exist_ok=True)
     shutil.copytree(legacy_dir, canonical_dir, dirs_exist_ok=overwrite)
 
+    from .legacy import finalize_legacy_migration  # local import to avoid circular dependency
+
+    finalize_legacy_migration(canonical_dir)
+
     if remove_legacy:
         shutil.rmtree(legacy_dir)
 

tidy3d/config/manager.py

@@ -318,7 +318,7 @@ def _reload(self) -> None:
         self._raw_tree = deep_merge(self._builtin_data, self._base_data, self._profile_data)
 
         runtime = deepcopy(self._runtime_overrides.get(self._profile, {}))
-        effective = deep_merge(self._raw_tree, runtime, self._env_overrides)
+        effective = deep_merge(self._raw_tree, self._env_overrides, runtime)
         self._effective_tree = effective
         self._build_models()
 

tidy3d/config/serializer.py

@@ -123,8 +123,8 @@ def _apply_value(
         if key in container:
             container[key] = table
         else:
-            if description:
-                container.add(tomlkit.comment(description))
+            if isinstance(container, tomlkit.TOMLDocument) and len(container) > 0:
+                container.add(tomlkit.nl())
             container.add(key, table)
         return
 

tidy3d/web/cli/app.py

@@ -197,7 +197,10 @@ def config_migrate(overwrite: bool, delete_legacy: bool) -> None:
     if delete_legacy:
         click.echo("The legacy '~/.tidy3d' directory was removed.")
     else:
-        click.echo(f"The legacy directory remains at '{legacy_dir}'.")
+        click.echo(
+            f"The legacy directory remains at '{legacy_dir}'. "
+            "Remove it after confirming the new configuration works, or rerun with '--delete-legacy'."
+        )
 
 
 @click.group()