✨ Add convert-config script for YAML and environment variable conversion

Author: Paillat-dev

pyproject.toml

@@ -36,6 +36,7 @@ export = "pdm export -o requirements.txt --prod"
 tests = "pytest tests"
 start = "python src"
 check-listings = {call = "scripts:check_listings.main"}
+convert-config = { call = "scripts.convert_config:main" }
 
 [tool.pdm]
 distribution = false
@@ -80,7 +81,7 @@ docstring-code-line-length = "dynamic"
 
 [tool.ruff.lint]
 select = ["ALL"]
-per-file-ignores = { "src/database/migrations/*"= ["INP001", "ARG001"] }
+per-file-ignores = { "src/database/migrations/*"= ["INP001", "ARG001"], "tests/*"= ["S101"], "scripts/**" = ["T201"]  }
 extend-ignore = [
     "N999",
     "D104",

readme.md

@@ -119,6 +119,22 @@ BOTKIT__cache__redis__host=redis.example.com
 BOTKIT__cache__redis__port=6379
 ```
 
+### Converting Configuration Formats
+
+When deploying in containerized or high-availability environments where persistent
+volumes might not be available or when using orchestration platforms like Docker Swarm,
+it's often preferable to use environment variables instead of configuration files.
+Botkit provides a convenient script to convert between YAML and environment variable
+formats:
+
+```bash
+pdm run convert-config -i config.yml --terminal
+```
+
+This will output your YAML configuration as environment variables that you can then use
+in your container configuration or deployment platform. For more details about the
+convert-config script, see the [Using scripts](#using-scripts) section.
+
 ### Cache Configuration
 
 Botkit supports two types of caching:
@@ -416,6 +432,45 @@ DiscordMe: # add this section if you want to check discord.me
 
 4. Run the script using `pdm run check-listings`.
 
+### `convert-config`
+
+This script converts the configuration between YAML and env formats.
+
+#### Usage
+
+By default, if run with no arguments, it converts the `config.yaml` or `config.yml`
+present in the root to `.env` format. If a `.env` file is present and is empty or one is
+not present, it converts there. Otherwise, it asks the user if it should overwrite the
+existing `.env` file. If not, it writes to a `datetime.converted.env` file.
+
+#### Options
+
+- `-i`, `--input`: Specify the input file path.
+- `--input-format`: Specify the input format (`yaml`, `yml`, `env`).
+- `--output`: Specify the output file path.
+- `--output-format`: Specify the output format (`yaml`, `yml`, `env`).
+- `--terminal`: Output to the terminal instead of a file.
+
+#### Examples
+
+Convert `config.yaml` to `.env`:
+
+```sh
+pdm run convert-config
+```
+
+Convert a specific file and output to the terminal:
+
+```sh
+pdm run convert-config -i config.yml --terminal
+```
+
+Convert `.env` to `config.yaml`:
+
+```sh
+pdm run convert-config -i .env --output config.yaml
+```
+
 ## Provided Extensions
 
 We provide multiple extensions directly within this project to get you started. These

scripts/__init__.py

@@ -1,6 +1,6 @@
 # Copyright (c) NiceBots.xyz
 # SPDX-License-Identifier: MIT
 
-from . import check_listings
+from . import check_listings, convert_config
 
-__all__ = ["check_listings"]
+__all__ = ["check_listings", "convert_config"]

scripts/convert_config/__init__.py

@@ -0,0 +1,6 @@
+# Copyright (c) NiceBots
+# SPDX-License-Identifier: MIT
+
+from .__main__ import main
+
+__all__ = ["main"]

scripts/convert_config/__main__.py

@@ -0,0 +1,68 @@
+# Copyright (c) NiceBots
+# SPDX-License-Identifier: MIT
+
+import argparse
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+from .convert import env_to_yaml, yaml_to_env
+
+
+def main() -> None:  # noqa: PLR0912
+    parser = argparse.ArgumentParser(description="Convert config between YAML and env formats.")
+    parser.add_argument("-i", "--input", help="Input file path", default=None)
+    parser.add_argument("--input-format", help="Input format (yaml, yml, env)", default=None)
+    parser.add_argument("--output", help="Output file path", default=None)
+    parser.add_argument("--output-format", help="Output format (yaml, yml, env)", default=None)
+    parser.add_argument("--terminal", action="store_true", help="Output to terminal instead of file")
+
+    args = parser.parse_args()
+
+    input_path = Path(args.input) if args.input else None
+    output_path = Path(args.output) if args.output else None
+    input_format = args.input_format
+    output_format = args.output_format
+    terminal = args.terminal
+
+    if not input_path:
+        if Path("config.yaml").exists():
+            input_path = Path("config.yaml")
+        elif Path("config.yml").exists():
+            input_path = Path("config.yml")
+        elif Path(".env").exists():
+            input_path = Path(".env")
+        else:
+            print("No input file found.")
+            sys.exit(1)
+
+    input_format = input_format or input_path.suffix[1:] if input_path.name != ".env" else "env"
+
+    if not output_format:
+        output_format = "env" if input_format in ["yaml", "yml"] else "yaml"
+
+    if terminal:
+        output_path = None
+    elif not output_path:
+        if output_format == "env":
+            output_path = Path(".env")
+            if output_path.exists() and output_path.stat().st_size != 0:
+                response = input(".env file is not empty. Overwrite? (y/n): ")
+                if response.lower() != "y":
+                    output_path = Path(f"{datetime.now(tz=UTC).strftime('%Y%m%d%H%M%S')}.converted.env")
+        else:
+            output_path = Path("config.yaml")
+
+    # Fixed conversion logic
+    if input_format in ["yaml", "yml"] and output_format == "env":
+        yaml_to_env(input_path, output_path)
+    elif input_format == "env" and output_format in ["yaml", "yml"]:
+        env_to_yaml(input_path, output_path)
+    else:
+        print(f"Invalid conversion from '{input_format}' to '{output_format}'")
+        print("Supported conversions: yaml->env, yml->env, env->yaml, env->yml")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/convert_config/convert.py

@@ -0,0 +1,60 @@
+# Copyright (c) NiceBots
+# SPDX-License-Identifier: MIT
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+from dotenv import dotenv_values
+
+
+def yaml_to_env(yaml_path: Path, output_path: Path | None = None) -> None:
+    with yaml_path.open("r", encoding="utf-8") as yaml_file:
+        config = yaml.safe_load(yaml_file)
+
+    env_lines = []
+
+    def recurse_dict(d: dict[Any, Any], prefix: str = "") -> None:
+        for key, val in d.items():
+            if isinstance(val, dict):
+                recurse_dict(val, f"{prefix}{key.lower()}__")
+            else:
+                # Convert boolean values to lowercase strings
+                env_value = str(val).lower() if isinstance(val, bool) else val
+                env_lines.append(f"BOTKIT__{prefix}{key.lower()}={env_value}")
+
+    recurse_dict(config)
+
+    if output_path:
+        with output_path.open("w", encoding="utf-8") as env_file:
+            env_file.write("\n".join(env_lines) + "\n")  # Add final newline
+    else:
+        print("\n".join(env_lines))
+
+
+def env_to_yaml(env_path: Path, output_path: Path | None = None) -> None:
+    env_config = dotenv_values(env_path)
+    yaml_config = {}
+
+    for key, val in env_config.items():
+        if not key.startswith("BOTKIT__"):
+            continue
+        parts = key[8:].split("__")  # Remove BOTKIT__ prefix
+        current = yaml_config
+        for part in parts[:-1]:
+            current = current.setdefault(part.lower(), {})
+
+        # Convert string boolean values to actual booleans
+        yaml_value = val
+        if val.lower() == "true":
+            yaml_value = True
+        elif val.lower() == "false":
+            yaml_value = False
+
+        current[parts[-1].lower()] = yaml_value
+
+    if output_path:
+        with output_path.open("w", encoding="utf-8") as yaml_file:
+            yaml.safe_dump(yaml_config, yaml_file)  # Use safe_dump to avoid string quotes
+    else:
+        print(yaml.safe_dump(yaml_config))

tests/test_convert_config.py

@@ -0,0 +1,87 @@
+# Copyright (c) NiceBots
+# SPDX-License-Identifier: MIT
+
+import os
+import tempfile
+from pathlib import Path
+
+from scripts.convert_config.convert import env_to_yaml, yaml_to_env
+
+
+def test_yaml_to_env() -> None:
+    yaml_content = """
+    bot:
+      token: "your_bot_token"
+    extensions:
+      listings:
+        enabled: false
+        topgg_token: "your_top.gg_token"
+      ping:
+        enabled: true
+    logging:
+      level: INFO
+    """
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as yaml_file:
+        yaml_file.write(yaml_content.encode("utf-8"))
+        yaml_path = Path(yaml_file.name)
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".env") as env_file:
+        env_path = Path(env_file.name)
+
+    yaml_to_env(yaml_path, env_path)
+
+    with env_path.open("r", encoding="utf-8") as env_file:
+        env_content = env_file.read()
+
+    expected_env_content = (
+        "BOTKIT__bot__token=your_bot_token\n"
+        "BOTKIT__extensions__listings__enabled=false\n"
+        "BOTKIT__extensions__listings__topgg_token=your_top.gg_token\n"
+        "BOTKIT__extensions__ping__enabled=true\n"
+        "BOTKIT__logging__level=INFO\n"
+    )
+
+    assert env_content == expected_env_content
+
+    os.remove(yaml_path)
+    os.remove(env_path)
+
+
+def test_env_to_yaml() -> None:
+    env_content = (
+        "BOTKIT__bot__token=your_bot_token\n"
+        "BOTKIT__extensions__listings__enabled=false\n"
+        "BOTKIT__extensions__listings__topgg_token=your_top.gg_token\n"
+        "BOTKIT__extensions__ping__enabled=true\n"
+        "BOTKIT__logging__level=INFO\n"
+        "SOME_OTHER_VAR=should_be_ignored\n"  # This should be ignored
+    )
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".env") as env_file:
+        env_file.write(env_content.encode("utf-8"))
+        env_path = Path(env_file.name)
+
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".yaml") as yaml_file:
+        yaml_path = Path(yaml_file.name)
+
+    env_to_yaml(env_path, yaml_path)
+
+    with yaml_path.open("r", encoding="utf-8") as yaml_file:
+        yaml_content = yaml_file.read()
+
+    expected_yaml_content = (
+        "bot:\n"
+        "  token: your_bot_token\n"
+        "extensions:\n"
+        "  listings:\n"
+        "    enabled: false\n"
+        "    topgg_token: your_top.gg_token\n"
+        "  ping:\n"
+        "    enabled: true\n"
+        "logging:\n"
+        "  level: INFO\n"
+    )
+
+    assert yaml_content == expected_yaml_content
+
+    os.remove(env_path)
+    os.remove(yaml_path)