Drop template generation for updating .tx/config (#283)

* Add generate_txconfig.py

* Move TEXT_TO_REPLACE outside function, move parse_args first

* Drop cpython checkout and change language dir

* Make environment variable available

* Handle new path in potodo.sh

* Fix script dir in commit.sh

* Handle lang_dir in stats.py

* Use Path to str

* black

Author: rffontenelle

.github/workflows/sync.yml

@@ -32,41 +32,22 @@ jobs:
       - name: Check out this repository
         uses: actions/checkout@v5
 
-      - name: Checkout CPython ${{ env.PYDOC_VERSION }}
-        uses: actions/checkout@v5
-        with:
-          repository: 'python/cpython'
-          ref: ${{ env.PYDOC_VERSION }}
-          path: cpython
-
       - name: Set language dir variable
         run:
-          echo "LANGUAGE_DIR=cpython/Doc/locales/${{ env.PYDOC_LANGUAGE }}/LC_MESSAGES" >> $GITHUB_ENV
+          echo "PYDOC_LANG_DIR=${{ env.PYDOC_VERSION }}" >> $GITHUB_ENV
 
       - name: Checkout this repository ${{ env.PYDOC_VERSION }}
         uses: actions/checkout@v5
         with:
           ref: ${{ env.PYDOC_VERSION }}
-          path: ${{ env.LANGUAGE_DIR }}
+          path: ${{ env.PYDOC_LANG_DIR }}
 
       - uses: actions/setup-python@v6
         with:
           python-version: ${{ inputs.version }}
           allow-prereleases: true
           cache: 'pip'
-          cache-dependency-path: |
-            requirements.txt
-            cpython/Doc/requirements.txt
-
-      - name: Check for Transifex API Token availability
-        id: secret-check
-        # perform secret check & put boolean result as an output
-        shell: bash
-        run: |
-          available=false
-          [[ "${{ secrets.TX_TOKEN }}" != '' ]] && available=true
-          echo "available=$available" >> $GITHUB_OUTPUT
-          echo "available=$available"
+          cache-dependency-path: requirements.txt
 
       # 2- Install dependencies
 
@@ -81,19 +62,19 @@ jobs:
       - name: Install Python dependencies
         run: |
           pip install -r requirements.txt
-          make -C cpython/Doc venv
 
       # 3- Pull translations
 
-      - name: Generate template files and Transifex config file
-        run: ./scripts/generate_templates.sh
+      - name: Generate updated .tx/config
+        if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
+        run: python scripts/generate_txconfig.py -p ./${{ env.PYDOC_LANG_DIR }} ${{ env.PYDOC_TX_PROJECT }}
 
       - name: Pull translations from Transifex
         id: pull
         if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
         run: |
           # Clean up obsolete files
-          find ./${{ env.LANGUAGE_DIR }} -name '*.po' -exec rm {} \;
+          find ./${{ env.PYDOC_LANG_DIR }} -name '*.po' -exec rm {} \;
           ./scripts/pull_translations.sh
         env:
           TX_TOKEN: ${{ secrets.TX_TOKEN }}
@@ -103,36 +84,42 @@ jobs:
         run: |
           newer_branch=${PYDOC_VERSION%%.*}.$((${PYDOC_VERSION##*.}+1))
           git clone --depth 1 --single-branch --branch $newer_branch https://github.com/python/python-docs-pt-br ${newer_branch}-dir
-          pomerge --from ./${newer_branch}-dir/{**/,}*.po --to ./${{ env.LANGUAGE_DIR }}/{**/,}*.po
+          pomerge --from ./${newer_branch}-dir/{**/,}*.po --to ./${{ env.PYDOC_LANG_DIR }}/{**/,}*.po
           rm -rf ./${newer_branch}-dir
 
       - name: powrap
         if: steps.pull.outcome == 'success'
         run: |
-          cd ./${{ env.LANGUAGE_DIR }}
+          cd ./${{ env.PYDOC_LANG_DIR }}
           powrap *.po **/*.po
 
       - name: Update statistics
         if: always()
         run: |
           ./scripts/stats.py
-          git -C ./${{ env.LANGUAGE_DIR }} diff stats.json
+          git -C ./${{ env.PYDOC_LANG_DIR }} diff stats.json
+        env:
+          PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }}
 
       - name: Update potodo.md
         if: always()
         run: |
           ./scripts/potodo.sh
-          git diff ./${{ env.LANGUAGE_DIR }}/potodo.md
+          git -C ./${{ env.PYDOC_LANG_DIR }} diff potodo.md
+        env:
+          PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }}
 
       # 4- Commit and push translations
 
       - name: Commit
         run: ./scripts/commit.sh
+        env:
+          PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }}
 
       - name: Push
         if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }}
         run: |
-          cd ./${{ env.LANGUAGE_DIR }}
+          cd ./${{ env.PYDOC_LANG_DIR }}
           git push
 
 

scripts/commit.sh

@@ -5,8 +5,12 @@
 
 set -eu
 
-rootdir=$(realpath $(dirname $0))
-cd $rootdir/../cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES
+test -n ${PYDOC_LANGUAGE+x}
+
+rootdir=$(realpath $(dirname $0)/..)
+language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}"
+
+cd "$language_dir"
 
 extra_files=".tx/config stats.json potodo.md"
 
@@ -44,5 +48,5 @@ set -u
 # Commit only if there is any cached file
 if ! git diff-index --cached --quiet HEAD; then
   git add -v $extra_files
-  git commit -vm "$($rootdir/generate_commit_msg.py)"
+  git commit -vm "$($rootdir/scripts/generate_commit_msg.py)"
 fi

scripts/generate_txconfig.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""
+Generate the .tx/config file based on the existing projects
+in Python docs Transifex project. Takes an project slug as
+positional argument, e.g. python-newest or python-313
+"""
+
+import argparse
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+# Replaces required to fix the default values set by 'tx add remote' command.
+# Add or remove
+TEXT_TO_REPLACE = {
+    "2_": "2.",
+    "3_": "3.",
+    "glossary_": "glossary",
+    "collections_": "collections.",
+    "compression_": "compression.",
+    "concurrent_": "concurrent.",
+    "curses_": "curses.",
+    "email_": "email.",
+    "html_": "html.",
+    "http_": "http.",
+    "importlib_resources_": "importlib.resources.",
+    "importlib_": "importlib.",
+    "logging_": "logging.",
+    "multiprocessing_": "multiprocessing.",
+    "os_": "os.",
+    "string_": "string.",
+    "sys_monitoring": "sys.monitoring",
+    "tkinter_": "tkinter.",
+    "unittest_": "unittest.",
+    "urllib_": "urllib.",
+    "xml_dom_": "xml.dom.",
+    "xml_etree_": "xml.etree.",
+    "xmlrpc_": "xmlrpc.",
+    "xml_sax_": "xml.sax.",
+    "xml_": "xml.",
+}
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--root-path",
+        "-p",
+        default=Path("."),
+        help="Path to the translation files, and also the .tx/config (defaults to current directory)",
+    )
+    parser.add_argument(
+        "tx_project", help="Slug of the Transifex project to query resources from"
+    )
+    return parser.parse_args()
+
+
+def reset_tx_config(txconfig: Path):
+    """Create or reset the .tx/config file with basic header."""
+    txconfig.parent.mkdir(exist_ok=True)
+    txconfig.write_text("[main]\nhost = https://www.transifex.com\n", encoding="utf-8")
+    print("Initialized .tx/config.")
+
+
+def populate_resources_from_remote(config_file: Path, tx_project: str):
+    """Add the remote resources from the Transifex project to .tx/config."""
+    result = subprocess.run(
+        [
+            "tx",
+            "--config",
+            str(config_file),
+            "add",
+            "remote",
+            "--file-filter",
+            "<lang>/<resource_slug>.<ext>",
+            f"https://app.transifex.com/python-doc/{tx_project}/",
+        ],
+        check=True,
+    )
+    if result.returncode != 0:
+        print("Failed to add the resources from remote:")
+        print(result.stderr.strip())
+        sys.exit(result.returncode)
+    print("Added remote resources to Transifex.")
+
+
+def patch_config(txconfig: Path):
+    """Patch .tx/config to fixing PO filenames to match the expected."""
+    content = txconfig.read_text(encoding="utf-8").splitlines()
+    new_lines = []
+
+    for line in content:
+        if line.startswith(("source_file", "source_lang")):
+            continue
+
+        if line.startswith("file_filter"):
+            line = line.replace("<lang>/", "")
+            line = line.replace("--", "/")
+
+            for pattern, replacement in TEXT_TO_REPLACE.items():
+                if pattern in line:
+                    line = line.replace(pattern, replacement)
+                    break
+
+        new_lines.append(line)
+
+    text = "\n".join(new_lines)
+
+    txconfig.write_text(text + "\n", encoding="utf-8")
+    print("Updated .tx/config with character substitutions")
+
+
+def main():
+    args = parse_args()
+    config_path = Path(".tx/config")
+    if args.root_path:
+        config_path = args.root_path / config_path
+    reset_tx_config(config_path)
+    populate_resources_from_remote(config_path, args.tx_project)
+    patch_config(config_path)
+
+
+if __name__ == "__main__":
+    main()

scripts/potodo.sh

@@ -10,8 +10,9 @@ test -n ${PYDOC_VERSION+x}
 test -n ${PYDOC_LANGUAGE+x}
 
 rootdir=$(realpath $(dirname $0)/..)
+language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}"
 
-cd "$rootdir"/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES
+cd "$language_dir"
 
 potodo --no-cache > potodo.md
 

scripts/pull_translations.sh

@@ -20,8 +20,12 @@
 set -xeu
 
 test -n ${PYDOC_TX_PROJECT+x}
+test -n ${PYDOC_LANGUAGE+x}
 
-cd "$(dirname $0)/../cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES/"
+rootdir=$(realpath $(dirname $0)/..)
+language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}"
+
+cd "$language_dir"
 
 # If a PO file is provided as input, convert it into Transifex resource
 # and add it be pulled (instead of pulling all translations files).

scripts/stats.py

@@ -17,11 +17,16 @@
 
 def main() -> None:
     """Main function to generate translation stats."""
-    language = os.environ.get("PYDOC_LANGUAGE")
-    if not language:
-        raise ValueError("Environment variable PYDOC_LANGUAGE is not set.")
 
-    pofiles_path = Path(f"cpython/Doc/locales/{language}/LC_MESSAGES")
+    lang_dir = os.environ.get("PYDOC_LANG_DIR")
+    if lang_dir:
+        pofiles_path = Path(lang_dir)
+    else:
+        language = os.environ.get("PYDOC_LANGUAGE")
+        if not language:
+            raise ValueError("Environment variable PYDOC_LANGUAGE is not set.")
+        pofiles_path = Path(f"cpython/Doc/locales/{language}/LC_MESSAGES")
+
     if not pofiles_path.exists():
         raise FileNotFoundError(f"Path does not exist: {pofiles_path}")