diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index b428fc10a..d187462bc 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -32,41 +32,22 @@ jobs: - name: Check out this repository uses: actions/checkout@v5 - - name: Checkout CPython ${{ env.PYDOC_VERSION }} - uses: actions/checkout@v5 - with: - repository: 'python/cpython' - ref: ${{ env.PYDOC_VERSION }} - path: cpython - - name: Set language dir variable run: - echo "LANGUAGE_DIR=cpython/Doc/locales/${{ env.PYDOC_LANGUAGE }}/LC_MESSAGES" >> $GITHUB_ENV + echo "PYDOC_LANG_DIR=${{ env.PYDOC_VERSION }}" >> $GITHUB_ENV - name: Checkout this repository ${{ env.PYDOC_VERSION }} uses: actions/checkout@v5 with: ref: ${{ env.PYDOC_VERSION }} - path: ${{ env.LANGUAGE_DIR }} + path: ${{ env.PYDOC_LANG_DIR }} - uses: actions/setup-python@v6 with: python-version: ${{ inputs.version }} allow-prereleases: true cache: 'pip' - cache-dependency-path: | - requirements.txt - cpython/Doc/requirements.txt - - - name: Check for Transifex API Token availability - id: secret-check - # perform secret check & put boolean result as an output - shell: bash - run: | - available=false - [[ "${{ secrets.TX_TOKEN }}" != '' ]] && available=true - echo "available=$available" >> $GITHUB_OUTPUT - echo "available=$available" + cache-dependency-path: requirements.txt # 2- Install dependencies @@ -81,19 +62,19 @@ jobs: - name: Install Python dependencies run: | pip install -r requirements.txt - make -C cpython/Doc venv # 3- Pull translations - - name: Generate template files and Transifex config file - run: ./scripts/generate_templates.sh + - name: Generate updated .tx/config + if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }} + run: python scripts/generate_txconfig.py -p ./${{ env.PYDOC_LANG_DIR }} ${{ env.PYDOC_TX_PROJECT }} - name: Pull translations from Transifex id: pull if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }} run: | # Clean up obsolete files - find ./${{ env.LANGUAGE_DIR }} -name '*.po' -exec rm {} \; + find ./${{ env.PYDOC_LANG_DIR }} -name '*.po' -exec rm {} \; ./scripts/pull_translations.sh env: TX_TOKEN: ${{ secrets.TX_TOKEN }} @@ -103,36 +84,42 @@ jobs: run: | newer_branch=${PYDOC_VERSION%%.*}.$((${PYDOC_VERSION##*.}+1)) git clone --depth 1 --single-branch --branch $newer_branch https://github.com/python/python-docs-pt-br ${newer_branch}-dir - pomerge --from ./${newer_branch}-dir/{**/,}*.po --to ./${{ env.LANGUAGE_DIR }}/{**/,}*.po + pomerge --from ./${newer_branch}-dir/{**/,}*.po --to ./${{ env.PYDOC_LANG_DIR }}/{**/,}*.po rm -rf ./${newer_branch}-dir - name: powrap if: steps.pull.outcome == 'success' run: | - cd ./${{ env.LANGUAGE_DIR }} + cd ./${{ env.PYDOC_LANG_DIR }} powrap *.po **/*.po - name: Update statistics if: always() run: | ./scripts/stats.py - git -C ./${{ env.LANGUAGE_DIR }} diff stats.json + git -C ./${{ env.PYDOC_LANG_DIR }} diff stats.json + env: + PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }} - name: Update potodo.md if: always() run: | ./scripts/potodo.sh - git diff ./${{ env.LANGUAGE_DIR }}/potodo.md + git -C ./${{ env.PYDOC_LANG_DIR }} diff potodo.md + env: + PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }} # 4- Commit and push translations - name: Commit run: ./scripts/commit.sh + env: + PYDOC_LANG_DIR: ${{ env.PYDOC_LANG_DIR }} - name: Push if: ${{ contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name) }} run: | - cd ./${{ env.LANGUAGE_DIR }} + cd ./${{ env.PYDOC_LANG_DIR }} git push diff --git a/scripts/commit.sh b/scripts/commit.sh index 06f265534..a5c5d742d 100755 --- a/scripts/commit.sh +++ b/scripts/commit.sh @@ -5,8 +5,12 @@ set -eu -rootdir=$(realpath $(dirname $0)) -cd $rootdir/../cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES +test -n ${PYDOC_LANGUAGE+x} + +rootdir=$(realpath $(dirname $0)/..) +language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}" + +cd "$language_dir" extra_files=".tx/config stats.json potodo.md" @@ -44,5 +48,5 @@ set -u # Commit only if there is any cached file if ! git diff-index --cached --quiet HEAD; then git add -v $extra_files - git commit -vm "$($rootdir/generate_commit_msg.py)" + git commit -vm "$($rootdir/scripts/generate_commit_msg.py)" fi diff --git a/scripts/generate_txconfig.py b/scripts/generate_txconfig.py new file mode 100755 index 000000000..727f9b5a4 --- /dev/null +++ b/scripts/generate_txconfig.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Generate the .tx/config file based on the existing projects +in Python docs Transifex project. Takes an project slug as +positional argument, e.g. python-newest or python-313 +""" + +import argparse +import re +import subprocess +import sys +from pathlib import Path + +# Replaces required to fix the default values set by 'tx add remote' command. +# Add or remove +TEXT_TO_REPLACE = { + "2_": "2.", + "3_": "3.", + "glossary_": "glossary", + "collections_": "collections.", + "compression_": "compression.", + "concurrent_": "concurrent.", + "curses_": "curses.", + "email_": "email.", + "html_": "html.", + "http_": "http.", + "importlib_resources_": "importlib.resources.", + "importlib_": "importlib.", + "logging_": "logging.", + "multiprocessing_": "multiprocessing.", + "os_": "os.", + "string_": "string.", + "sys_monitoring": "sys.monitoring", + "tkinter_": "tkinter.", + "unittest_": "unittest.", + "urllib_": "urllib.", + "xml_dom_": "xml.dom.", + "xml_etree_": "xml.etree.", + "xmlrpc_": "xmlrpc.", + "xml_sax_": "xml.sax.", + "xml_": "xml.", +} + + +def parse_args(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root-path", + "-p", + default=Path("."), + help="Path to the translation files, and also the .tx/config (defaults to current directory)", + ) + parser.add_argument( + "tx_project", help="Slug of the Transifex project to query resources from" + ) + return parser.parse_args() + + +def reset_tx_config(txconfig: Path): + """Create or reset the .tx/config file with basic header.""" + txconfig.parent.mkdir(exist_ok=True) + txconfig.write_text("[main]\nhost = https://www.transifex.com\n", encoding="utf-8") + print("Initialized .tx/config.") + + +def populate_resources_from_remote(config_file: Path, tx_project: str): + """Add the remote resources from the Transifex project to .tx/config.""" + result = subprocess.run( + [ + "tx", + "--config", + str(config_file), + "add", + "remote", + "--file-filter", + "/.", + f"https://app.transifex.com/python-doc/{tx_project}/", + ], + check=True, + ) + if result.returncode != 0: + print("Failed to add the resources from remote:") + print(result.stderr.strip()) + sys.exit(result.returncode) + print("Added remote resources to Transifex.") + + +def patch_config(txconfig: Path): + """Patch .tx/config to fixing PO filenames to match the expected.""" + content = txconfig.read_text(encoding="utf-8").splitlines() + new_lines = [] + + for line in content: + if line.startswith(("source_file", "source_lang")): + continue + + if line.startswith("file_filter"): + line = line.replace("/", "") + line = line.replace("--", "/") + + for pattern, replacement in TEXT_TO_REPLACE.items(): + if pattern in line: + line = line.replace(pattern, replacement) + break + + new_lines.append(line) + + text = "\n".join(new_lines) + + txconfig.write_text(text + "\n", encoding="utf-8") + print("Updated .tx/config with character substitutions") + + +def main(): + args = parse_args() + config_path = Path(".tx/config") + if args.root_path: + config_path = args.root_path / config_path + reset_tx_config(config_path) + populate_resources_from_remote(config_path, args.tx_project) + patch_config(config_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/potodo.sh b/scripts/potodo.sh index 0d7e1e2c2..6b343e0da 100755 --- a/scripts/potodo.sh +++ b/scripts/potodo.sh @@ -10,8 +10,9 @@ test -n ${PYDOC_VERSION+x} test -n ${PYDOC_LANGUAGE+x} rootdir=$(realpath $(dirname $0)/..) +language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}" -cd "$rootdir"/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES +cd "$language_dir" potodo --no-cache > potodo.md diff --git a/scripts/pull_translations.sh b/scripts/pull_translations.sh index d1fd34b0c..a85d79b69 100755 --- a/scripts/pull_translations.sh +++ b/scripts/pull_translations.sh @@ -20,8 +20,12 @@ set -xeu test -n ${PYDOC_TX_PROJECT+x} +test -n ${PYDOC_LANGUAGE+x} -cd "$(dirname $0)/../cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES/" +rootdir=$(realpath $(dirname $0)/..) +language_dir="${PYDOC_LANG_DIR:-$rootdir/cpython/Doc/locales/${PYDOC_LANGUAGE}/LC_MESSAGES}" + +cd "$language_dir" # If a PO file is provided as input, convert it into Transifex resource # and add it be pulled (instead of pulling all translations files). diff --git a/scripts/stats.py b/scripts/stats.py index 81fe1aed5..4dde51b18 100755 --- a/scripts/stats.py +++ b/scripts/stats.py @@ -17,11 +17,16 @@ def main() -> None: """Main function to generate translation stats.""" - language = os.environ.get("PYDOC_LANGUAGE") - if not language: - raise ValueError("Environment variable PYDOC_LANGUAGE is not set.") - pofiles_path = Path(f"cpython/Doc/locales/{language}/LC_MESSAGES") + lang_dir = os.environ.get("PYDOC_LANG_DIR") + if lang_dir: + pofiles_path = Path(lang_dir) + else: + language = os.environ.get("PYDOC_LANGUAGE") + if not language: + raise ValueError("Environment variable PYDOC_LANGUAGE is not set.") + pofiles_path = Path(f"cpython/Doc/locales/{language}/LC_MESSAGES") + if not pofiles_path.exists(): raise FileNotFoundError(f"Path does not exist: {pofiles_path}")