Migrate pdf-doc build to meson

.github/workflows/doc-build-pdf.yml

@@ -2,148 +2,93 @@ name: Build documentation (PDF)
 
 on:
   pull_request:
+  merge_group:
   push:
+    tags:
+      # Match all release tags including beta, rc
+      - '[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+'
+      - '[0-9]+.[0-9]+.beta[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+.beta[0-9]+'
+      - '[0-9]+.[0-9]+.rc[0-9]+'
+      - '[0-9]+.[0-9]+.[0-9]+.rc[0-9]+'
+    branches:
+      - develop
   workflow_dispatch:
     # Allow to run manually
-    inputs:
-      platform:
-        description: 'Platform'
-        required: true
-        default: 'ubuntu-noble-standard'
-      docker_tag:
-        description: 'Docker tag'
-        required: true
-        default: 'dev'
 
 concurrency:
   # Cancel previous runs of this workflow for the same branch
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-  # Same as in build.yml
-  TOX_ENV:                "docker-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-incremental"
-  BUILD_IMAGE:            "localhost:5000/${{ github.repository }}/sage-${{ github.event.inputs.platform || 'ubuntu-noble-standard' }}-with-targets:ci"
-  FROM_DOCKER_REPOSITORY: "ghcr.io/sagemath/sage/"
-  FROM_DOCKER_TARGET:     "with-targets"
-  FROM_DOCKER_TAG:        ${{ github.event.inputs.docker_tag || 'dev'}}
-  EXTRA_CONFIGURE_ARGS:   --enable-fat-binary
+  PYTHON_VERSION: 3.11
 
 jobs:
-  build-doc-pdf:
+  doc-pdf:
     runs-on: ubuntu-latest
-    services:
-      # https://docs.docker.com/build/ci/github-actions/local-registry/
-      registry:
-        image: registry:2
-        ports:
-          - 5000:5000
     steps:
-      - name: Maximize build disk space
-        uses: easimon/maximize-build-space@v10
-        with:
-          # need space in /var for Docker images
-          root-reserve-mb:      30000
-          remove-dotnet:        true
-          remove-android:       true
-          remove-haskell:       true
-          remove-codeql:        true
-          remove-docker-images: true
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Install test prerequisites
-        # From docker.yml
-        run: |
-          sudo DEBIAN_FRONTEND=noninteractive apt-get update
-          sudo DEBIAN_FRONTEND=noninteractive apt-get install tox
-          sudo apt-get clean
-          df -h
+
       - name: Merge CI fixes from sagemath/sage
         run: |
-          mkdir -p upstream
-          .github/workflows/merge-fixes.sh 2>&1 | tee upstream/ci_fixes.log
+          .github/workflows/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
-          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
-
-      # Building
-
-      - name: Generate Dockerfile
-        # From docker.yml
-        run: |
-          tox -e ${{ env.TOX_ENV }}
-          cp .tox/${{ env.TOX_ENV }}/Dockerfile .
-        env:
-          # Only generate the Dockerfile, do not run 'docker build' here
-          DOCKER_TARGETS: ""
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+      - name: Cache conda packages
+        uses: actions/cache@v4
         with:
-          driver-opts: network=host
-
-      - name: Build Docker image
-        id: image
-        uses: docker/build-push-action@v6
+          path: ~/conda_pkgs_dir
+          key:
+            ${{ runner.os }}-conda-${{ hashFiles('environment-${{ env.PYTHON_VERSION }}-linux.yml') }}
+
+      - name: Compiler cache
+        uses: hendrikmuhs/ccache-action@v1.2
         with:
-          # push and load may not be set together at the moment
-          push:       true
-          load:       false
-          context:    .
-          tags:       ${{ env.BUILD_IMAGE }}
-          target:     with-targets
-          cache-from: type=gha
-          cache-to:   type=gha,mode=max
-          build-args: |
-            NUMPROC=6
-            USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
-            TARGETS_PRE=build/make/Makefile
-            TARGETS=ci-build-with-fallback
-
-      - name: Start container
-        id: container
-        # Try to continue when "exporting to GitHub Actions Cache" failed with timeout
-        if: (success() || failure())
-        run: |
-          docker run --name BUILD -dit \
-                     --mount type=bind,src=$(pwd),dst=$(pwd) \
-                     --workdir $(pwd) \
-                     ${{ env.BUILD_IMAGE }} /bin/sh
+          key: ${{ runner.os }}-meson-${{ env.PYTHON_VERSION }}
 
-      #
-      # On PRs and pushes to branches
-      #
-
-      - name: Update system packages
-        id: packages
-        if: (success() || failure()) && steps.container.outcome == 'success'
+      - name: Setup Conda environment
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          # Disabled for now due to
+          # https://github.com/conda-incubator/setup-miniconda/issues/379
+          # miniforge-version: latest
+          use-mamba: true
+          channels: conda-forge
+          channel-priority: true
+          activate-environment: sage-dev
+          environment-file: environment-${{ env.PYTHON_VERSION }}-linux.yml
+
+      - name: Build Sage
+        shell: bash -l {0}
         run: |
-          export PATH="build/bin:$PATH"
-          eval $(sage-print-system-package-command auto update)
-          eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip)
-          eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive texlive_luatex free_fonts xindy)
-        shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0}
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+          export CC="ccache $CC"
+          export CXX="ccache $CXX"
+          pip install --no-build-isolation --config-settings=builddir=builddir . -v
 
-      - name: Build doc (PDF)
-        id: docbuild
-        if: (success() || failure()) && steps.packages.outcome == 'success'
+      - name: Build documentation
+        shell: bash -l {0}
         run: |
-          export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
-          make doc-clean doc-uninstall; make sagemath_doc_html-build-deps sagemath_doc_pdf-no-deps
-        shell: sh .github/workflows/docker-exec-script.sh BUILD /sage {0}
+          sudo DEBIAN_FRONTEND=noninteractive sudo apt-get update
+          sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian texlive texlive_luatex free_fonts xindy)
+          meson compile -C builddir doc-pdf
+        env:
+          SAGE_DOCBUILD_OPTS: "--include-tests-blocks"
 
       - name: Copy doc
         id: copy
-        if: (success() || failure()) && steps.docbuild.outcome == 'success'
         run: |
           mkdir -p ./doc
-          # We copy everything to a local folder
-          docker cp BUILD:/sage/local/share/doc/sage/pdf doc
+          cp -r builddir/src/doc/pdf doc/
           # Zip everything for increased performance
           zip -r doc-pdf.zip doc
 
       - name: Upload doc
-        if: (success() || failure()) && steps.copy.outcome == 'success'
         uses: actions/upload-artifact@v4
         with:
           name: doc-pdf

.github/workflows/doc-build.yml

@@ -26,7 +26,7 @@ env:
   PYTHON_VERSION: 3.11
 
 jobs:
-  build-doc:
+  doc-html:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

pyproject.toml

@@ -205,6 +205,13 @@ extra = [
   "pkg:generic/qepcad",
   "pkg:generic/tides",
 ]
+# Uncommented for now as most devs probably don't need to run it locally
+# docs-pdf = [
+#   "pkg:generic/latexmk",
+#   "pkg:generic/lualatex",
+#   "pkg:fonts/freefonts",
+#   "pkg:generic/xindy",
+# ]
 
 [dependency-groups]
 test = ["pytest", "pytest-xdist", "coverage"]

src/sage_docbuild/builders.py

@@ -66,6 +66,7 @@
 import os
 import pickle
 import re
+import shlex
 import shutil
 import subprocess
 import sys
@@ -174,10 +175,10 @@
 
         EXAMPLES::
 
             sage: from sage_docbuild.builders import DocBuilder
             sage: from sage_docbuild.build_options import BuildOptions
             sage: import tempfile
             sage: with tempfile.TemporaryDirectory() as directory:
             ....:   options = BuildOptions(output_dir=Path(directory), source_dir=Path('src/doc'))
             ....:   builder = DocBuilder('en/tutorial', options)
             ....:   builder._output_dir('html')
@@ -196,10 +197,10 @@
 
         EXAMPLES::
 
             sage: from sage_docbuild.builders import DocBuilder
             sage: from sage_docbuild.build_options import BuildOptions
             sage: import tempfile
             sage: with tempfile.TemporaryDirectory() as directory:
             ....:   options = BuildOptions(output_dir=Path(directory), source_dir=Path('src/doc'))
             ....:   builder = DocBuilder('en/tutorial', options)
             ....:   builder._doctrees_dir()
@@ -215,10 +216,10 @@
 
         EXAMPLES::
 
             sage: from sage_docbuild.builders import DocBuilder
             sage: from sage_docbuild.build_options import BuildOptions
             sage: options = BuildOptions(source_dir=Path('src/doc'))
             sage: builder = DocBuilder('tutorial', options)
             sage: builder._output_formats()
             ['changes', 'html', 'htmlhelp', 'inventory', 'json', 'latex', 'linkcheck', 'pickle', 'web']
         """
@@ -265,13 +266,37 @@
             with open(tex_file, 'w') as f:
                 f.write(ref)
 
-        make_target = "cd '%s' && $MAKE %s && mv -f *.pdf '%s'"
-        error_message = "failed to run $MAKE %s in %s"
-        command = 'all-pdf'
+        make_cmd = os.environ.get('MAKE', 'make')
+        command = shlex.split(make_cmd) + ['all-pdf']
+        logger.debug(f"Running {' '.join(command)} in {tex_dir}")
+
+        proc = subprocess.run(
+            command,
+            check=False, cwd=tex_dir,
+            capture_output=True,
+            text=True,
+        )
+
+        if proc.returncode != 0:
+            logger.error(f"stdout from {make_cmd}:\n{proc.stdout}")
+            logger.error(f"stderr from {make_cmd}:\n{proc.stderr}")
+            raise RuntimeError(f"failed to run {' '.join(command)} in {tex_dir}")
+
+        if proc.stdout:
+            logger.debug(f"make stdout:\n{proc.stdout}")
+        if proc.stderr:
+            # Still surface stderr even on success, but at debug level
+            logger.debug(f"make stderr:\n{proc.stderr}")
+
+        # Move generated PDFs
+        for pdf in tex_dir.glob("*.pdf"):
+            try:
+                shutil.move(str(pdf), pdf_dir)
+            except Exception as e:
+                logger.error(f"Failed moving {pdf} to {pdf_dir}: {e}")
+                raise
 
-        if subprocess.call(make_target % (tex_dir, command, pdf_dir), close_fds=False, shell=True):
-            raise RuntimeError(error_message % (command, tex_dir))
-        logger.warning(f"Build finished. The built documents can be found in {pdf_dir}.")
+        logger.info(f"Build finished. The built documents can be found in {pdf_dir}.")
 
     def clean(self, *args):
         shutil.rmtree(self._doctrees_dir())

tools/update-conda.py

@@ -168,6 +168,7 @@ def get_dependencies(pyproject_toml: Path, python: str, platform: str) -> set[st
         .replace("symengine", "python-symengine")
         .replace("memory_allocator", "memory-allocator")
         .replace("pkg:generic/r-lattice", "r-lattice")
+        .replace("pkg:generic/latexmk", "latexmk")
         for req in all_requirements
     }
     # Exclude requirements not available on conda (for a given platform)