From cc28349d068fb0f12dbd5bc4d5661b6387f48b6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Villemain?= Date: Thu, 28 Sep 2023 22:45:58 +0200 Subject: [PATCH] GHA & Rework docs Makefiles and related Dockerfiles the main point here is to allow faster builds and regression testing. So instead of building container image with pgaf source code included we build more specialized images without source code. We then pass USE_DOCKER=1 to make in order to trigger building stages in docker, like this: ``` cd docs make USE_DOCKER=1 all ``` Also split the GHA workflow to select only jobs relevant given files changes. For PR: * no ci for docs changes * no docs for !docs changes * no latex for docs changes * latex only for tikz changes * linting alwauys For push: all run, no exception --- .github/workflows/{run-tests.yml => ci.yml} | 36 ++------------ .github/workflows/docs.yml | 42 ++++++++++++++++ .github/workflows/latex-pdftocairo.yml | 41 ++++++++++++++++ .github/workflows/linting.yml | 39 +++++++++++++++ .gitignore | 3 ++ docs/Dockerfile | 28 +++++++++++ docs/Makefile | 49 ++++++++++++++++--- docs/conf.py | 2 +- docs/requirements.txt | 4 +- docs/tikz/Dockerfile | 35 ++++++++++++++ docs/tikz/Makefile | 53 ++++++++++++++++----- 11 files changed, 280 insertions(+), 52 deletions(-) rename .github/workflows/{run-tests.yml => ci.yml} (54%) create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/latex-pdftocairo.yml create mode 100644 .github/workflows/linting.yml create mode 100644 docs/Dockerfile create mode 100644 docs/tikz/Dockerfile diff --git a/.github/workflows/run-tests.yml b/.github/workflows/ci.yml similarity index 54% rename from .github/workflows/run-tests.yml rename to .github/workflows/ci.yml index 25f0f0a04..9d55094d8 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/ci.yml @@ -1,13 +1,14 @@ -name: Run Tests +name: Continuous Integration on: - push: + pull_request: branches: - main - pull_request: + paths-ignore: + - 'docs/**' + push: branches: - main - workflow_dispatch: jobs: @@ -33,8 +34,6 @@ jobs: include: - PGVERSION: 14 TEST: tablespaces - - PGVERSION: 14 - TEST: linting steps: - name: Checkout repository uses: actions/checkout@v3 @@ -43,38 +42,13 @@ jobs: run: | echo "PGVERSION=${{ matrix.PGVERSION }}" >> $GITHUB_ENV echo "TEST=${{ matrix.TEST }}" >> $GITHUB_ENV - echo "LINTING=${{ matrix.LINTING }}" >> $GITHUB_ENV echo "TRAVIS_BUILD_DIR=$(pwd)" >> $GITHUB_ENV - - name: Clone and install linting tools - if: ${{ env.TEST == 'linting' }} - run: | - sudo apt-get install python3-pip - pip3 install --user black - black --version - gcc --version - git clone -b v0.8.19 --depth 1 https://github.com/citusdata/tools.git ../tools - sudo make -C ../tools install - install_uncrustify - rm -rf uncrustify* - - - name: Check code formatting and banned function - if: ${{ env.TEST == 'linting' }} - run: | - make lint - - - name: Build documentation - if: ${{ env.TEST == 'linting' }} - run: | - make build-docs - - name: Build Docker Test Image - if: ${{ env.TEST != 'linting' }} run: | make build-test-image - name: Run Test - if: ${{ env.TEST != 'linting' }} timeout-minutes: 15 run: | make ci-test diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..bcfcba618 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,42 @@ +name: Build docs + +env: + context: /docs + sphinx_image: hapostgres/sphinx-citus:latest + +on: + pull_request: + branches: + - main + paths: + - 'docs/**' + - '!docs/tikz/**' + push: + branches: + - main + workflow_dispatch: + +jobs: + docs: + name: Build sphinx html + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build hapostgres/sphinx-citus docker image + uses: docker/build-push-action@v5 + with: + context: ".${{ env.context }}" + load: true + tags: ${{ env.sphinx_image }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build docs with sphinx + run: "docker run --rm -v .${{ env.context }}:/docs ${{ env.sphinx_image }} make html" diff --git a/.github/workflows/latex-pdftocairo.yml b/.github/workflows/latex-pdftocairo.yml new file mode 100644 index 000000000..19a5cf449 --- /dev/null +++ b/.github/workflows/latex-pdftocairo.yml @@ -0,0 +1,41 @@ +name: Build pictures + +env: + context: /docs/tikz + tikz_image: hapostgres/latexmk-pdftocairo:latest + +on: + pull_request: + branches: + - main + paths: + - 'docs/tikz/**' + push: + branches: + - main + workflow_dispatch: + +jobs: + pictures: + name: Building latexmk/pdftocairo pictures + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build hapostgres/latexmk-pdftocairo docker image + uses: docker/build-push-action@v5 + with: + context: ".${{ env.context }}" + load: true + tags: ${{ env.tikz_image }} + cache-from: type=gha + cache-to: type=gha,mode=min + + - name: Build pdf/png/svg files + run: "docker run --rm -v .${{ env.context }}:/tikz ${{ env.tikz_image }} make all" diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 000000000..b8e86b7db --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,39 @@ +name: Code Linting/Checking + +on: + pull_request: + branches: + - main + push: + branches: + - main + workflow_dispatch: + +jobs: + linting: + name: Run Linting and friends + runs-on: ubuntu-latest + strategy: + fail-fast: true + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set environment variables + run: | + echo "TRAVIS_BUILD_DIR=$(pwd)" >> $GITHUB_ENV + + - name: Clone and install linting tools + run: | + sudo apt-get install python3-pip + pip3 install --user black + black --version + gcc --version + git clone -b v0.8.19 --depth 1 https://github.com/citusdata/tools.git ../tools + sudo make -C ../tools install + install_uncrustify + rm -rf uncrustify* + + - name: Check code formatting and banned function + run: | + make lint diff --git a/.gitignore b/.gitignore index 3d0a5f4fa..26cedb2f5 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,6 @@ docs/tikz/*.png # Exclude our demo/test tmux directory tmux/ valgrind/ + +# Exclude docker ID files +docker.id* diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 000000000..4ded14764 --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,28 @@ +ARG DEBIAN_TAG="bullseye-slim" + +FROM debian:${DEBIAN_TAG} as python-slim + +ARG DEBIAN_TAG + +LABEL org.opencontainers.image.authors = "Data Bene " +LABEL org.opencontainers.image.source = "https://github.com/hapostgres/pg_auto_failover/tree/main/docs/Dockerfile" +LABEL org.opencontainers.image.version = "${DEBIAN_TAG}" +LABEL org.opencontainers.image.licenses = "PostgreSQL License" +LABEL org.opencontainers.image.description = "A better python3 slim with pip" + +RUN apt-get update \ + && apt-get install --no-install-recommends --yes \ + make \ + python3-minimal \ + python3-pip \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +FROM python-slim + +WORKDIR /docs +COPY requirements.txt . +RUN python3 -m pip install --upgrade --no-cache-dir pip install -r requirements.txt + +CMD ["python3", "--help"] diff --git a/docs/Makefile b/docs/Makefile index 9e5b49e51..472318f09 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,5 +1,4 @@ -# Minimal makefile for Sphinx documentation -# +.DEFAULT_GOAL := all # You can set these variables from the command line. SPHINXOPTS = @@ -8,13 +7,51 @@ SPHINXPROJ = pg_auto_failover SOURCEDIR = . BUILDDIR = _build -# Put it first so that "make" without argument is like "make help". +DEPS = $(wildcard *.rst) $(wildcard ref/*.rst) + +# If USE_DOCKER variable is defined, we're working with docker and use the tools +# from image/container instead of current OS. +ifdef USE_DOCKER +SPHINX_IMAGE = hapostgres/sphinx-citus:latest +endif + +.PHONY: all +all: tikz html + +.PHONY: clean +clean: + rm -f docker.id + rm -rf _build _html + $(MAKE) -C tikz $@ + +.PHONY: maintainer-clean +maintainer-clean: clean + $(MAKE) -C tikz $@ +ifdef USE_DOCKER + docker image rm $(SPHINX_IMAGE) +endif + +ifdef USE_DOCKER +SPHINXBUILD = docker run --rm -u $(shell id -u):$(shell id -g) -v $(SOURCEDIR)/:/docs $(SPHINX_IMAGE) sphinx-build +docker.id: Dockerfile requirements.txt + docker build --iidfile docker.id --file $< -t $(SPHINX_IMAGE) . +else +docker.id: ; +endif + +.PHONY: help help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: help Makefile +.PHONY: html +html: $(BUILDDIR)/html + +.PHONY: tikz +tikz: + $(MAKE) -C tikz all # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file +$(BUILDDIR)/html: $(DEPS) docker.id + echo $? + $(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/conf.py b/docs/conf.py index e477b5bb2..e85abbda8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -99,7 +99,7 @@ def __init__(self, **options): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = "sphinx_rtd_theme" +html_theme = "sphinx_rtd_theme_citus" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/requirements.txt b/docs/requirements.txt index 51a5ae0bb..8f433b873 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -Sphinx==4.0.2 +Sphinx==5.3.0 sphinx_rtd_theme_citus==0.5.25 docutils==0.16 -readthedocs-sphinx-search==0.1.0 +readthedocs-sphinx-search==0.3.1 diff --git a/docs/tikz/Dockerfile b/docs/tikz/Dockerfile new file mode 100644 index 000000000..6fe223267 --- /dev/null +++ b/docs/tikz/Dockerfile @@ -0,0 +1,35 @@ +ARG DEBIAN_TAG="bullseye-slim" + +FROM debian:${DEBIAN_TAG} as luatex + +ARG DEBIAN_TAG + +LABEL org.opencontainers.image.authors = "Data Bene " +LABEL org.opencontainers.image.source = "https://github.com/hapostgres/pg_auto_failover/tree/main/docs/tikz/Dockerfile" +LABEL org.opencontainers.image.version = "${DEBIAN_TAG}" +LABEL org.opencontainers.image.licenses = "PostgreSQL License" +LABEL org.opencontainers.image.description = "Docs builder with latexmk and pdftocairo" + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + make \ + latexmk \ + poppler-utils \ + texlive \ + texlive-luatex \ + texlive-latex-extra \ + texlive-fonts-extra \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +FROM luatex + +WORKDIR /tikz + +ENV TERM=dumb +ENV TEXMFHOME = /.cache/ +ENV TEXMFVAR = /.cache/texmf-var/ +ENV TEXMFCONFIG = $TEXMFSYSCONFIG + +RUN luaotfload-tool --update diff --git a/docs/tikz/Makefile b/docs/tikz/Makefile index 84567c935..65b4f383e 100644 --- a/docs/tikz/Makefile +++ b/docs/tikz/Makefile @@ -1,28 +1,57 @@ +.DEFAULT_GOAL := all + SRC = $(wildcard arch*.tex fsm.tex) PDF = $(SRC:.tex=.pdf) SVG = $(SRC:.tex=.svg) PNG = $(SRC:.tex=.png) -all: pdf svg png ; +# If USE_DOCKER variable is defined, we're working with docker and use the tools +# from image/container instead of current OS. +ifdef USE_DOCKER +TIKZ_IMAGE = hapostgres/latexmk-pdftocairo:latest +endif + +.PHONY: all +all: pdf png svg ; +.PHONY: pdf pdf: $(SRC) $(PDF) ; -svg: $(SRC) $(SVG) ; +.PHONY: png png: $(SRC) $(PNG) ; +.PHONY: svg +svg: $(SRC) $(SVG) ; +.PHONY: clean clean: - latexmk -C + $(LATEXMK) -C rm -rf $(PDF) - rm -rf $(SVG) rm -rf $(PNG) + rm -rf $(SVG) + rm -f docker.id -%.pdf: %.tex common.tex - latexmk -silent -lualatex --interaction=nonstopmode -shell-escape $< - latexmk -c +.PHONY: maintainer-clean +maintainer-clean: clean +ifdef USE_DOCKER + docker image rm $(TIKZ_IMAGE) +endif -%.png: %.pdf - pdftocairo -singlefile -r 300 -transp -png $< +ifdef USE_DOCKER +LATEXMK = docker run --rm -u $(shell id -u):$(shell id -g) -v ./:/tikz $(TIKZ_IMAGE) latexmk +PDFTOCAIRO = docker run --rm -u $(shell id -u):$(shell id -g) -v ./:/tikz $(TIKZ_IMAGE) pdftocairo +docker.id: Dockerfile + docker build --iidfile docker.id --file $< -t $(TIKZ_IMAGE) . +else +LATEXMK = latexmk +PDFTOCAIRO = pdftocairo +docker.id: ; +endif -%.svg: %.pdf - pdftocairo -svg $< +%.pdf: %.tex common.tex docker.id + $(LATEXMK) -silent -lualatex --interaction=nonstopmode -shell-escape $< + $(LATEXMK) -c -.PHONY: clean +%.png: %.pdf docker.id + $(PDFTOCAIRO) -singlefile -r 300 -transp -png $< + +%.svg: %.pdf docker.id + $(PDFTOCAIRO) -svg $<