From 4ae7ab9618eb13a19b39d8f080279337d814d46f Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 30 Sep 2025 13:44:43 +0200 Subject: [PATCH 1/8] CI on Github: enable main workflow on ci/* branches for debugging --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d2fd055..4de8a51e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,9 +1,11 @@ name: Build on: - pull_request: + pull_request: # to master schedule: - cron: "0 0 * * 0" - + push: + branches: # CI debugging + - "ci/**" jobs: build: runs-on: ${{ matrix.os }} From d7888a47f40cfd4d73cb5dbeec9f977f993a3e8c Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 30 Sep 2025 16:43:24 +0200 Subject: [PATCH 2/8] CI on Github: separate linter/test, shared setup * tox.ini: tox-gh-actions integration * tox.ini: lint removed from envlist, default scope is test-only * workflows: test job added (matrixed) * workflows: lint job is separated, no-matrix, non-blocking on failure * workflows: reusable env lifting steps factored out into ./github/actions --- .github/actions/setup/action.yml | 46 ++++++++++++++++++++++++++++++++ .github/workflows/main.yml | 43 +++++++++++++++++------------ tox.ini | 15 +++++++++-- 3 files changed, 85 insertions(+), 19 deletions(-) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..48e761d3 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,46 @@ +name: Setup base (python, pip cache, tox) +inputs: + python: + description: "Python version to use" + required: true + type: string +runs: + using: "composite" + steps: + - name: pip cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + key: ${{ runner.os }}-pip-${{ inputs.python }} + + - name: Cargo cache + uses: actions/cache/@v4 + with: + path: "~/.cargo" + key: ${{ runner.os }}-cargo + + - name: Poetry cache + uses: actions/cache/@v4 + with: + path: "~/.cache/pypoetry" + key: ${{ runner.os }}-poetry-${{ inputs.python }} + restore-keys: | + ${{ runner.os }}-poetry- + + - uses: actions/setup-python@v6 + with: + python-version: ${{ inputs.python }} + + - name: upgrade pip and install tox + shell: bash + run: | + python -m pip -q install --upgrade pip "setuptools==65.6.2" + pip -q install "tox<4" tox-gh-actions + + - name: install Rust and Poetry + shell: bash + run : | + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal + source "$HOME/.cargo/env" + pip -q install poetry>=1.2.0 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4de8a51e..a8d80463 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,27 +7,36 @@ on: branches: # CI debugging - "ci/**" jobs: - build: - runs-on: ${{ matrix.os }} + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + with: + python: 3.12 + - name: Lint + id: lint + run: tox -e lint + continue-on-error: true + - name: Emit warning if lint failed + if: ${{ steps.lint.outcome != 'success' }} + run: echo "::warning::Linter failure suppressed (continue-on-error=true)" + test: strategy: fail-fast: false matrix: os: [ ubuntu-latest ] - python: ["3.12"] - include: - - python: "3.12" - tox_env: "lint" + python: + - "3.12" + #- "3.11" + #- "3.10" + #- "3.9.14" + #- "3.8" + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup with: - python-version: ${{ matrix.python }} - - name: Install tox - run: | - python -m pip install --upgrade pip setuptools - pip install tox + python: ${{ matrix.python }} - name: Test - run: | - tox -e ${{ matrix.tox_env }} - + run: tox diff --git a/tox.ini b/tox.ini index a92da61a..be81c9d2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,19 @@ [tox] isolated_build = true + # The *-web environments test the latest versions of Django and Flask with the full test suite. For # older version of the web frameworks, just run the tests that are specific to them. -envlist = py3{10,11,12}-{django5}, py3{8,9,10,11,12}-{web,django3,django4,flask2,sqlalchemy1},lint + +# Default envlist is only for matrix testing. Linter and vendoring should be called explicitly +envlist = py3{10,11,12}-{django5}, py3{8,9,10,11,12}-{web,django3,django4,flask2,sqlalchemy1} + +[gh-actions] +python = + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 [web-deps] deps= @@ -54,4 +65,4 @@ deps = vendoring commands = poetry run vendoring {posargs:sync} # We don't need the .pyi files vendoring generates - python -c 'from pathlib import Path; all(map(Path.unlink, Path("vendor").rglob("*.pyi")))' \ No newline at end of file + python -c 'from pathlib import Path; all(map(Path.unlink, Path("vendor").rglob("*.pyi")))' From 81228ad08a2acb610ed0739804feec462e224485 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 30 Sep 2025 20:30:21 +0200 Subject: [PATCH 3/8] CI on Github: smoketest --- .github/actions/dockerhub-login/action.yml | 8 ++++++++ .github/actions/setup/action.yml | 5 +++++ .github/workflows/main.yml | 20 ++++++++++++++++++-- ci/run_tests.sh | 4 +++- 4 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 .github/actions/dockerhub-login/action.yml diff --git a/.github/actions/dockerhub-login/action.yml b/.github/actions/dockerhub-login/action.yml new file mode 100644 index 00000000..6b519ce0 --- /dev/null +++ b/.github/actions/dockerhub-login/action.yml @@ -0,0 +1,8 @@ +name: login to Dockerhub (to prevent image pull trottling) +runs: + using: composite + steps: + - name: docker login + run: | + docker login -u "$DOCKERHUB_USERNAME" --password-stdin <<< "$DOCKERHUB_PASSWORD" + shell: bash diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 48e761d3..daba8580 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -4,6 +4,10 @@ inputs: description: "Python version to use" required: true type: string + default: 3.12 +outputs: + 'python-version': + value: ${{ steps.python.outputs.python-version }} runs: using: "composite" steps: @@ -29,6 +33,7 @@ runs: ${{ runner.os }}-poetry- - uses: actions/setup-python@v6 + id: python with: python-version: ${{ inputs.python }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a8d80463..f3cbef94 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,8 +12,6 @@ jobs: steps: - uses: actions/checkout@v5 - uses: ./.github/actions/setup - with: - python: 3.12 - name: Lint id: lint run: tox -e lint @@ -40,3 +38,21 @@ jobs: python: ${{ matrix.python }} - name: Test run: tox + smoketest: + runs-on: ubuntu-latest + needs: [ 'lint','test' ] + steps: + - uses: actions/checkout@v5 + - name: dockerhub login (for seamless docker pulling) + uses: ./.github/actions/dockerhub-login + env: + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + continue-on-error: true + - id: setup + uses: ./.github/actions/setup + + - run: poetry build + - run: ci/run_tests.sh + env: + SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} diff --git a/ci/run_tests.sh b/ci/run_tests.sh index c78d5415..977d9f17 100755 --- a/ci/run_tests.sh +++ b/ci/run_tests.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +SMOKETEST_DOCKER_IMAGE=${SMOKETEST_DOCKER_IMAGE:-"python:3.11"} + set -x t=$([ -t 0 ] && echo 't') docker run -q -i${t} --rm\ @@ -7,4 +9,4 @@ docker run -q -i${t} --rm\ -v $PWD/ci:/ci\ -w /tmp\ -v $PWD/ci/readonly-mount-appmap.log:/tmp/appmap.log:ro\ - python:3.11 bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" + $SMOKETEST_DOCKER_IMAGE bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" From 411e8889ae8103a1777612afb3d0354741cb06c4 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Wed, 1 Oct 2025 01:28:35 +0200 Subject: [PATCH 4/8] CI on Github: release --- .github/actions/dockerhub-login/action.yml | 2 +- .../actions/setup-semantic-release/action.yml | 18 +++++++++++++++ .github/workflows/main.yml | 23 +++++++++++++++++++ .releaserc.yml | 2 +- 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 .github/actions/setup-semantic-release/action.yml diff --git a/.github/actions/dockerhub-login/action.yml b/.github/actions/dockerhub-login/action.yml index 6b519ce0..6e07493b 100644 --- a/.github/actions/dockerhub-login/action.yml +++ b/.github/actions/dockerhub-login/action.yml @@ -4,5 +4,5 @@ runs: steps: - name: docker login run: | - docker login -u "$DOCKERHUB_USERNAME" --password-stdin <<< "$DOCKERHUB_PASSWORD" + docker login -u "$DOCKERHUB_USERNAME" --password-stdin <<< "$DOCKERHUB_PASSWORD" || echo "::warning::docker-login failed, ignoring" shell: bash diff --git a/.github/actions/setup-semantic-release/action.yml b/.github/actions/setup-semantic-release/action.yml new file mode 100644 index 00000000..455ef854 --- /dev/null +++ b/.github/actions/setup-semantic-release/action.yml @@ -0,0 +1,18 @@ +name: setup semantic-release with plugins +runs: + using: composite + steps: + - uses: actions/setup-node@v5 + id: setup-node + - uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-npm-${{ steps.setup-node.node-version }} + - shell: bash + run: | + npm i -g \ + semantic-release \ + @semantic-release/exec \ + @semantic-release/git \ + @semantic-release/changelog \ + @google/semantic-release-replace-plugin diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f3cbef94..a5eff90e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,6 +39,7 @@ jobs: - name: Test run: tox smoketest: + if: false runs-on: ubuntu-latest needs: [ 'lint','test' ] steps: @@ -56,3 +57,25 @@ jobs: - run: ci/run_tests.sh env: SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} + release: + if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') ) + #needs: ['smoketest','lint','test'] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup-semantic-release # node+semantic-release + - uses: ./.github/actions/setup # poetry + - name: configure poetry repos + run: | + poetry config repositories.pypi https://upload.pypi.org/legacy/ + poetry config repositories.testpypi https://test.pypi.org/legacy/ + - run: semantic-release --branches ${{ github.ref_name }} + env: + GIT_AUTHOR_NAME: appland-release + GIT_AUTHOR_EMAIL: release@app.land + GIT_COMMITTER_NAME: appland-release + GIT_COMMITTER_EMAIL: release@app.land + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} + POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} + POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.POETRY_PYPI_TOKEN_TESTPYPI }} diff --git a/.releaserc.yml b/.releaserc.yml index 22f1d3c7..b82a7e45 100644 --- a/.releaserc.yml +++ b/.releaserc.yml @@ -18,4 +18,4 @@ plugins: - CHANGELOG.md - pyproject.toml - - '@semantic-release/exec' - - publishCmd: poetry publish --build + - publishCmd: "poetry publish --build -r <%= process.env.PYPI_PUBLISH_REPO ? process.env.PYPI_PUBLISH_REPO : 'pypi' %>" From 415d0f8a4b9f83cb71480922a98d0b2c37ff59c2 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Wed, 1 Oct 2025 04:45:00 +0200 Subject: [PATCH 5/8] CI on Github: final rules, disabled Travis, remarks --- .github/workflows/main.yml | 44 +++++++++++++++++++++++++------------- .travis.yml | 5 ++++- 2 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a5eff90e..c995e4ed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,11 +1,20 @@ -name: Build +name: CI & Release + +# Requires secrets: +# DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD -- optional (for seamless pulls) +# GITHUB_TOKEN -- implicitly injected +# POETRY_PYPI_TOKEN_PYPI -- Pypi token, required for release +# POETRY_PYPI_TOKEN_TESTPYPI -- testpypi token, for CI debugging + on: - pull_request: # to master - schedule: - - cron: "0 0 * * 0" - push: - branches: # CI debugging - - "ci/**" + # NOTE: Release job includes extra guardrails (see job condition below) + # Without them, only linting and testing run + pull_request: + push: + branches: + - "master" + - "ci/**" # CI debugging branches + #- "v*.*" # optional: support for independent major release branches (e.g., 2.x, 3.x) jobs: lint: runs-on: ubuntu-latest @@ -26,10 +35,10 @@ jobs: os: [ ubuntu-latest ] python: - "3.12" - #- "3.11" - #- "3.10" - #- "3.9.14" - #- "3.8" + - "3.11" + - "3.10" + - "3.9.14" + - "3.8" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 @@ -39,7 +48,6 @@ jobs: - name: Test run: tox smoketest: - if: false runs-on: ubuntu-latest needs: [ 'lint','test' ] steps: @@ -58,8 +66,14 @@ jobs: env: SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} release: - if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') ) - #needs: ['smoketest','lint','test'] + + # NOTE: release is allowed only on selected branches + # NOTE: publishing target (prod or testpypi) is derived as PYPI_PUBLISH_REPO below + # full example: + # if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') || startsWith(github.ref_name,'v') ) + if: github.ref_name == 'master' + + needs: ['smoketest','lint','test'] runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 @@ -76,6 +90,6 @@ jobs: GIT_COMMITTER_NAME: appland-release GIT_COMMITTER_EMAIL: release@app.land GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.POETRY_PYPI_TOKEN_TESTPYPI }} + PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} diff --git a/.travis.yml b/.travis.yml index d54311ce..78a77e56 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,11 @@ python: - "3.9.14" - "3.8" +# Travis CI disabled in October 2025; this file is kept temporary for reference and possibility of rollback; can be deleted safely after migration +if: false + # https://github.com/travis-ci/travis-ci/issues/1147#issuecomment-441393807 -if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/ +#if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/ before_install: | curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal From 61ef6cec3aed1781c361baedf72e160ff12a1a77 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Tue, 11 Nov 2025 21:20:14 +0100 Subject: [PATCH 6/8] chore(ci): refactor workflows, enable github releases and define branch policies under .releaserc, publish to pypi via trusted publishing mechanism --- .github/actions/refetch-artifacts/action.yml | 17 ++ .../actions/setup-semantic-release/action.yml | 1 + .github/workflows/lint-and-test.yml | 41 +++++ .github/workflows/main.yml | 95 ------------ .github/workflows/release.yml | 146 ++++++++++++++++++ .releaserc.yml | 30 +++- ci/run_tests.sh | 12 -- ci/scripts/build_with_poetry.sh | 19 +++ ...tifacts_if_distribution_name_is_altered.sh | 29 ++++ ci/scripts/run_tests.sh | 16 ++ ci/{ => tests/data}/readonly-mount-appmap.log | 0 ci/{ => tests}/smoketest.sh | 10 +- ci/{ => tests}/test_pipenv.sh | 0 ci/{ => tests}/test_poetry.sh | 0 14 files changed, 306 insertions(+), 110 deletions(-) create mode 100644 .github/actions/refetch-artifacts/action.yml create mode 100644 .github/workflows/lint-and-test.yml delete mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/release.yml delete mode 100755 ci/run_tests.sh create mode 100755 ci/scripts/build_with_poetry.sh create mode 100755 ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh create mode 100755 ci/scripts/run_tests.sh rename ci/{ => tests/data}/readonly-mount-appmap.log (100%) rename ci/{ => tests}/smoketest.sh (88%) rename ci/{ => tests}/test_pipenv.sh (100%) rename ci/{ => tests}/test_poetry.sh (100%) diff --git a/.github/actions/refetch-artifacts/action.yml b/.github/actions/refetch-artifacts/action.yml new file mode 100644 index 00000000..82f52567 --- /dev/null +++ b/.github/actions/refetch-artifacts/action.yml @@ -0,0 +1,17 @@ +name: Refetch artifacts +runs: + using: "composite" + steps: + - name: download wheel.zip + uses: actions/download-artifact@v4 + with: + name: wheel + path: ./dist + - name: download sdist.zip + uses: actions/download-artifact@v4 + with: + name: sdist + path: ./dist + - name: inspect + shell: bash + run: ls dist/ diff --git a/.github/actions/setup-semantic-release/action.yml b/.github/actions/setup-semantic-release/action.yml index 455ef854..ceb4e22f 100644 --- a/.github/actions/setup-semantic-release/action.yml +++ b/.github/actions/setup-semantic-release/action.yml @@ -14,5 +14,6 @@ runs: semantic-release \ @semantic-release/exec \ @semantic-release/git \ + @semantic-release/github \ @semantic-release/changelog \ @google/semantic-release-replace-plugin diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml new file mode 100644 index 00000000..e2fe0415 --- /dev/null +++ b/.github/workflows/lint-and-test.yml @@ -0,0 +1,41 @@ +name: Lint and test +on: + pull_request: + push: + branches: + - master + - 'ci/**' # ci testing, pre-releases + #- 'feature/**' + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + - name: Lint + id: lint + run: tox -e lint + continue-on-error: true + - name: Emit warning if lint failed + if: ${{ steps.lint.outcome != 'success' }} + run: echo "::warning::Linter failure suppressed (continue-on-error=true)" + test: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + python: + - "3.12" + - "3.11" + - "3.10" + - "3.9.14" + - "3.8" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + with: + python: ${{ matrix.python }} + - name: Test + run: tox diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index c995e4ed..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: CI & Release - -# Requires secrets: -# DOCKERHUB_USERNAME, DOCKERHUB_PASSWORD -- optional (for seamless pulls) -# GITHUB_TOKEN -- implicitly injected -# POETRY_PYPI_TOKEN_PYPI -- Pypi token, required for release -# POETRY_PYPI_TOKEN_TESTPYPI -- testpypi token, for CI debugging - -on: - # NOTE: Release job includes extra guardrails (see job condition below) - # Without them, only linting and testing run - pull_request: - push: - branches: - - "master" - - "ci/**" # CI debugging branches - #- "v*.*" # optional: support for independent major release branches (e.g., 2.x, 3.x) -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/setup - - name: Lint - id: lint - run: tox -e lint - continue-on-error: true - - name: Emit warning if lint failed - if: ${{ steps.lint.outcome != 'success' }} - run: echo "::warning::Linter failure suppressed (continue-on-error=true)" - test: - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python: - - "3.12" - - "3.11" - - "3.10" - - "3.9.14" - - "3.8" - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/setup - with: - python: ${{ matrix.python }} - - name: Test - run: tox - smoketest: - runs-on: ubuntu-latest - needs: [ 'lint','test' ] - steps: - - uses: actions/checkout@v5 - - name: dockerhub login (for seamless docker pulling) - uses: ./.github/actions/dockerhub-login - env: - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} - continue-on-error: true - - id: setup - uses: ./.github/actions/setup - - - run: poetry build - - run: ci/run_tests.sh - env: - SMOKETEST_DOCKER_IMAGE: python:${{ steps.setup.outputs.python-version }} - release: - - # NOTE: release is allowed only on selected branches - # NOTE: publishing target (prod or testpypi) is derived as PYPI_PUBLISH_REPO below - # full example: - # if: ( github.ref_name == 'master' || startsWith(github.ref_name, 'ci/') || startsWith(github.ref_name,'v') ) - if: github.ref_name == 'master' - - needs: ['smoketest','lint','test'] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/setup-semantic-release # node+semantic-release - - uses: ./.github/actions/setup # poetry - - name: configure poetry repos - run: | - poetry config repositories.pypi https://upload.pypi.org/legacy/ - poetry config repositories.testpypi https://test.pypi.org/legacy/ - - run: semantic-release --branches ${{ github.ref_name }} - env: - GIT_AUTHOR_NAME: appland-release - GIT_AUTHOR_EMAIL: release@app.land - GIT_COMMITTER_NAME: appland-release - GIT_COMMITTER_EMAIL: release@app.land - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }} - POETRY_PYPI_TOKEN_TESTPYPI: ${{ secrets.POETRY_PYPI_TOKEN_TESTPYPI }} - PYPI_PUBLISH_REPO: ${{ github.ref == 'refs/heads/master' && 'pypi' || 'testpypi' }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..2928b1af --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,146 @@ +name: Release + +on: + workflow_run: # would only fire after file is merged to master + workflows: ["Lint and test"] + types: + - completed + branches: + - master + - 'ci/**' # ci testing, pre-releases + #- develop # can emit -dev releases but we do not want to + workflow_dispatch: + inputs: + dry_run: + description: "Run in dry-run mode (no publish)" + required: false + default: "true" + push: # only temporary, until this file lands on master (see above) + branches: + - 'ci/**' + +# MUSTHAVE: Trusted publisher access for both repos. +# NOTE: according to docs, 'test' repo accounts are ephemeral and can be wiped at any time +# NOTE: 'test' accs are not that ephmeperal -- losing access to sandbox account (2FA issue) effectively locked us out of project; good test for workarounds though +# NOTE: as a part of regaining-control scenario we may use distinct project names in pyroject.toml (e.g. appmap-dev, appmap-ng) +env: + DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} + pypi_project: appmap + #testpypi_project: appmap-dev # workaround for lost-access scenario + testpypi_project: appmapcitest + +jobs: + + setup: + runs-on: ubuntu-latest + outputs: + distribution_name: ${{ steps.configure.outputs.distribution_name }} + publish_to: ${{ steps.configure.outputs.publish_to }} + publish_env: ${{ steps.configure.outputs.publish_env }} + steps: + - id: configure + shell: bash + run: | + case "${{ github.ref_name }}" in + ci/*) + echo "publish_env=testpypi" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.testpypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://test.pypi.org/project/${{ env.testpypi_project }}" >> $GITHUB_OUTPUT + ;; + master) + echo "publish_env=pypi" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.pypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://pypi.org/project/${{ env.pypi_project }}" >> $GITHUB_OUTPUT + ;; + *) + echo "publish_env=SKIP" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.pypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://test.pypi.org/project/${{ env.pypi_project }}" >> $GITHUB_OUTPUT + ;; + esac + + release: + runs-on: ubuntu-latest + needs: setup + if: github.event_name == 'workflow_dispatch' || (github.event_name=='push' && startsWith(github.ref_name,'ci/') ) || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.head_branch == 'master' || startsWith(github.event.workflow_run.head_branch, 'ci/') ) ) + permissions: + contents: write + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup-semantic-release # node+semantic-release + - uses: ./.github/actions/setup # poetry + - id: semantic-release # branch policies defined in .releaserc + env: + GIT_AUTHOR_NAME: appland-release + GIT_AUTHOR_EMAIL: release@app.land + GIT_COMMITTER_NAME: appland-release + GIT_COMMITTER_EMAIL: release@app.land + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} + run: | + if [ "$DRY_RUN" = "true" ]; then + semantic-release --dry-run + else + semantic-release + fi + + - name: Upload wheel + if: env.DRY_RUN != 'true' + uses: actions/upload-artifact@v4 + with: + name: wheel + path: dist/*.whl + - name: Upload sdist + if: env.DRY_RUN != 'true' + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + outputs: # not reused in fact + release_tag: ${{ steps.semantic-release.outputs.next_release_tag }} + + smoketest: + runs-on: ubuntu-latest + needs: ['setup', 'release'] + if: github.event.inputs.dry_run!='true' + continue-on-error: ${{ needs.setup.outputs.distribution_name!='appmap' }} # altered names won't work anyway + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/refetch-artifacts + - name: dockerhub login (for seamless docker pulling) + uses: ./.github/actions/dockerhub-login + env: + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + continue-on-error: true + - run: ci/scripts/run_tests.sh + env: + SMOKETEST_DOCKER_IMAGE: python:3.12-slim + DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} + + # as a workaround to ownership issues (lost access to project) + publish: + name: publish package on PyPI + needs: ['setup', 'release','smoketest'] + if: (( github.event.inputs.dry_run != 'true' ) && ( (needs.setup.outputs.publish_env == 'pypi') || (needs.setup.outputs.publish_env == 'testpypi') ) ) + runs-on: ubuntu-latest + environment: + name: ${{ needs.setup.outputs.publish_env }} + url: ${{ needs.setup.outputs.publish_to }} + permissions: + id-token: write + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/refetch-artifacts + + - name: Publish to PyPI + if: needs.setup.outputs.publish_env=='pypi' + uses: pypa/gh-action-pypi-publish@release/v1 + + - name: Publish to TestPyPI + if: needs.setup.outputs.publish_env=='testpypi' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ # trailing slash matters! diff --git a/.releaserc.yml b/.releaserc.yml index b82a7e45..b5efcfc6 100644 --- a/.releaserc.yml +++ b/.releaserc.yml @@ -1,3 +1,18 @@ +# Allowed number of prerelease rules: 1..3 +# While semantic-release allows globs, they must be combined with `prerelease: true` and suffix is derived from name than. It conflicts with PEP440 +# PEP440 version rules (not compatible with SemVer): [N!]N(.N)*[{a|b|rc}N][.postN][.devN] +# Consequences: +# - prerelease branches must be explicitly specified, no asterisks +# - prerelease parameter should be one of: a,b,rc,dev,post +# - translation from SemVer prerelease notation to PEP440 is tone in 'replacements' section +branches: # only branches listed here will create releases + - master + - name: ci/trusted_publishing_test + prerelease: dev + #- name: develop + # prerelease: dev + #- name: feature/* + # prerelease: true # will use branch name as suffix plugins: - '@semantic-release/commit-analyzer' - '@semantic-release/release-notes-generator' @@ -13,9 +28,22 @@ plugins: hasChanged: true numMatches: 1 numReplacements: 1 +- - '@google/semantic-release-replace-plugin' # optional SemVer -> PEP440 coercion + - replacements: + - files: [pyproject.toml] # optional: SemVer prerelease -> PEP440 ("1.2.3-dev.1" -> "1.2.3.dev1") + from: '^version = "(\\d+\\.\\d+\\.\\d+)-(dev|post)\\.(\\d+)"' + to: 'version = "\\1.\\2\\3"' + - files: [pyproject.toml] # optional: SemVer prerelease -> PEP440 ("1.2.3-rc.10" -> "1.2.3rc10" ) + from: '^version = "(\\d+\\.\\d+\\.\\d+)-(a|b|rc)\\.(\\d+)"' + to: 'version = "\\1\\2\\3"' - - '@semantic-release/git' - assets: - CHANGELOG.md - pyproject.toml - - '@semantic-release/exec' - - publishCmd: "poetry publish --build -r <%= process.env.PYPI_PUBLISH_REPO ? process.env.PYPI_PUBLISH_REPO : 'pypi' %>" + - prepareCmd: | + /bin/bash ./ci/scripts/build_with_poetry.sh +- - '@semantic-release/github': + - assets: + - dist/*.whl + - dist/*.tar.gz diff --git a/ci/run_tests.sh b/ci/run_tests.sh deleted file mode 100755 index 977d9f17..00000000 --- a/ci/run_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -SMOKETEST_DOCKER_IMAGE=${SMOKETEST_DOCKER_IMAGE:-"python:3.11"} - -set -x -t=$([ -t 0 ] && echo 't') -docker run -q -i${t} --rm\ - -v $PWD/dist:/dist -v $PWD/_appmap/test/data/unittest:/_appmap/test/data/unittest\ - -v $PWD/ci:/ci\ - -w /tmp\ - -v $PWD/ci/readonly-mount-appmap.log:/tmp/appmap.log:ro\ - $SMOKETEST_DOCKER_IMAGE bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" diff --git a/ci/scripts/build_with_poetry.sh b/ci/scripts/build_with_poetry.sh new file mode 100755 index 00000000..53c89791 --- /dev/null +++ b/ci/scripts/build_with_poetry.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +set -o pipefail + +if [ -z "$DISTRIBUTION_NAME" ] || [ "$DISTRIBUTION_NAME" = "appmap" ] ; then + exec poetry build $* +fi + +echo "Altering distribution name to $DISTRIBUTION_NAME" + +cp -v pyproject.toml /tmp/pyproject.bak +sed -i -e "s/^name = \".*\"/name = \"${DISTRIBUTION_NAME}\"/" pyproject.toml +grep -n 'name = "' pyproject.toml + +poetry build $* + +echo "Not patching artifacts with Provides-Dist, they won't work anyway (this flow is solely for publishing test)" +cp -v /tmp/pyproject.bak pyproject.toml diff --git a/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh b/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh new file mode 100755 index 00000000..acee04ff --- /dev/null +++ b/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +set -o pipefail + +artifacts=$* +injection_string="Provides-Dist: appmap" +if [ -n "$artifacts" ] && [ -n "$DISTRIBUTION_NAME" ] && [ "$DISTRIBUTION_NAME" != "appmap" ]; then + echo "Altered distribution name detected, injecting '$injection_string' into artifacts: $artifacts" + for artifact in $artifacts ; do + TMP=$(mktemp -d) + ARTIFACT_PATH="$(realpath ${artifact})" + if [[ $artifact == *.whl ]]; then + unzip -q "$ARTIFACT_PATH" -d "$TMP" + DISTINFO=$(find "$TMP" -type d -name "*.dist-info") + echo "$injection_string" >> "$DISTINFO/METADATA" + (cd "$TMP" && zip -qr "$ARTIFACT_PATH" .) + else + tar -xzf "$ARTIFACT_PATH" -C "$TMP" + PKG_INFO_FILE=$(find "$TMP" -type f -name "PKG-INFO") + echo "$injection_string" >> "$PKG_INFO_FILE" + + # Get the top-level directory to repack correctly + PKGDIR=$(find "$TMP" -mindepth 1 -maxdepth 1 -type d) + (cd "$TMP" && tar -czf "$ARTIFACT_PATH" "$(basename "$PKGDIR")") + fi + echo "($injection_string): patched $ARTIFACT_PATH" + rm -rf "$TMP" + done +fi diff --git a/ci/scripts/run_tests.sh b/ci/scripts/run_tests.sh new file mode 100755 index 00000000..d780a187 --- /dev/null +++ b/ci/scripts/run_tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +SMOKETEST_DOCKER_IMAGE=${SMOKETEST_DOCKER_IMAGE:-"python:3.11"} +DISTRIBUTION_NAME=${DISTRIBUTION_NAME:-appmap} + +set -x +t=$([ -t 0 ] && echo 't') +docker run -q -i${t} --rm \ + -v $PWD/dist:/dist \ + -v $PWD/_appmap/test/data/unittest:/_appmap/test/data/unittest\ + -v $PWD/ci/tests:/ci/tests\ + -v $PWD/.git:/tmp/.git:ro\ + -v $PWD/ci/tests/data/readonly-mount-appmap.log:/tmp/appmap.log:ro\ + -w /tmp\ + -e DISTRIBUTION_NAME \ + $SMOKETEST_DOCKER_IMAGE bash -ce "${@:-/ci/tests/smoketest.sh; /ci/tests/test_pipenv.sh; /ci/tests/test_poetry.sh}" diff --git a/ci/readonly-mount-appmap.log b/ci/tests/data/readonly-mount-appmap.log similarity index 100% rename from ci/readonly-mount-appmap.log rename to ci/tests/data/readonly-mount-appmap.log diff --git a/ci/smoketest.sh b/ci/tests/smoketest.sh similarity index 88% rename from ci/smoketest.sh rename to ci/tests/smoketest.sh index 3d0eeb13..cd0abade 100755 --- a/ci/smoketest.sh +++ b/ci/tests/smoketest.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash + test_recording_when_appmap_not_true() { cat < test_client.py @@ -36,8 +37,13 @@ EOF } set -ex + +# now appmap requires git +apt-get update -qq \ + && apt-get install -y --no-install-recommends git + pip -q install -U pip pytest "flask>=2,<3" python-decouple -pip -q install /dist/appmap-*-py3-none-any.whl +pip -q install /dist/${DISTRIBUTION_NAME//-/_}-*-py3-none-any.whl cp -R /_appmap/test/data/unittest/simple ./. @@ -66,4 +72,4 @@ else exit 1 fi -test_log_file_not_writable \ No newline at end of file +test_log_file_not_writable diff --git a/ci/test_pipenv.sh b/ci/tests/test_pipenv.sh similarity index 100% rename from ci/test_pipenv.sh rename to ci/tests/test_pipenv.sh diff --git a/ci/test_poetry.sh b/ci/tests/test_poetry.sh similarity index 100% rename from ci/test_poetry.sh rename to ci/tests/test_poetry.sh From 1541f032e165e891c4bb9f14daa277c7c13cab73 Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Mon, 24 Nov 2025 19:40:16 +0100 Subject: [PATCH 7/8] chore(ci): remove travis.yml --- .travis.yml | 63 ----------------------------------------------------- 1 file changed, 63 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 78a77e56..00000000 --- a/.travis.yml +++ /dev/null @@ -1,63 +0,0 @@ -os: linux -dist: jammy -language: python -python: -- "3.12" -- "3.11" -- "3.10" -- "3.9.14" -- "3.8" - -# Travis CI disabled in October 2025; this file is kept temporary for reference and possibility of rollback; can be deleted safely after migration -if: false - -# https://github.com/travis-ci/travis-ci/issues/1147#issuecomment-441393807 -#if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/ - -before_install: | - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal - source "$HOME/.cargo/env" - pip -q install --upgrade pip 'setuptools==65.6.2' 'poetry>=1.2.0' - -install: pip -q install --upgrade "tox < 4" tox-travis -script: tox - -cache: - cargo: true - pip: true - directories: - - $TRAVIS_BUILD_DIR/.tox/ - - $HOME/.cache/pypoetry - -jobs: - include: - - stage: smoke test - services: - - docker - script: - - pip -q install poetry - - poetry build - - echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - - ci/run_tests.sh - - stage: release - if: branch = master - script: skip - before_deploy: - - pip -q install poetry - - nvm install lts/* - - npm i -g - semantic-release - @semantic-release/exec - @semantic-release/git - @semantic-release/changelog - @google/semantic-release-replace-plugin - # Note publishing this way requires the PyPI credentials to be - # present in the environment. Travis doesn't currently support - # providing environment variables to deploy providers through - # the build config (i.e. in this file). So, they must be - # provided through the build settings instead. - deploy: - - provider: script - script: semantic-release - on: - branch: master From d904e7e8c108658a60ec036d2df9996ac45f7b7c Mon Sep 17 00:00:00 2001 From: Hleb Rubanau Date: Mon, 1 Dec 2025 16:10:07 +0100 Subject: [PATCH 8/8] chore(ci): add validation job to Lint-and-Test workflow, as umbrella/rollup check of matrix tests status --- .github/workflows/lint-and-test.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml index e2fe0415..f0ad33d8 100644 --- a/.github/workflows/lint-and-test.yml +++ b/.github/workflows/lint-and-test.yml @@ -39,3 +39,17 @@ jobs: python: ${{ matrix.python }} - name: Test run: tox + validation: + name: Validation + runs-on: ubuntu-latest + needs: [test] + if: always() + steps: + - name: Validate matrix test success + run: | + # Check the status of the 'test' job (which includes all matrix variations) + if [ "${{ needs.test.result }}" != "success" ]; then + echo "One or more matrix test jobs failed." + exit 1 + fi + echo "All matrix test jobs passed."