diff --git a/.github/actions/dockerhub-login/action.yml b/.github/actions/dockerhub-login/action.yml new file mode 100644 index 00000000..6e07493b --- /dev/null +++ b/.github/actions/dockerhub-login/action.yml @@ -0,0 +1,8 @@ +name: login to Dockerhub (to prevent image pull trottling) +runs: + using: composite + steps: + - name: docker login + run: | + docker login -u "$DOCKERHUB_USERNAME" --password-stdin <<< "$DOCKERHUB_PASSWORD" || echo "::warning::docker-login failed, ignoring" + shell: bash diff --git a/.github/actions/refetch-artifacts/action.yml b/.github/actions/refetch-artifacts/action.yml new file mode 100644 index 00000000..82f52567 --- /dev/null +++ b/.github/actions/refetch-artifacts/action.yml @@ -0,0 +1,17 @@ +name: Refetch artifacts +runs: + using: "composite" + steps: + - name: download wheel.zip + uses: actions/download-artifact@v4 + with: + name: wheel + path: ./dist + - name: download sdist.zip + uses: actions/download-artifact@v4 + with: + name: sdist + path: ./dist + - name: inspect + shell: bash + run: ls dist/ diff --git a/.github/actions/setup-semantic-release/action.yml b/.github/actions/setup-semantic-release/action.yml new file mode 100644 index 00000000..ceb4e22f --- /dev/null +++ b/.github/actions/setup-semantic-release/action.yml @@ -0,0 +1,19 @@ +name: setup semantic-release with plugins +runs: + using: composite + steps: + - uses: actions/setup-node@v5 + id: setup-node + - uses: actions/cache@v4 + with: + path: ~/.npm + key: ${{ runner.os }}-npm-${{ steps.setup-node.node-version }} + - shell: bash + run: | + npm i -g \ + semantic-release \ + @semantic-release/exec \ + @semantic-release/git \ + @semantic-release/github \ + @semantic-release/changelog \ + @google/semantic-release-replace-plugin diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..daba8580 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,51 @@ +name: Setup base (python, pip cache, tox) +inputs: + python: + description: "Python version to use" + required: true + type: string + default: 3.12 +outputs: + 'python-version': + value: ${{ steps.python.outputs.python-version }} +runs: + using: "composite" + steps: + - name: pip cache + uses: actions/cache@v4 + with: + path: | + ~/.cache/pip + key: ${{ runner.os }}-pip-${{ inputs.python }} + + - name: Cargo cache + uses: actions/cache/@v4 + with: + path: "~/.cargo" + key: ${{ runner.os }}-cargo + + - name: Poetry cache + uses: actions/cache/@v4 + with: + path: "~/.cache/pypoetry" + key: ${{ runner.os }}-poetry-${{ inputs.python }} + restore-keys: | + ${{ runner.os }}-poetry- + + - uses: actions/setup-python@v6 + id: python + with: + python-version: ${{ inputs.python }} + + - name: upgrade pip and install tox + shell: bash + run: | + python -m pip -q install --upgrade pip "setuptools==65.6.2" + pip -q install "tox<4" tox-gh-actions + + - name: install Rust and Poetry + shell: bash + run : | + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal + source "$HOME/.cargo/env" + pip -q install poetry>=1.2.0 diff --git a/.github/workflows/lint-and-test.yml b/.github/workflows/lint-and-test.yml new file mode 100644 index 00000000..f0ad33d8 --- /dev/null +++ b/.github/workflows/lint-and-test.yml @@ -0,0 +1,55 @@ +name: Lint and test +on: + pull_request: + push: + branches: + - master + - 'ci/**' # ci testing, pre-releases + #- 'feature/**' + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + - name: Lint + id: lint + run: tox -e lint + continue-on-error: true + - name: Emit warning if lint failed + if: ${{ steps.lint.outcome != 'success' }} + run: echo "::warning::Linter failure suppressed (continue-on-error=true)" + test: + strategy: + fail-fast: false + matrix: + os: [ ubuntu-latest ] + python: + - "3.12" + - "3.11" + - "3.10" + - "3.9.14" + - "3.8" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup + with: + python: ${{ matrix.python }} + - name: Test + run: tox + validation: + name: Validation + runs-on: ubuntu-latest + needs: [test] + if: always() + steps: + - name: Validate matrix test success + run: | + # Check the status of the 'test' job (which includes all matrix variations) + if [ "${{ needs.test.result }}" != "success" ]; then + echo "One or more matrix test jobs failed." + exit 1 + fi + echo "All matrix test jobs passed." diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 5d2fd055..00000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Build -on: - pull_request: - schedule: - - cron: "0 0 * * 0" - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ ubuntu-latest ] - python: ["3.12"] - include: - - python: "3.12" - tox_env: "lint" - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - name: Install tox - run: | - python -m pip install --upgrade pip setuptools - pip install tox - - name: Test - run: | - tox -e ${{ matrix.tox_env }} - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..2928b1af --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,146 @@ +name: Release + +on: + workflow_run: # would only fire after file is merged to master + workflows: ["Lint and test"] + types: + - completed + branches: + - master + - 'ci/**' # ci testing, pre-releases + #- develop # can emit -dev releases but we do not want to + workflow_dispatch: + inputs: + dry_run: + description: "Run in dry-run mode (no publish)" + required: false + default: "true" + push: # only temporary, until this file lands on master (see above) + branches: + - 'ci/**' + +# MUSTHAVE: Trusted publisher access for both repos. +# NOTE: according to docs, 'test' repo accounts are ephemeral and can be wiped at any time +# NOTE: 'test' accs are not that ephmeperal -- losing access to sandbox account (2FA issue) effectively locked us out of project; good test for workarounds though +# NOTE: as a part of regaining-control scenario we may use distinct project names in pyroject.toml (e.g. appmap-dev, appmap-ng) +env: + DRY_RUN: ${{ github.event.inputs.dry_run || 'false' }} + pypi_project: appmap + #testpypi_project: appmap-dev # workaround for lost-access scenario + testpypi_project: appmapcitest + +jobs: + + setup: + runs-on: ubuntu-latest + outputs: + distribution_name: ${{ steps.configure.outputs.distribution_name }} + publish_to: ${{ steps.configure.outputs.publish_to }} + publish_env: ${{ steps.configure.outputs.publish_env }} + steps: + - id: configure + shell: bash + run: | + case "${{ github.ref_name }}" in + ci/*) + echo "publish_env=testpypi" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.testpypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://test.pypi.org/project/${{ env.testpypi_project }}" >> $GITHUB_OUTPUT + ;; + master) + echo "publish_env=pypi" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.pypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://pypi.org/project/${{ env.pypi_project }}" >> $GITHUB_OUTPUT + ;; + *) + echo "publish_env=SKIP" >> $GITHUB_OUTPUT + echo "distribution_name=${{ env.pypi_project }}" >> $GITHUB_OUTPUT + echo "publish_to=https://test.pypi.org/project/${{ env.pypi_project }}" >> $GITHUB_OUTPUT + ;; + esac + + release: + runs-on: ubuntu-latest + needs: setup + if: github.event_name == 'workflow_dispatch' || (github.event_name=='push' && startsWith(github.ref_name,'ci/') ) || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && (github.event.workflow_run.head_branch == 'master' || startsWith(github.event.workflow_run.head_branch, 'ci/') ) ) + permissions: + contents: write + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/setup-semantic-release # node+semantic-release + - uses: ./.github/actions/setup # poetry + - id: semantic-release # branch policies defined in .releaserc + env: + GIT_AUTHOR_NAME: appland-release + GIT_AUTHOR_EMAIL: release@app.land + GIT_COMMITTER_NAME: appland-release + GIT_COMMITTER_EMAIL: release@app.land + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} + run: | + if [ "$DRY_RUN" = "true" ]; then + semantic-release --dry-run + else + semantic-release + fi + + - name: Upload wheel + if: env.DRY_RUN != 'true' + uses: actions/upload-artifact@v4 + with: + name: wheel + path: dist/*.whl + - name: Upload sdist + if: env.DRY_RUN != 'true' + uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + outputs: # not reused in fact + release_tag: ${{ steps.semantic-release.outputs.next_release_tag }} + + smoketest: + runs-on: ubuntu-latest + needs: ['setup', 'release'] + if: github.event.inputs.dry_run!='true' + continue-on-error: ${{ needs.setup.outputs.distribution_name!='appmap' }} # altered names won't work anyway + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/refetch-artifacts + - name: dockerhub login (for seamless docker pulling) + uses: ./.github/actions/dockerhub-login + env: + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USERNAME }} + continue-on-error: true + - run: ci/scripts/run_tests.sh + env: + SMOKETEST_DOCKER_IMAGE: python:3.12-slim + DISTRIBUTION_NAME: ${{ needs.setup.outputs.distribution_name }} + + # as a workaround to ownership issues (lost access to project) + publish: + name: publish package on PyPI + needs: ['setup', 'release','smoketest'] + if: (( github.event.inputs.dry_run != 'true' ) && ( (needs.setup.outputs.publish_env == 'pypi') || (needs.setup.outputs.publish_env == 'testpypi') ) ) + runs-on: ubuntu-latest + environment: + name: ${{ needs.setup.outputs.publish_env }} + url: ${{ needs.setup.outputs.publish_to }} + permissions: + id-token: write + steps: + - uses: actions/checkout@v5 + - uses: ./.github/actions/refetch-artifacts + + - name: Publish to PyPI + if: needs.setup.outputs.publish_env=='pypi' + uses: pypa/gh-action-pypi-publish@release/v1 + + - name: Publish to TestPyPI + if: needs.setup.outputs.publish_env=='testpypi' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ # trailing slash matters! diff --git a/.releaserc.yml b/.releaserc.yml index 22f1d3c7..b5efcfc6 100644 --- a/.releaserc.yml +++ b/.releaserc.yml @@ -1,3 +1,18 @@ +# Allowed number of prerelease rules: 1..3 +# While semantic-release allows globs, they must be combined with `prerelease: true` and suffix is derived from name than. It conflicts with PEP440 +# PEP440 version rules (not compatible with SemVer): [N!]N(.N)*[{a|b|rc}N][.postN][.devN] +# Consequences: +# - prerelease branches must be explicitly specified, no asterisks +# - prerelease parameter should be one of: a,b,rc,dev,post +# - translation from SemVer prerelease notation to PEP440 is tone in 'replacements' section +branches: # only branches listed here will create releases + - master + - name: ci/trusted_publishing_test + prerelease: dev + #- name: develop + # prerelease: dev + #- name: feature/* + # prerelease: true # will use branch name as suffix plugins: - '@semantic-release/commit-analyzer' - '@semantic-release/release-notes-generator' @@ -13,9 +28,22 @@ plugins: hasChanged: true numMatches: 1 numReplacements: 1 +- - '@google/semantic-release-replace-plugin' # optional SemVer -> PEP440 coercion + - replacements: + - files: [pyproject.toml] # optional: SemVer prerelease -> PEP440 ("1.2.3-dev.1" -> "1.2.3.dev1") + from: '^version = "(\\d+\\.\\d+\\.\\d+)-(dev|post)\\.(\\d+)"' + to: 'version = "\\1.\\2\\3"' + - files: [pyproject.toml] # optional: SemVer prerelease -> PEP440 ("1.2.3-rc.10" -> "1.2.3rc10" ) + from: '^version = "(\\d+\\.\\d+\\.\\d+)-(a|b|rc)\\.(\\d+)"' + to: 'version = "\\1\\2\\3"' - - '@semantic-release/git' - assets: - CHANGELOG.md - pyproject.toml - - '@semantic-release/exec' - - publishCmd: poetry publish --build + - prepareCmd: | + /bin/bash ./ci/scripts/build_with_poetry.sh +- - '@semantic-release/github': + - assets: + - dist/*.whl + - dist/*.tar.gz diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d54311ce..00000000 --- a/.travis.yml +++ /dev/null @@ -1,60 +0,0 @@ -os: linux -dist: jammy -language: python -python: -- "3.12" -- "3.11" -- "3.10" -- "3.9.14" -- "3.8" - -# https://github.com/travis-ci/travis-ci/issues/1147#issuecomment-441393807 -if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/ - -before_install: | - curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal - source "$HOME/.cargo/env" - pip -q install --upgrade pip 'setuptools==65.6.2' 'poetry>=1.2.0' - -install: pip -q install --upgrade "tox < 4" tox-travis -script: tox - -cache: - cargo: true - pip: true - directories: - - $TRAVIS_BUILD_DIR/.tox/ - - $HOME/.cache/pypoetry - -jobs: - include: - - stage: smoke test - services: - - docker - script: - - pip -q install poetry - - poetry build - - echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin - - ci/run_tests.sh - - stage: release - if: branch = master - script: skip - before_deploy: - - pip -q install poetry - - nvm install lts/* - - npm i -g - semantic-release - @semantic-release/exec - @semantic-release/git - @semantic-release/changelog - @google/semantic-release-replace-plugin - # Note publishing this way requires the PyPI credentials to be - # present in the environment. Travis doesn't currently support - # providing environment variables to deploy providers through - # the build config (i.e. in this file). So, they must be - # provided through the build settings instead. - deploy: - - provider: script - script: semantic-release - on: - branch: master diff --git a/ci/run_tests.sh b/ci/run_tests.sh deleted file mode 100755 index c78d5415..00000000 --- a/ci/run_tests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -x -t=$([ -t 0 ] && echo 't') -docker run -q -i${t} --rm\ - -v $PWD/dist:/dist -v $PWD/_appmap/test/data/unittest:/_appmap/test/data/unittest\ - -v $PWD/ci:/ci\ - -w /tmp\ - -v $PWD/ci/readonly-mount-appmap.log:/tmp/appmap.log:ro\ - python:3.11 bash -ce "${@:-/ci/smoketest.sh; /ci/test_pipenv.sh; /ci/test_poetry.sh}" diff --git a/ci/scripts/build_with_poetry.sh b/ci/scripts/build_with_poetry.sh new file mode 100755 index 00000000..53c89791 --- /dev/null +++ b/ci/scripts/build_with_poetry.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +set -o pipefail + +if [ -z "$DISTRIBUTION_NAME" ] || [ "$DISTRIBUTION_NAME" = "appmap" ] ; then + exec poetry build $* +fi + +echo "Altering distribution name to $DISTRIBUTION_NAME" + +cp -v pyproject.toml /tmp/pyproject.bak +sed -i -e "s/^name = \".*\"/name = \"${DISTRIBUTION_NAME}\"/" pyproject.toml +grep -n 'name = "' pyproject.toml + +poetry build $* + +echo "Not patching artifacts with Provides-Dist, they won't work anyway (this flow is solely for publishing test)" +cp -v /tmp/pyproject.bak pyproject.toml diff --git a/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh b/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh new file mode 100755 index 00000000..acee04ff --- /dev/null +++ b/ci/scripts/patch_artifacts_if_distribution_name_is_altered.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e +set -o pipefail + +artifacts=$* +injection_string="Provides-Dist: appmap" +if [ -n "$artifacts" ] && [ -n "$DISTRIBUTION_NAME" ] && [ "$DISTRIBUTION_NAME" != "appmap" ]; then + echo "Altered distribution name detected, injecting '$injection_string' into artifacts: $artifacts" + for artifact in $artifacts ; do + TMP=$(mktemp -d) + ARTIFACT_PATH="$(realpath ${artifact})" + if [[ $artifact == *.whl ]]; then + unzip -q "$ARTIFACT_PATH" -d "$TMP" + DISTINFO=$(find "$TMP" -type d -name "*.dist-info") + echo "$injection_string" >> "$DISTINFO/METADATA" + (cd "$TMP" && zip -qr "$ARTIFACT_PATH" .) + else + tar -xzf "$ARTIFACT_PATH" -C "$TMP" + PKG_INFO_FILE=$(find "$TMP" -type f -name "PKG-INFO") + echo "$injection_string" >> "$PKG_INFO_FILE" + + # Get the top-level directory to repack correctly + PKGDIR=$(find "$TMP" -mindepth 1 -maxdepth 1 -type d) + (cd "$TMP" && tar -czf "$ARTIFACT_PATH" "$(basename "$PKGDIR")") + fi + echo "($injection_string): patched $ARTIFACT_PATH" + rm -rf "$TMP" + done +fi diff --git a/ci/scripts/run_tests.sh b/ci/scripts/run_tests.sh new file mode 100755 index 00000000..d780a187 --- /dev/null +++ b/ci/scripts/run_tests.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +SMOKETEST_DOCKER_IMAGE=${SMOKETEST_DOCKER_IMAGE:-"python:3.11"} +DISTRIBUTION_NAME=${DISTRIBUTION_NAME:-appmap} + +set -x +t=$([ -t 0 ] && echo 't') +docker run -q -i${t} --rm \ + -v $PWD/dist:/dist \ + -v $PWD/_appmap/test/data/unittest:/_appmap/test/data/unittest\ + -v $PWD/ci/tests:/ci/tests\ + -v $PWD/.git:/tmp/.git:ro\ + -v $PWD/ci/tests/data/readonly-mount-appmap.log:/tmp/appmap.log:ro\ + -w /tmp\ + -e DISTRIBUTION_NAME \ + $SMOKETEST_DOCKER_IMAGE bash -ce "${@:-/ci/tests/smoketest.sh; /ci/tests/test_pipenv.sh; /ci/tests/test_poetry.sh}" diff --git a/ci/readonly-mount-appmap.log b/ci/tests/data/readonly-mount-appmap.log similarity index 100% rename from ci/readonly-mount-appmap.log rename to ci/tests/data/readonly-mount-appmap.log diff --git a/ci/smoketest.sh b/ci/tests/smoketest.sh similarity index 88% rename from ci/smoketest.sh rename to ci/tests/smoketest.sh index 3d0eeb13..cd0abade 100755 --- a/ci/smoketest.sh +++ b/ci/tests/smoketest.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash + test_recording_when_appmap_not_true() { cat < test_client.py @@ -36,8 +37,13 @@ EOF } set -ex + +# now appmap requires git +apt-get update -qq \ + && apt-get install -y --no-install-recommends git + pip -q install -U pip pytest "flask>=2,<3" python-decouple -pip -q install /dist/appmap-*-py3-none-any.whl +pip -q install /dist/${DISTRIBUTION_NAME//-/_}-*-py3-none-any.whl cp -R /_appmap/test/data/unittest/simple ./. @@ -66,4 +72,4 @@ else exit 1 fi -test_log_file_not_writable \ No newline at end of file +test_log_file_not_writable diff --git a/ci/test_pipenv.sh b/ci/tests/test_pipenv.sh similarity index 100% rename from ci/test_pipenv.sh rename to ci/tests/test_pipenv.sh diff --git a/ci/test_poetry.sh b/ci/tests/test_poetry.sh similarity index 100% rename from ci/test_poetry.sh rename to ci/tests/test_poetry.sh diff --git a/tox.ini b/tox.ini index a92da61a..be81c9d2 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,19 @@ [tox] isolated_build = true + # The *-web environments test the latest versions of Django and Flask with the full test suite. For # older version of the web frameworks, just run the tests that are specific to them. -envlist = py3{10,11,12}-{django5}, py3{8,9,10,11,12}-{web,django3,django4,flask2,sqlalchemy1},lint + +# Default envlist is only for matrix testing. Linter and vendoring should be called explicitly +envlist = py3{10,11,12}-{django5}, py3{8,9,10,11,12}-{web,django3,django4,flask2,sqlalchemy1} + +[gh-actions] +python = + 3.8: py38 + 3.9: py39 + 3.10: py310 + 3.11: py311 + 3.12: py312 [web-deps] deps= @@ -54,4 +65,4 @@ deps = vendoring commands = poetry run vendoring {posargs:sync} # We don't need the .pyi files vendoring generates - python -c 'from pathlib import Path; all(map(Path.unlink, Path("vendor").rglob("*.pyi")))' \ No newline at end of file + python -c 'from pathlib import Path; all(map(Path.unlink, Path("vendor").rglob("*.pyi")))'