diff --git a/.github/workflows/faucet_test.yml b/.github/workflows/faucet_test.yml index 0a0a56ff3..c6c4bdf8c 100644 --- a/.github/workflows/faucet_test.yml +++ b/.github/workflows/faucet_test.yml @@ -4,6 +4,7 @@ on: push: branches: [main] workflow_dispatch: + workflow_call: env: POETRY_VERSION: 2.1.1 diff --git a/.github/workflows/integration_test.yml b/.github/workflows/integration_test.yml index e37c96790..03a5f5981 100644 --- a/.github/workflows/integration_test.yml +++ b/.github/workflows/integration_test.yml @@ -9,6 +9,7 @@ on: branches: [main] pull_request: workflow_dispatch: + workflow_call: jobs: integration-test: diff --git a/.github/workflows/publish_to_pypi.yml b/.github/workflows/publish_to_pypi.yml deleted file mode 100644 index eab0e0195..000000000 --- a/.github/workflows/publish_to_pypi.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Publish xrpl-py 🐍 distribution πŸ“¦ to PyPI -on: - push: - tags: - - "*" - -jobs: - build: - name: Build distribution πŸ“¦ - runs-on: ubuntu-latest - env: - POETRY_VERSION: 2.1.1 - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - # Use the lowest supported version of Python for CI/CD - python-version: "3.8" - - name: Load cached .local - id: cache-poetry - uses: actions/cache@v3 - with: - path: /home/runner/.local - key: dotlocal-${{ env.POETRY_VERSION }}-${{ hashFiles('poetry.lock') }} - - name: Install poetry - if: steps.cache-poetry.outputs.cache-hit != 'true' - run: | - curl -sSL "https://install.python-poetry.org/" | python - --version "${{ env.POETRY_VERSION }}" - echo "${HOME}/.local/bin" >> $GITHUB_PATH - poetry --version || exit 1 # Verify installation - - name: Build a binary wheel and a source tarball - run: poetry build - - name: Store the distribution packages - uses: actions/upload-artifact@v4 - with: - name: python-package-distributions - path: dist/ - publish-to-pypi: - name: >- - Publish Python 🐍 distribution πŸ“¦ to PyPI - needs: build # Explicit dependency on build job - runs-on: ubuntu-latest - timeout-minutes: 10 # Adjust based on typical publishing time - permissions: - # More information about Trusted Publishing and OpenID Connect: https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ - id-token: write # IMPORTANT: mandatory for trusted publishing - steps: - - name: Download all the dists - uses: actions/download-artifact@v4 - with: - name: python-package-distributions - path: dist/ - - name: Verify downloaded artifacts - run: | - ls dist/*.whl dist/*.tar.gz || exit 1 - - name: Publish distribution πŸ“¦ to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - verbose: true - verify-metadata: true - - github-release: - name: >- - Sign the Python 🐍 distribution πŸ“¦ with Sigstore - and upload them to GitHub Release - needs: - - publish-to-pypi - runs-on: ubuntu-latest - timeout-minutes: 15 # Adjust based on typical signing and release time - - permissions: - contents: write # IMPORTANT: mandatory for making GitHub Releases - id-token: write # IMPORTANT: mandatory for sigstore - - steps: - - name: Download all the dists - uses: actions/download-artifact@v4 - with: - name: python-package-distributions - path: dist/ - - name: Sign the dists with Sigstore - uses: sigstore/gh-action-sigstore-python@v2.1.1 - with: - inputs: >- - ./dist/*.tar.gz - ./dist/*.whl - - name: Create GitHub Release - env: - GITHUB_TOKEN: ${{ github.token }} - run: >- - gh release create - '${{ github.ref_name }}' - --repo '${{ github.repository }}' - --generate-notes || - (echo "::error::Failed to create release" && exit 1) - - name: Upload artifact signatures to GitHub Release - env: - GITHUB_TOKEN: ${{ github.token }} - # Upload to GitHub Release using the `gh` CLI. - # `dist/` contains the built packages, and the - # sigstore-produced signatures and certificates. - run: >- - gh release upload - '${{ github.ref_name }}' dist/** - --repo '${{ github.repository }}' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..033ea896a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,664 @@ +name: Publish xrpl-py 🐍 distribution πŸ“¦ to PyPI +on: + workflow_dispatch: + inputs: + release_branch: + description: "Release branch to package (e.g., release/1.0.0)" + required: true + type: string + +jobs: + input-validate: + name: Validate release inputs + runs-on: ubuntu-latest + outputs: + package_version: ${{ steps.package_version.outputs.version }} + is_beta_release: ${{ steps.detect_release_kind.outputs.is_beta_release }} + # release_branch output no longer needed; reference github.event.inputs.release_branch directly + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.release_branch }} + - name: Install toml-cli + run: | + set -euo pipefail + python3 -m venv /tmp/tomlcli + /tmp/tomlcli/bin/pip install --upgrade pip + /tmp/tomlcli/bin/pip install 'toml-cli==0.8.2' + echo "/tmp/tomlcli/bin" >> "${GITHUB_PATH}" + + - name: Extract package version + id: package_version + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + + rm -f /tmp/toml_err + if ! VERSION="$(toml get project.version --toml-path pyproject.toml 2>/tmp/toml_err)"; then + cat /tmp/toml_err >&2 || true + echo "Unable to retrieve version from pyproject.toml using toml-cli" >&2 + exit 1 + fi + rm -f /tmp/toml_err + if [[ -z "${VERSION}" ]]; then + echo "Version value is empty in pyproject.toml" >&2 + exit 1 + fi + # Ensure no existing remote git tag matches this version (protect against re-releases) + if gh api -X GET "repos/$REPO/git/ref/tags/${VERSION}" >/dev/null 2>&1 || \ + gh api -X GET "repos/$REPO/git/ref/tags/v${VERSION}" >/dev/null 2>&1; then + echo "❌ A remote git tag matching the version already exists: '${VERSION}' or 'v${VERSION}'." >&2 + echo "Please bump the version in pyproject.toml or remove the existing git tag before releasing." >&2 + exit 1 + fi + echo "version=${VERSION}" >> "${GITHUB_OUTPUT}" + echo "Detected package version: ${VERSION}" + + - name: Determine release type + id: detect_release_kind + run: | + set -euo pipefail + VERSION="${{ steps.package_version.outputs.version }}" + if [[ "$VERSION" =~ (a|b|rc) ]]; then + echo "is_beta_release=true" >> "$GITHUB_OUTPUT" + else + echo "is_beta_release=false" >> "$GITHUB_OUTPUT" + fi + - name: Validate inputs + id: validate_inputs + env: + TRIGGER_BRANCH: ${{ github.ref_name }} + IS_BETA_RELEASE: ${{ steps.detect_release_kind.outputs.is_beta_release }} + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + # if [[ "${TRIGGER_BRANCH}" != "main" ]]; then + # echo "❌ This workflow must be dispatched from the main branch (current: ${TRIGGER_BRANCH})." >&2 + # exit 1 + # fi + + RELEASE_BRANCH="${{ github.event.inputs.release_branch }}" + + if [[ -z "$RELEASE_BRANCH" ]]; then + echo "❌ Unable to determine branch name." >&2 + exit 1 + fi + + if [[ "${IS_BETA_RELEASE}" != "true" ]] && [[ ! "${RELEASE_BRANCH,,}" =~ ^release[-/] ]]; then + echo "❌ Release branch '$RELEASE_BRANCH' must start with 'release-' or 'release/' for stable releases." >&2 + exit 1 + fi + + if [[ "${IS_BETA_RELEASE}" != "true" ]]; then + OWNER="${REPO%%/*}" + echo "πŸ”Ž Validating that a release PR exists for ${RELEASE_BRANCH} β†’ main…" + PRS_JSON="$(gh api -H 'Accept: application/vnd.github+json' \ + --method GET \ + -f state=open \ + -f base=main \ + -f head="${OWNER}:${RELEASE_BRANCH}" \ + "/repos/$REPO/pulls")" + PR_NUMBER="$(printf '%s' "$PRS_JSON" | jq -r '.[0].number // empty')" + PR_URL="$(printf '%s' "$PRS_JSON" | jq -r '.[0].html_url // empty')" + if [ -z "$PR_NUMBER" ]; then + echo "❌ No open PR found from ${RELEASE_BRANCH} to main. Please create the release PR before running this workflow." >&2 + exit 1 + fi + echo "ℹ️ Found release PR: #$PR_NUMBER ($PR_URL)" + else + echo "ℹ️ Beta release detected; skipping PR existence check." + fi + + if grep -R --exclude-dir=.git --exclude-dir=.github "artifactory.ops.ripple.com" .; then + echo "❌ Internal Artifactory URL found" + exit 1 + else + echo "βœ… No Internal Artifactory URL found" + fi + echo "release_branch=${RELEASE_BRANCH}" >> "$GITHUB_OUTPUT" + + faucet-tests: + name: Run faucet tests matrix (${{ needs.input-validate.outputs.package_version }}) + needs: + - input-validate + uses: ./.github/workflows/faucet_test.yml + secrets: inherit + + integration-tests: + name: Run integration tests matrix (${{ needs.input-validate.outputs.package_version }}) + needs: + - input-validate + uses: ./.github/workflows/integration_test.yml + secrets: inherit + + unit-tests: + name: Run unit tests matrix (${{ needs.input-validate.outputs.package_version }}) + needs: + - input-validate + uses: ./.github/workflows/unit_test.yml + secrets: inherit + + pre-release: + name: Pre-release distribution πŸ“¦ (${{ needs.input-validate.outputs.package_version }}) + needs: + - input-validate + - faucet-tests + - integration-tests + - unit-tests + if: ${{ always() + && needs.input-validate.result == 'success' + && (needs['faucet-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['integration-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['unit-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') }} + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + attestations: write + issues: write + env: + POETRY_VERSION: 2.1.1 + CYCLONEDX_BOM_VERSION: 7.2.0 + PACKAGE_VERSION: ${{ needs.input-validate.outputs.package_version }} + outputs: + package_version: ${{ needs.input-validate.outputs.package_version }} + vuln_art_url: ${{ steps.vuln_art.outputs.art_url }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.release_branch }} + - name: Load cached .local + id: cache-poetry + uses: actions/cache@v4 + with: + path: /home/runner/.local + key: dotlocal-${{ env.POETRY_VERSION }} + + - name: Install poetry + if: steps.cache-poetry.outputs.cache-hit != 'true' + run: | + python --version + curl -sSL https://install.python-poetry.org/ | python - --version ${{ env.POETRY_VERSION }} + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Install Python + Retrieve Poetry dependencies from cache + uses: actions/setup-python@v5 + with: + python-version: "3.8" + cache: "poetry" + - name: Build a binary wheel and a source tarball + run: poetry build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Generate build provenance attestation + id: provenance + uses: actions/attest-build-provenance@v1 + with: + subject-path: "dist/*" + - name: Store provenance attestation + if: steps.provenance.outputs.bundle-path != '' + uses: actions/upload-artifact@v4 + with: + name: python-package-provenance + path: ${{ steps.provenance.outputs.bundle-path }} + - name: Prepare vulnerbility scan + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Install CycloneDX Python tool + run: | + set -euo pipefail + python -m pip install --upgrade "cyclonedx-bom==${CYCLONEDX_BOM_VERSION}" + - name: Generate CycloneDX SBOM + run: | + set -euo pipefail + cyclonedx-py poetry > sbom.json + if [[ ! -s sbom.json ]]; then + echo "Generated SBOM is empty" >&2 + exit 1 + fi + - name: Scan SBOM for vulnerabilities using Trivy + uses: aquasecurity/trivy-action@0.33.1 + with: + scan-type: sbom + scan-ref: sbom.json + format: table + exit-code: 0 + output: vuln-report.txt + severity: CRITICAL,HIGH + - name: Upload sbom to OWASP + run: | + set -euo pipefail + curl -X POST \ + -H "X-Api-Key: ${{ secrets.OWASP_TOKEN }}" \ + -F "autoCreate=true" \ + -F "projectName=xrpl-py" \ + -F "projectVersion=${{ env.PACKAGE_VERSION }}" \ + -F "bom=@sbom.json" \ + https://owasp-dt-api.prod.ripplex.io/api/v1/bom + - name: Upload SBOM artifact + uses: actions/upload-artifact@v4 + with: + name: sbom + path: sbom.json + - name: Show scan report + id: show_scan_report + run: | + set -euo pipefail + if ! grep -qE "CRITICAL|HIGH" vuln-report.txt; then + printf '\n%s\n' "βœ… No CRITICAL or HIGH vulnerabilities detected for xrpl-py." >> vuln-report.txt + echo "found_vulnerability=false" >> "$GITHUB_OUTPUT" + else + echo "found_vulnerability=true" >> "$GITHUB_OUTPUT" + fi + cat vuln-report.txt + - name: Upload vulnerability report artifact + id: upload_vuln + uses: actions/upload-artifact@v4 + with: + name: vulnerability-report + path: vuln-report.txt + - name: Build vuln artifact URL + id: vuln_art + run: | + set -euo pipefail + echo "art_url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts/${{ steps.upload_vuln.outputs.artifact-id }}" >> "$GITHUB_OUTPUT" + - name: Create GitHub Issue for vulnerabilities + if: steps.show_scan_report.outputs.found_vulnerability == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PKG_VER: ${{ env.PACKAGE_VERSION }} + REL_BRANCH: ${{ github.event.inputs.release_branch }} + VULN_ART_URL: ${{ steps.vuln_art.outputs.art_url }} + LABELS: security + run: | + set -euo pipefail + TITLE="πŸ”’ Security vulnerabilities in xrpl-py@${PKG_VER}" + : > issue_body.md + { + echo "The vulnerability scan has detected **CRITICAL/HIGH** vulnerabilities for \`xrpl-py@${PKG_VER}\` on branch \`${REL_BRANCH}\`." + echo "" + echo "**Release Branch:** \`${REL_BRANCH}\`" + echo "**Package Version:** \`${PKG_VER}\`" + echo "" + echo "**Full vulnerability report:** ${VULN_ART_URL}" + echo "" + echo "Please review the report and take necessary action." + echo "" + echo "---" + echo "_This issue was automatically generated by the Publish to PyPI workflow._" + } >> issue_body.md + gh issue create --title "$TITLE" --body-file issue_body.md --label "$LABELS" + ask_for_dev_team_review: + name: Summarize release and request Dev review + runs-on: ubuntu-latest + needs: + - input-validate + - faucet-tests + - integration-tests + - unit-tests + - pre-release + if: ${{ always() + && needs.input-validate.result == 'success' + && needs.pre-release.result == 'success' + && (needs['faucet-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['integration-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['unit-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') }} + permissions: + pull-requests: write + outputs: + reviewers_dev: ${{ steps.get_reviewers.outputs.reviewers_dev }} + reviewers_sec: ${{ steps.get_reviewers.outputs.reviewers_sec }} + env: + PACKAGE_VERSION: ${{ needs.input-validate.outputs.package_version }} + IS_BETA_RELEASE: ${{ needs.input-validate.outputs.is_beta_release }} + RELEASE_BRANCH: ${{ github.event.inputs.release_branch }} + SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + ENV_DEV_NAME: ${{ needs.input-validate.outputs.is_beta_release == 'true' && 'beta-release' || 'first-review' }} + ENV_SEC_NAME: ${{ needs.input-validate.outputs.is_beta_release == 'true' && 'beta-release' || 'official-release' }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ env.RELEASE_BRANCH }} + + - name: Get reviewers + id: get_reviewers + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + PACKAGE_VERSION: ${{ env.PACKAGE_VERSION }} + GITHUB_ACTOR: ${{ github.actor }} + GITHUB_TRIGGERING_ACTOR: ${{ github.triggering_actor }} + run: | + set -euo pipefail + + fetch_reviewers() { + local env_name="$1" + local env_json reviewers + env_json="$(curl -sSf \ + -H "Authorization: Bearer $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$REPO/environments/$env_name")" || true + + reviewers="$(printf '%s' "$env_json" | jq -r ' + (.protection_rules // []) + | map(select(.type=="required_reviewers") | .reviewers // []) + | add // [] + | map( + if .type=="User" then (.reviewer.login) + elif .type=="Team" then (.reviewer.slug) + else (.reviewer.login // .reviewer.slug // "unknown") + end + ) + | unique + | join(", ") + ')" + if [ -z "$reviewers" ] || [ "$reviewers" = "null" ]; then + reviewers="(no required reviewers configured)" + fi + printf '%s' "$reviewers" + } + + # Get reviewer lists + REVIEWERS_DEV="$(fetch_reviewers "$ENV_DEV_NAME")" + REVIEWERS_SEC="$(fetch_reviewers "$ENV_SEC_NAME")" + + # Output messages + echo "reviewers_dev=$REVIEWERS_DEV" >> "$GITHUB_OUTPUT" + echo "reviewers_sec=$REVIEWERS_SEC" >> "$GITHUB_OUTPUT" + + - name: Release summary for review + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + PACKAGE_VERSION: ${{ env.PACKAGE_VERSION }} + RELEASE_BRANCH: ${{ env.RELEASE_BRANCH }} + run: | + set -euo pipefail + ARTIFACT_NAME="vulnerability-report" + ARTIFACTS=$(curl -s -H "Authorization: Bearer $GH_TOKEN" \ + -H "Accept: application/vnd.github+json" \ + "https://api.github.com/repos/$REPO/actions/runs/$RUN_ID/artifacts") + + ARTIFACT_ID=$(echo "$ARTIFACTS" | jq -r ".artifacts[]? | select(.name == \"$ARTIFACT_NAME\") | .id") + + echo "πŸ“¦ Package version: $PACKAGE_VERSION" + echo "🌿 Release branch: $RELEASE_BRANCH" + if [ -n "${ARTIFACT_ID:-}" ]; then + echo "πŸ›‘οΈ Vulnerability report: https://github.com/$REPO/actions/runs/$RUN_ID/artifacts/$ARTIFACT_ID" + else + echo "⚠️ Vulnerability report artifact not found" + fi + + - name: Send Dev review message to Slack + if: always() + shell: bash + env: + SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + CHANNEL: "#xrpl-py" + EXECUTOR: ${{ github.triggering_actor || github.actor }} + RELEASE_BRANCH: ${{ env.RELEASE_BRANCH }} + PACKAGE_VERSION: ${{ env.PACKAGE_VERSION }} + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + DEV_REVIEWERS: ${{ steps.get_reviewers.outputs.reviewers_dev }} + run: | + set -euo pipefail + + MSG="${EXECUTOR} is releasing xrpl-py ${PACKAGE_VERSION} from ${RELEASE_BRANCH}. A member from the dev team (${DEV_REVIEWERS}) needs to review the release artifacts and approve/reject the release. (${RUN_URL})" + MSG=$(printf '%b' "$MSG") + # Post once + curl -sS -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer $SLACK_TOKEN" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$(jq -n --arg channel "$CHANNEL" --arg text "$MSG" '{channel:$channel, text:$text}')" \ + | jq -er '.ok' >/dev/null + + + first_review: + name: First approval (dev team) + runs-on: ubuntu-latest + needs: + - input-validate + - faucet-tests + - integration-tests + - unit-tests + - pre-release + - ask_for_dev_team_review + if: ${{ always() + && needs.pre-release.result == 'success' + && needs.ask_for_dev_team_review.result == 'success' + && (needs['faucet-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['integration-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['unit-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') }} + environment: + name: first-review + url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - name: Awaiting approval + run: echo "Awaiting Dev team approval" + + ask_for_sec_team_review: + name: Request security team review + runs-on: ubuntu-latest + needs: + - input-validate + - faucet-tests + - integration-tests + - unit-tests + - pre-release + - ask_for_dev_team_review + - first_review + if: ${{ always() + && needs.pre-release.result == 'success' + && needs.ask_for_dev_team_review.result == 'success' + && needs.first_review.result == 'success' + && (needs['faucet-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['integration-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['unit-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') }} + env: + PACKAGE_VERSION: ${{ needs.input-validate.outputs.package_version }} + SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + EXECUTOR: ${{ github.triggering_actor || github.actor }} + RELEASE_BRANCH: ${{ github.event.inputs.release_branch }} + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + SEC_REVIEWERS: ${{ needs.ask_for_dev_team_review.outputs.reviewers_sec }} + VULN_ART_URL: ${{ needs.pre-release.outputs.vuln_art_url }} + steps: + - name: Notify security reviewers on Slack + env: + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + PACKAGE_VERSION: ${{ env.PACKAGE_VERSION }} + VULN_ART_URL: ${{ env.VULN_ART_URL }} + run: | + set -euo pipefail + MSG="${EXECUTOR} is releasing xrpl-py ${PACKAGE_VERSION} from ${RELEASE_BRANCH}. A member from the sec reviewer team (${SEC_REVIEWERS}) needs to take the following action:\nReview the vulnerabilities ${VULN_ART_URL} and approve/reject the release. (${RUN_URL})" + MSG=$(printf '%b' "$MSG") + curl -sS -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer $SLACK_TOKEN" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$(jq -n --arg channel "#ripplex-security" --arg text "$MSG" '{channel:$channel, text:$text}')" \ + | jq -er '.ok' >/dev/null + + - name: Awaiting security approval + run: echo "Waiting for security team review" + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution πŸ“¦ to PyPI (${{ needs.pre-release.outputs.package_version }}) + needs: + - input-validate + - faucet-tests + - integration-tests + - pre-release + - ask_for_dev_team_review + - first_review + - ask_for_sec_team_review + - unit-tests + if: ${{ always() + && needs.pre-release.result == 'success' + && needs.ask_for_dev_team_review.result == 'success' + && needs.first_review.result == 'success' + && needs.ask_for_sec_team_review.result == 'success' + && (needs['faucet-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['integration-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['unit-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 10 # Adjust based on typical publishing time + environment: + name: ${{ needs.input-validate.outputs.is_beta_release == 'true' && 'beta-release' || 'official-release' }} + url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + env: + PACKAGE_VERSION: ${{ needs.pre-release.outputs.package_version }} + permissions: + # More information about Trusted Publishing and OpenID Connect: https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/ + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Prevent second attempt + run: | + if (( ${GITHUB_RUN_ATTEMPT:-1} > 1 )); then + echo "❌ Workflow rerun (attempt ${GITHUB_RUN_ATTEMPT}). Second attempts are not allowed." + exit 1 + fi + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Verify downloaded artifacts + run: | + ls dist/*.whl dist/*.tar.gz || exit 1 + - name: Publish distribution πŸ“¦ to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + verbose: true + verify-metadata: true + attestations: true + + github-release: + name: Github Release (${{ needs.pre-release.outputs.package_version }}) + needs: + - faucet-tests + - integration-tests + - input-validate + - pre-release + - ask_for_dev_team_review + - first_review + - ask_for_sec_team_review + - publish-to-pypi + - unit-tests + if: ${{ always() + && needs.pre-release.result == 'success' + && needs.ask_for_dev_team_review.result == 'success' + && needs.first_review.result == 'success' + && needs.ask_for_sec_team_review.result == 'success' + && needs['publish-to-pypi'].result == 'success' + && (needs['faucet-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['integration-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') + && (needs['unit-tests'].result == 'success' || needs.input-validate.outputs.is_beta_release == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 15 # Adjust based on typical signing and release time + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + env: + PACKAGE_VERSION: ${{ needs.pre-release.outputs.package_version }} + IS_BETA_RELEASE: ${{ needs.input-validate.outputs.is_beta_release }} + + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Download provenance attestations + uses: actions/download-artifact@v4 + with: + name: python-package-provenance + path: provenance/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v3.0.1 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release and upload assets + uses: softprops/action-gh-release@v2 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + tag_name: v${{ env.PACKAGE_VERSION }} + generate_release_notes: true + prerelease: ${{ env.IS_BETA_RELEASE == 'true' }} + make_latest: ${{ env.IS_BETA_RELEASE != 'true' }} + files: | + dist/** + provenance/** + + - name: Notify Slack success (single-line) + if: success() + env: + SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + REPO: ${{ github.repository }} + PACKAGE_VERSION: ${{ env.PACKAGE_VERSION }} + TAG: ${{ env.PACKAGE_VERSION }} + run: | + set -euo pipefail + + # Build release URL from tag (URL-encoded to handle '@' etc.) + TAG="${TAG:-${PACKAGE_VERSION}}" + RELEASE_URL="https://github.com/$REPO/releases/tag/v$TAG" + + text="xrpl-py ${PACKAGE_VERSION} has been successfully released and published to pypi. Release URL: ${RELEASE_URL}" + text="${text//\\n/ }" + + curl -sS -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer $SLACK_TOKEN" \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$(jq -n --arg channel "#xrpl-py" --arg text "$text" '{channel:$channel, text:$text}')" + + - name: Notify Slack if tests fail + if: >- + ${{ always() && ( + needs.input-validate.result == 'failure' || + (needs['faucet-tests'].result == 'failure' && needs.input-validate.outputs.is_beta_release != 'true') || + (needs['integration-tests'].result == 'failure' && needs.input-validate.outputs.is_beta_release != 'true') || + (needs['unit-tests'].result == 'failure' && needs.input-validate.outputs.is_beta_release != 'true') || + needs.pre-release.result == 'failure' || + needs.ask_for_dev_team_review.result == 'failure' || + needs.first_review.result == 'failure' || + needs.ask_for_sec_team_review.result == 'failure' || + needs['publish-to-pypi'].result == 'failure' + ) }} + env: + SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + run: | + MESSAGE="❌ Release failed for xrpl-py ${{ env.PACKAGE_VERSION }}. Check the logs: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + curl -X POST https://slack.com/api/chat.postMessage \ + -H "Authorization: Bearer $SLACK_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$(jq -n \ + --arg channel "#xrpl-py" \ + --arg text "$MESSAGE" \ + '{channel: $channel, text: $text}')" diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 88cc19d2d..767183f52 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -5,6 +5,7 @@ on: branches: [main] pull_request: workflow_dispatch: + workflow_call: env: POETRY_VERSION: 2.1.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index af59b6fd6..b1567bf2f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -205,44 +205,6 @@ poetry run poe generate https://github.com/XRPLF/rippled/tree/develop Verify that the changes make sense by inspection before submitting, as there may be updates required for the `xrpl-codec-gen` tool depending on the latest amendments we're updating to match. -## Release process - -### Editing the Code - -- Your changes should have unit and/or integration tests. -- Your changes should pass the linter. -- Your code should pass all the unit and integration tests on Github (which check all versions of Python). -- Open a PR against `main` and ensure that all CI passes. -- Get a full code review from one of the maintainers. -- Merge your changes. - -### Release - -1. Please increment the version in `pyproject.toml` and update the `CHANGELOG.md` file appropriately. We follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -2. Please select a commit that is suitable for release and create a tag. The following commands can be helpful: - `git tag -s -a -m "Optional Message describing the tag"` - `git tag` -- This command displays all the tags in the repository. - `git push tag ` -3. A [Github Workflow](.github/workflows/publish_to_pypi.yml) completes the rest of the Release steps (building the project, generating a .whl and tarball, publishing on the PyPI platform). The workflow uses OpenID Connect's temporary keys to obtain the necessary PyPI authorization. - As a prerequisite, the PyPI `xrpl-py` project needs to authorize Github Actions as a "Trusted Publisher". This page contains helpful resources: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/#configuring-trusted-publishing -4. Send an email to [xrpl-announce](https://groups.google.com/g/xrpl-announce). -5. Post an announcement in the [XRPL Discord #python channel](https://discord.com/channels/886050993802985492/886053080913821717) with a link to the changes and highlighting key changes. - -**Note: If maintainers prefer to manually release the xrpl-py software distribution, the below steps are relevant.** - -1. Create a branch off main that properly increments the version in `pyproject.toml` and updates the `CHANGELOG` appropriately. We follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -2. Merge this branch into `main`. -3. Locally build and download the package. - 1. Pull main locally. - 2. Run `poetry build` to build the package locally. - 3. Locally download the package by running `pip install path/to/local/xrpl-py/dist/.whl`. - 4. Make sure that this local installation works as intended, and that the changes are reflected properly. -4. Run `poetry publish --dry-run` and make sure everything looks good. -5. Publish the update by running `poetry publish`. - - This will require entering PyPI login info. -6. Create a new Github release/tag off of this branch. -7. Send an email to [xrpl-announce](https://groups.google.com/g/xrpl-announce). -8. Post an announcement in the [XRPL Discord #python channel](https://discord.com/channels/886050993802985492/886053080913821717) with a link to the changes and highlighting key changes. ## Mailing Lists diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..c039f8689 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,120 @@ +# xrpl-py Release Playbook + +This guide document describes how to cut and ship a new `xrpl-py` version using the +`Publish xrpl-py 🐍 distribution πŸ“¦ to PyPI` GitHub Actions workflow (see +`.github/workflows/release.yml`). + +## 0. Configurations required for this pipeline + +- Protected environments `first-review` and `official-release`. +- Access to the shared Slack workspace (notifications go to `#xrpl-py` and `#ripplex-security`). +- Reviewers from dev team and infosec team to approve GitHub environment gates and review pull requests. +- PyPI Trusted Publisher to trust the workflow and the protected environment. + +### Beta vs. Stable Releases + +The workflow automatically differentiates between beta/pre-release versions +and standard releases by reading version under the [project] section from `pyproject.toml`: + +- **Beta release**: + - Skips creating the release PR from the release branch back to `main`. + - The GitHub Release is created with the `--prerelease` flag. + - The `latest` tag on GitHub remains unchanged (beta/prerelease builds do not become the + default download). + +- **Stable release**: + - A PR from the release branch to `main` is created (or reused) so the Dev + team can review and merge after PyPI verification. + - The GitHub Release is created with `--latest`, updating the repository’s + default published release. + +## 1. Prepare the Release Branch + +1. Create release branch using name with prefix `release-` (or `release/`). +2. Bump `project.version` inside `pyproject.toml` and update `CHANGELOG.md` + (or other release notes). + +The workflow will fail immediately if the version already exists, if the branch +name does not match the required prefix. + +## 2. Run the Release Workflow + +1. Navigate to **Actions β†’ Publish xrpl-py 🐍 distribution πŸ“¦ to PyPI**. +2. Select the release branch and click **Run workflow**. + +### What the workflow does + +The high-level pipeline is: + +| Stage | Purpose (key steps) | +| --- | --- | +| `input-validate` | Checks branch naming, ensures the version in `pyproject.toml` does not already exist as a tag, Detects whether the release is a beta (`a`, `b`, or `rc`). | +| `faucet-tests`, `integration-tests` | Re-usable workflows that run faucet, unit, and integration test matrices against supported Python versions. | +| `pre-release` | Builds the wheel and sdist with Poetry 2.1.1, uploads build artifacts, generates a CycloneDX SBOM, scans it with Trivy, uploads results to OWASP Dependency-Track, and stores both SBOM and vulnerability reports as Actions artifacts. If any CRITICAL/HIGH findings exist, the job opens a GitHub issue linking to the report. | +| `ask_for_dev_team_review` | Creates or reuses a PR from the release branch to `main` (skipped for beta releases), gathers required reviewers from environment protection rules, prints a summary, and posts a Slack message requesting review/approval. | +| `first_review` | Waits for the Dev environment (`first-review`) approval. | +| `ask_for_sec_team_review` | Notifies security reviewers on Slack and waits for the `official-release` environment approval. | +| `publish-to-pypi` | Downloads the built artifacts from previous step, enforces single-run (no retries), and publishes to PyPI via trusted publishing once approvals are in place. | +| `github-release` | Signs artifacts with the Sigstore action, creates or updates the GitHub Release (`--prerelease` for beta versions, `--latest` for stable releases), uploads signatures/provenance, and posts a Slack success message. | + +## 3. Approvals & Reviews + +- **Dev review**: When `ask_for_dev_team_review` finishes, reviewers receive a + Slack ping. Approvers must visit the workflow run and approve the + `first-review` environment gate. +- **Security review**: After the Dev gate is cleared, the workflow pauses at + `official-release`. Security reviewers receive a Slack ping and must review the vulnerability reports and ++ approve that environment gate. + +## 4. Verify Publication & Finish Up + +1. Wait for `publish-to-pypi` and `github-release` to complete successfully. + Trusted publishing relies on the approved environment gatesβ€”reruns are + blocked, so restart the workflow from scratch if it fails after publishing. +2. Confirm the new version is visible on PyPI: + `https://pypi.org/project/xrpl-py//` +3. Confirm the GitHub Release looks correct (artifacts, provenance, and the + pre-release flag if applicable). +4. Merge the automated release PR into `main` (stable releases only). Do this + after you verify PyPI. +5. Create any follow-up housekeeping PRs (e.g., bumping `dev` version or + updating docs) if needed. +6. Send an email to [xrpl-announce](https://groups.google.com/g/xrpl-announce). +7. Post an announcement in the [XRPL Discord #python channel](https://discord.com/channels/886050993802985492/886053080913821717) with a link to the changes and highlighting key changes. + +## 5. Troubleshooting + +- **Workflow fails during validation**: Check the branch name, version bump, + existing tags (`git ls-remote --tags origin`), and Artifactory references. +- **GitHub release creation fails**: Look at the step output; the workflow will + show the exact `gh release create` command and any API error returned. + +This document should cover the normal release cadence for `xrpl-py`. If the +automation needs adjustments, update both `.github/workflows/release.yml` and +this guide so they stay in sync. + + +## Manual Release process(Just in case Github action is not available for unforseeable reason) + +### Editing the Code + +- Your changes should have passed unit and/or integration tests. +- Your changes should have passed the linter. +- Your code should pass all the unit and integration tests on Github (which check all versions of Python). +- Open a PR against `main` and ensure that all CI passes. +- Get a full code review from one of the maintainers. +- Merge your changes. + +### Release + +1. Create a branch off main that properly increments the version in `pyproject.toml` and updates the `CHANGELOG` appropriately. We follow [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +2. Locally build and install the package. + 1. Run `poetry build` to build the package locally. + 2. Locally download the package by running `pip install path/to/local/xrpl-py/dist/.whl`. + 3. Make sure that this local installation works as intended, and that the changes are reflected properly. +3. Run `poetry publish --dry-run` and make sure everything looks good. +4. Reach out to platform team to for a pypi publishing token. +5. Publish the update by running `poetry publish` with pypi publishing token. +6. Create a new Github release/tag off of this branch. +7. Send an email to [xrpl-announce](https://groups.google.com/g/xrpl-announce). +8. Post an announcement in the [XRPL Discord #python channel](https://discord.com/channels/886050993802985492/886053080913821717) with a link to the changes and highlighting key changes. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 13946abca..5f5e1d1d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,8 @@ documentation = "https://xrpl-py.readthedocs.io" "Bug Tracker" = "https://github.com/XRPLF/xrpl-py/issues" [tool.poetry] +name = "xrpl-py" +description = "A complete Python library for interacting with the XRP ledger" packages = [{ include = "xrpl" }, { include = "LICENSE" }] [tool.poetry.dependencies]