From 76097b4e4195f45dd8496cbb0bddbde9b1a16eed Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 8 Jan 2025 19:50:44 +0400 Subject: [PATCH 1/4] Execute smaller subset of benchmarks on PR --- .github/workflows/benchmark.yml | 3 ++- benchmark/build.gradle.kts | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7ba86569..c733235c 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,6 +39,7 @@ on: env: REPORT_FORMAT: ${{ (github.event_name == 'push' || github.event_name == 'pull_request' ) && 'json' || 'csv' }} + MAIN_BENCHMARK_TASK: ${{ github.event.pull_request.number && 'prValidationBenchmark' || 'benchmark' }} concurrency: cancel-in-progress: true @@ -89,7 +90,7 @@ jobs: gradle-version: wrapper - name: Run benchmarks run: > - ./gradlew --no-daemon :benchmark:benchmark ${{ matrix.additional-task }} + ./gradlew --no-daemon :benchmark:${{ env.MAIN_BENCHMARK_TASK }} ${{ matrix.additional-task }} -Pbenchmark_warmups=${{ inputs.warmups }} -Pbenchmark_iterations=${{ inputs.iterations }} -Pbenchmark_iteration_time=${{ inputs.iteration-time }} diff --git a/benchmark/build.gradle.kts b/benchmark/build.gradle.kts index b11ab446..ac876c3b 100644 --- a/benchmark/build.gradle.kts +++ b/benchmark/build.gradle.kts @@ -62,6 +62,9 @@ benchmark { getByName("main") { include(".*Common.*Bench.*") } + create("prValidation") { + include(".*CommonAvgTimeBench.*") + } create("comparison") { include(".*Comparison.*Benchmark.*") } From ba07e0eb17b6a6775e12bb38f29efe11025a30ec Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 8 Jan 2025 20:04:57 +0400 Subject: [PATCH 2/4] Execute smaller subset for comparison benchmarks --- .github/workflows/benchmark.yml | 2 +- benchmark/build.gradle.kts | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c733235c..c831eb21 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -51,7 +51,7 @@ jobs: matrix: include: - os: ubuntu-latest - additional-task: ':benchmark:jvmComparisonBenchmark' + additional-task: ":benchmark:${{ github.event.pull_request.number && 'jvmPrValidationComparisonBenchmark' || 'jvmComparisonBenchmark' }}" - os: macos-latest additional-task: '-x :benchmark:jvmBenchmark' - os: macos-13 # for macosX64 diff --git a/benchmark/build.gradle.kts b/benchmark/build.gradle.kts index ac876c3b..d083493f 100644 --- a/benchmark/build.gradle.kts +++ b/benchmark/build.gradle.kts @@ -68,6 +68,9 @@ benchmark { create("comparison") { include(".*Comparison.*Benchmark.*") } + create("prValidationComparison") { + include(".*ComparisonAvgTimeBenchmark.*") + } } targets { register("jvm") From 75e9abaf940338bb91d4fe6750db6315e1b54a55 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 8 Jan 2025 20:32:21 +0400 Subject: [PATCH 3/4] Correct names for result directories --- .github/workflows/benchmark.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c831eb21..be140fab 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -39,7 +39,9 @@ on: env: REPORT_FORMAT: ${{ (github.event_name == 'push' || github.event_name == 'pull_request' ) && 'json' || 'csv' }} - MAIN_BENCHMARK_TASK: ${{ github.event.pull_request.number && 'prValidationBenchmark' || 'benchmark' }} + MAIN_BENCH_TASK: ${{ github.event.pull_request.number && 'prValidationBenchmark' || 'benchmark' }} + MAIN_BENCH_RESULTS: ${{ github.event.pull_request.number && 'prValidation' || 'main' }} + COMPARISON_BENCH_RESULTS: ${{ github.event.pull_request.number && 'prValidationComparison' || 'comparison' }} concurrency: cancel-in-progress: true @@ -90,7 +92,7 @@ jobs: gradle-version: wrapper - name: Run benchmarks run: > - ./gradlew --no-daemon :benchmark:${{ env.MAIN_BENCHMARK_TASK }} ${{ matrix.additional-task }} + ./gradlew --no-daemon :benchmark:${{ env.MAIN_BENCH_TASK }} ${{ matrix.additional-task }} -Pbenchmark_warmups=${{ inputs.warmups }} -Pbenchmark_iterations=${{ inputs.iterations }} -Pbenchmark_iteration_time=${{ inputs.iteration-time }} @@ -125,13 +127,13 @@ jobs: uses: actions/upload-artifact@v4 with: name: bench-result-${{ matrix.os }} - path: ${{ env.BENCHMARK_RESULTS }}/main/**/*.json + path: ${{ env.BENCHMARK_RESULTS }}/${{ env.MAIN_BENCH_RESULTS }}/**/*.json - name: Store comparison results as artifact if: env.REPORT_FORMAT == 'json' && matrix.os == 'ubuntu-latest' uses: actions/upload-artifact@v4 with: name: bench-comparison-result-${{ matrix.os }} - path: ${{ env.BENCHMARK_RESULTS }}/comparison/**/*.json + path: ${{ env.BENCHMARK_RESULTS }}/${{ env.COMPARISON_BENCH_RESULTS }}/**/*.json upload-benchmark-results: if: (github.event_name == 'push' || github.event_name == 'pull_request') && github.repository == 'OptimumCode/json-schema-validator' From 1597722c35f30e2235d1ad79a2793e12273e1b40 Mon Sep 17 00:00:00 2001 From: Oleg Smirnov Date: Wed, 8 Jan 2025 20:50:53 +0400 Subject: [PATCH 4/4] Exclude correct task jvm on other platforms --- .github/workflows/benchmark.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index be140fab..dd16e4ce 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -55,11 +55,11 @@ jobs: - os: ubuntu-latest additional-task: ":benchmark:${{ github.event.pull_request.number && 'jvmPrValidationComparisonBenchmark' || 'jvmComparisonBenchmark' }}" - os: macos-latest - additional-task: '-x :benchmark:jvmBenchmark' + additional-task: "-x :benchmark:${{ github.event.pull_request.number && 'jvmPrValidationBenchmark' || 'jvmBenchmark' }}" - os: macos-13 # for macosX64 - additional-task: '-x :benchmark:jvmBenchmark' + additional-task: "-x :benchmark:${{ github.event.pull_request.number && 'jvmPrValidationBenchmark' || 'jvmBenchmark' }}" - os: windows-latest - additional-task: '-x :benchmark:jvmBenchmark' + additional-task: "-x :benchmark:${{ github.event.pull_request.number && 'jvmPrValidationBenchmark' || 'jvmBenchmark' }}" runs-on: ${{ matrix.os }} name: Run benchmarks on ${{ matrix.os }} env: