From d93625cf29dc6303b0f726d7b4129de9d9b7e122 Mon Sep 17 00:00:00 2001 From: Ben Manes Date: Sat, 19 Oct 2024 09:14:45 -0700 Subject: [PATCH] allow jmh benchmarks to be cached for faster ci runs --- .github/workflows/benchmarks.yml | 101 +++++++++++++++--- .../benmanes/caffeine/FactoryBenchmark.java | 2 +- .../caffeine/cache/BuilderBenchmark.java | 2 +- .../caffeine/cache/EvictionBenchmark.java | 2 +- .../cache/FrequencySketchBenchmark.java | 2 +- .../caffeine/cache/GetPutBenchmark.java | 3 +- .../caffeine/cache/PutRemoveBenchmark.java | 2 +- .../caffeine/cache/TimerWheelBenchmark.java | 2 +- gradle/libs.versions.toml | 6 +- .../kotlin/analyze/jmh.caffeine.gradle.kts | 2 +- jcache/build.gradle.kts | 2 +- 11 files changed, 99 insertions(+), 27 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 7116f0e1d3..b302e2cf10 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -46,7 +46,7 @@ jobs: java: ${{ matrix.java }} cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} arguments: > - caffeine:jmh --no-daemon -q + caffeine:jmh -q -PincludePattern=ComputeBenchmark -PbenchmarkParameters=computeType=Caffeine,Guava,ConcurrentHashMap - name: Upload Compute JMH Results to Gist @@ -54,7 +54,6 @@ jobs: if: > github.event_name == 'push' && github.event.repository.fork == false - && endsWith(github.ref, github.event.repository.default_branch) continue-on-error: true with: auth: ${{ secrets.GIST_TOKEN }} @@ -68,7 +67,7 @@ jobs: java: ${{ matrix.java }} cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} arguments: > - caffeine:jmh --no-daemon -q + caffeine:jmh -q -PincludePattern=GetPutBenchmark -PbenchmarkParameters=cacheType=Caffeine,Guava,ConcurrentHashMap - name: Upload Get/Put JMH Results to Gist @@ -76,7 +75,6 @@ jobs: if: > github.event_name == 'push' && github.event.repository.fork == false - && endsWith(github.ref, github.event.repository.default_branch) continue-on-error: true with: auth: ${{ secrets.GIST_TOKEN }} @@ -90,39 +88,112 @@ jobs: java: ${{ matrix.java }} cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} arguments: > - caffeine:jmh --no-daemon -q + caffeine:jmh -q -PincludePattern=PutRemoveBenchmark -PbenchmarkParameters=cacheType=Caffeine,Guava,ConcurrentHashMap + - name: Upload Put/Remove JMH Results to Gist + uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0 + if: > + github.event_name == 'push' + && github.event.repository.fork == false + continue-on-error: true + with: + auth: ${{ secrets.GIST_TOKEN }} + gist_description: Put/Remove JMH Results + gist_title: putremove_results_${{ matrix.java }}.json + github_file: ./caffeine/build/reports/jmh/results.json + gist_url: https://gist.githubusercontent.com/ben-manes/1359c399cb717e81976ee635c513340b + - name: EvictionBenchmark JMH Benchmark + uses: ./.github/actions/run-gradle + with: + java: ${{ matrix.java }} + cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} + arguments: > + caffeine:jmh -q + -PincludePattern=EvictionBenchmark + -PbenchmarkParameters=cacheType=Caffeine,Guava + - name: Upload EvictionBenchmark JMH Results to Gist + uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0 + if: > + github.event_name == 'push' + && github.event.repository.fork == false + continue-on-error: true + with: + auth: ${{ secrets.GIST_TOKEN }} + gist_description: Eviction JMH Results + gist_title: eviction_results_${{ matrix.java }}.json + github_file: ./caffeine/build/reports/jmh/results.json + gist_url: https://gist.githubusercontent.com/ben-manes/2f5b54540c2e67edc0d1a7f28a1139f5 - name: FrequencySketch JMH Benchmark uses: ./.github/actions/run-gradle with: java: ${{ matrix.java }} cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} arguments: > - caffeine:jmh --no-daemon -q + caffeine:jmh -q -PincludePattern=FrequencySketchBenchmark -PbenchmarkParameters=tableSize=134217728 + - name: Upload FrequencySketch JMH Results to Gist + uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0 + if: > + github.event_name == 'push' + && github.event.repository.fork == false + continue-on-error: true + with: + auth: ${{ secrets.GIST_TOKEN }} + gist_description: FrequencySketch JMH Results + gist_title: freq_results_${{ matrix.java }}.json + github_file: ./caffeine/build/reports/jmh/results.json + gist_url: https://gist.githubusercontent.com/ben-manes/f93df1b66aa210bc23a1e672ef9bee16 - name: TimerWheelBenchmark JMH Benchmark uses: ./.github/actions/run-gradle with: java: ${{ matrix.java }} cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }} - arguments: caffeine:jmh --no-daemon -q -PincludePattern=TimerWheelBenchmark + arguments: caffeine:jmh -q -PincludePattern=TimerWheelBenchmark + - name: Upload TimerWheelBenchmark JMH Results to Gist + uses: popsiclestick/gist-sync-action@88f8633178625914f2a01abf1a765f7272a580fa # v1.2.0 + if: > + github.event_name == 'push' + && github.event.repository.fork == false + continue-on-error: true + with: + auth: ${{ secrets.GIST_TOKEN }} + gist_description: TimerWheel JMH Results + gist_title: timerwheel_results_${{ matrix.java }}.json + github_file: ./caffeine/build/reports/jmh/results.json + gist_url: https://gist.githubusercontent.com/ben-manes/9abafdbc76fe34c0e63cef386b0b9ac0 - name: Publish JMH benchmarks if: > matrix.java == env.PUBLISH_JDK && github.event_name == 'push' - && endsWith(github.ref, github.event.repository.default_branch) run: | { echo '#### [Compute](https://jmh.morethan.io/?gists=511298014cc5629cbc5e57f09fd4c430)' - echo -n 'This benchmark that evaluates the overhead due to locking when the entry is' - echo -n 'present. The scenarios graphed are all threads retrieving a single entry' - echo -n '("sameKey") and threads retrieving different keys based on a Zipf distribution' - echo -n '("spread").' + echo -n 'This benchmark that evaluates the overhead due to locking when the entry is ' + echo -n 'present. The scenarios graphed are all threads retrieving a single entry ' + echo -n '("sameKey") and threads retrieving different keys based on a Zipf ' + echo -n 'distribution ("spread").' echo -e '\n' echo '#### [Get/Put](https://jmh.morethan.io/?gists=b231cf57cf8e144e2247716e777edcf3)' - echo -n 'A benchmark that evaluates the read/write performance of a cache. The cache is' - echo -n 'pre-populated for a 100% hit rate and a Zipf distribution of keys is used to' - echo -n 'mimic application usage patterns.' + echo -n 'A benchmark that evaluates the read/write performance of a cache. The cache ' + echo -n 'is pre-populated for a 100% hit rate and a Zipf distribution of keys is used ' + echo -n 'to mimic application usage patterns.' + echo -e '\n' + echo '#### [Put/Remove](https://jmh.morethan.io/?gists=1359c399cb717e81976ee635c513340b)' + echo -n 'A benchmark that evaluates the insert/update/remove performance of a cache. ' + echo -n 'A Zipf distribution of keys is used but it is unpredictable if the entry is ' + echo -n 'present given the competing operations.' + echo -e '\n' + echo '#### [Eviction](https://jmh.morethan.io/?gists=2f5b54540c2e67edc0d1a7f28a1139f5)' + echo -n 'A benchmark that evaluates the performance of a evictions by always inserting ' + echo -n 'a new entry.' + echo -e '\n' + echo '#### [FrequencySketch](https://jmh.morethan.io/?gists=f93df1b66aa210bc23a1e672ef9bee16)' + echo -n 'A benchmark that evaluates the performance of a frequency sketch for use by ' + echo -n 'size eviction.' + echo -e '\n' + echo '#### [TimerWheel](https://jmh.morethan.io/?gists=9abafdbc76fe34c0e63cef386b0b9ac0)' + echo -n 'A benchmark that evaluates the performance of the timer wheel for use by ' + echo -n 'variable expiration.' } >> $GITHUB_STEP_SUMMARY diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/FactoryBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/FactoryBenchmark.java index 2781547ddf..5a98a017b5 100644 --- a/caffeine/src/jmh/java/com/github/benmanes/caffeine/FactoryBenchmark.java +++ b/caffeine/src/jmh/java/com/github/benmanes/caffeine/FactoryBenchmark.java @@ -30,7 +30,7 @@ * This benchmark can be run by optionally specifying the target jvm in the command. *

* 

{@code
- *   JAVA_VERSION=20 ./gradlew jmh -PincludePattern=FactoryBenchmark --no-daemon
+ *   ./gradlew jmh -PincludePattern=FactoryBenchmark -q --rerun
  * }
* * @author ben.manes@gmail.com (Ben Manes) diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/BuilderBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/BuilderBenchmark.java index 74dae640ae..763682f284 100644 --- a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/BuilderBenchmark.java +++ b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/BuilderBenchmark.java @@ -31,7 +31,7 @@ /** *

* 

{@code
- * ./gradlew jmh -PincludePattern=BuilderBenchmark --no-daemon
+ *   ./gradlew jmh -PincludePattern=BuilderBenchmark -q --rerun
  * }
* * @author ben.manes@gmail.com (Ben Manes) diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/EvictionBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/EvictionBenchmark.java index d914bc2646..b5036905b4 100644 --- a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/EvictionBenchmark.java +++ b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/EvictionBenchmark.java @@ -28,7 +28,7 @@ * a 100% eviction rate to mimic worst case behavior. *

* 

{@code
- *   ./gradlew jmh -PincludePattern=EvictionBenchmark
+ *   ./gradlew jmh -PincludePattern=EvictionBenchmark -q --rerun
  * }
* * @author ben.manes@gmail.com (Ben Manes) diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/FrequencySketchBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/FrequencySketchBenchmark.java index 1d6cfe1eea..b24beef83c 100644 --- a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/FrequencySketchBenchmark.java +++ b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/FrequencySketchBenchmark.java @@ -29,7 +29,7 @@ /** * 
{@code
- *   ./gradlew jmh -PincludePattern=FrequencySketchBenchmark
+ *   ./gradlew jmh -PincludePattern=FrequencySketchBenchmark -q --rerun
  * }
* * @author ben.manes@gmail.com (Ben Manes) diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/GetPutBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/GetPutBenchmark.java index 92a61396a6..386a48ed22 100644 --- a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/GetPutBenchmark.java +++ b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/GetPutBenchmark.java @@ -35,7 +35,8 @@ * a 100% hit rate and a Zipf distribution of keys is used to mimic application usage patterns. *

* 

{@code
- *   ./gradlew jmh -PincludePattern=GetPutBenchmark
+ *   // JAVA_VERSION=?? for an alternative jdk
+ *   ./gradlew jmh -PincludePattern=GetPutBenchmark -q --rerun
  * }
* * @author ben.manes@gmail.com (Ben Manes) diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/PutRemoveBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/PutRemoveBenchmark.java index 4834407678..9ccc1cee6f 100644 --- a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/PutRemoveBenchmark.java +++ b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/PutRemoveBenchmark.java @@ -41,7 +41,7 @@ * concerns and should not be used to compare implementations. *

* 

{@code
- *   ./gradlew jmh -PincludePattern=PutRemoveBenchmark
+ *   ./gradlew jmh -PincludePattern=PutRemoveBenchmark -q --rerun
  * }
* * @author ben.manes@gmail.com (Ben Manes) diff --git a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/TimerWheelBenchmark.java b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/TimerWheelBenchmark.java index 62f77e930a..74881b3e66 100644 --- a/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/TimerWheelBenchmark.java +++ b/caffeine/src/jmh/java/com/github/benmanes/caffeine/cache/TimerWheelBenchmark.java @@ -27,7 +27,7 @@ /** * 
{@code
- *   ./gradlew jmh -PincludePattern=TimerWheelBenchmark
+ *   ./gradlew jmh -PincludePattern=TimerWheelBenchmark -q --rerun
  * }
* * @author ben.manes@gmail.com (Ben Manes) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0e9471247f..12354c1533 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,11 +23,11 @@ dependency-check = "10.0.4" eclipse-collections = "12.0.0.M3" ehcache3 = "3.10.8" errorprone = "2.34.0" -errorprone-plugin = "4.0.1" +errorprone-plugin = "4.1.0" errorprone-support = "0.18.0" expiring-map = "0.5.11" fast-filter = "1.0.2" -fastutil = "8.5.14" +fastutil = "8.5.15" felix-framework = "7.0.5" felix-scr = "2.2.12" findsecbugs = "1.13.0" @@ -69,7 +69,7 @@ lincheck = "2.34" mockito = "5.14.2" nexus-publish = "2.0.0" nullaway = "0.12.0" -nullaway-plugin = "2.0.0" +nullaway-plugin = "2.1.0" okhttp-bom = "4.12.0" okio-bom = "3.9.1" osgi-annotations = "1.5.1" diff --git a/gradle/plugins/src/main/kotlin/analyze/jmh.caffeine.gradle.kts b/gradle/plugins/src/main/kotlin/analyze/jmh.caffeine.gradle.kts index a757fa643d..dba68a95a1 100644 --- a/gradle/plugins/src/main/kotlin/analyze/jmh.caffeine.gradle.kts +++ b/gradle/plugins/src/main/kotlin/analyze/jmh.caffeine.gradle.kts @@ -59,11 +59,11 @@ jmhReport { jmhReportOutput = file(layout.buildDirectory.file("reports/jmh")).toString() } +// Use --rerun for repeated executions tasks.withType().configureEach { group = "Benchmarks" description = "Executes a Java microbenchmark" incompatibleWithConfigurationCache() - outputs.upToDateWhen { false } doFirst { if (!project.hasProperty("includePattern")) { diff --git a/jcache/build.gradle.kts b/jcache/build.gradle.kts index 667b93625f..fd9433019d 100644 --- a/jcache/build.gradle.kts +++ b/jcache/build.gradle.kts @@ -102,7 +102,7 @@ tasks.named("javadoc").configure { tasks.withType().configureEach { useJUnitPlatform() dependsOn(unzipTestKit) - testClassesDirs += layout.buildDirectory.files("tck") + testClassesDirs = files(testClassesDirs, layout.buildDirectory.files("tck")) project(":caffeine").plugins.withId("java-library") { val caffeineJar = project(":caffeine").tasks.named("jar")