From 76f3b09706181e32ff35f3e0970b55ff65ec6d2a Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Mon, 14 Nov 2022 09:19:39 -0500 Subject: [PATCH 01/12] [WIP] #4597 Java 17 support --- build.sbt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build.sbt b/build.sbt index 1d3f4d1010..269b328a29 100644 --- a/build.sbt +++ b/build.sbt @@ -223,6 +223,16 @@ val commonSettings = Def resolvers ++= Resolver.sonatypeOssRepos("public"), Test / javaOptions += "-Dscio.ignoreVersionWarning=true", Test / testOptions += Tests.Argument("-oD"), + Test / javaOptions ++= sys.props("java.version") match { + case v if v.startsWith("17.") => + Seq( + "--add-opens", + "java.base/java.util=ALL-UNNAMED", + "--add-opens", + "java.base/java.lang.invoke=ALL-UNNAMED" + ) + case _ => Seq.empty + }, testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v", "-a"), testOptions ++= { if (sys.env.contains("SLOW")) { From 772039ae58b863d4a66849f5035ad7f64909e5d5 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Thu, 19 Jan 2023 16:29:48 -0500 Subject: [PATCH 02/12] Test Java 17 GHA config --- .github/workflows/it-tests.yml | 1 + .github/workflows/tests.yml | 4 ++++ build.sbt | 10 ---------- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/it-tests.yml b/.github/workflows/it-tests.yml index 5f17bc6a71..d6f2caf5ec 100644 --- a/.github/workflows/it-tests.yml +++ b/.github/workflows/it-tests.yml @@ -31,6 +31,7 @@ jobs: java: - 8 - 11 + - 17 scala: - 2.12.17 - 2.13.8 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4ec4b684f0..05b52f9260 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,6 +18,9 @@ jobs: with: distribution: temurin java-version: ${{matrix.java}} + - if: ${{ matrix.java == 17}} + run: | + export JAVA_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" - if: ${{ !matrix.coverage }} run: sbt "++${{matrix.scala}} test" - if: ${{ matrix.coverage }} @@ -29,6 +32,7 @@ jobs: java: - 8 - 11 + - 17 scala: - 2.12.17 - 2.13.8 diff --git a/build.sbt b/build.sbt index 269b328a29..1d3f4d1010 100644 --- a/build.sbt +++ b/build.sbt @@ -223,16 +223,6 @@ val commonSettings = Def resolvers ++= Resolver.sonatypeOssRepos("public"), Test / javaOptions += "-Dscio.ignoreVersionWarning=true", Test / testOptions += Tests.Argument("-oD"), - Test / javaOptions ++= sys.props("java.version") match { - case v if v.startsWith("17.") => - Seq( - "--add-opens", - "java.base/java.util=ALL-UNNAMED", - "--add-opens", - "java.base/java.lang.invoke=ALL-UNNAMED" - ) - case _ => Seq.empty - }, testOptions += Tests.Argument(TestFrameworks.JUnit, "-q", "-v", "-a"), testOptions ++= { if (sys.env.contains("SLOW")) { From c602608a88bd00cb5cf6e1cf68c7ca2135710139 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Thu, 19 Jan 2023 21:21:12 -0500 Subject: [PATCH 03/12] debug --- .github/workflows/tests.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 05b52f9260..ad1a7e3238 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,14 +18,11 @@ jobs: with: distribution: temurin java-version: ${{matrix.java}} - - if: ${{ matrix.java == 17}} - run: | - export JAVA_OPTS="--add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED" - if: ${{ !matrix.coverage }} - run: sbt "++${{matrix.scala}} test" + run: sbt ${{matrix.java_opts}} "++${{matrix.scala}} test" - if: ${{ matrix.coverage }} run: | - sbt coverage "++${{matrix.scala}} test" coverageReport + sbt ${{matrix.java_opts}} coverage "++${{matrix.scala}} test" coverageReport bash <(curl -s https://codecov.io/bash) strategy: matrix: @@ -40,6 +37,12 @@ jobs: - scala: 2.13.8 java: 11 coverage: true + - java: 17 + java_opts: '-J--add-opens=java.base/java.util=ALL-UNNAMED -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED' + - java: 11 + java_opts: '' + - java: 8 + java_opts: '' exclude: - scala: 2.12.17 java: 8 From 03af44ec42ed5e33b32e89f4898f964a2e671b01 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Fri, 20 Jan 2023 11:40:53 -0500 Subject: [PATCH 04/12] Add java17Settings --- build.sbt | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 1d3f4d1010..3aa7a7f6fe 100644 --- a/build.sbt +++ b/build.sbt @@ -306,7 +306,8 @@ val commonSettings = Def ) ), mimaSettings, - formatSettings + formatSettings, + java17Settings ) lazy val publishSettings = Def.settings( @@ -452,6 +453,17 @@ def splitTests(tests: Seq[TestDefinition], filter: Seq[String], forkOptions: For } } +lazy val java17Settings = Seq(Test / javaOptions ++= (sys.props("java.version") match { + case v if v.startsWith("17.") => + Seq( + "--add-opens", + "java.base/java.util=ALL-UNNAMED", + "--add-opens", + "java.base/java.lang.invoke=ALL-UNNAMED" + ) + case _ => Seq.empty +})) + lazy val root: Project = Project("scio", file(".")) .settings(commonSettings) .settings( From 21f6c06fced94ed91b5077d6f166da4bb7a5e8f8 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Fri, 20 Jan 2023 12:25:06 -0500 Subject: [PATCH 05/12] Remove msg assertion from CoderTest --- .../src/test/scala/com/spotify/scio/coders/CoderTest.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala index 565a0b3cb1..6c08dbc720 100644 --- a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala +++ b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala @@ -207,14 +207,14 @@ final class CoderTest extends AnyFlatSpec with Matchers { ) val cw = new ClassWrapper() + try { cw.runWithImplicit throw new Throwable("Is expected to throw when passing implicit from outer class") } catch { - case e: NullPointerException => + case _: NullPointerException => // In this case outer field is called "$cw" and it is hard to wrap it with proper exception // so we allow it to fail with NullPointerException - e.getMessage should be(null) } { From b6d246812b503602e117b0a3cb276c0c9c31e7cd Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Fri, 20 Jan 2023 12:46:19 -0500 Subject: [PATCH 06/12] scalafmt --- .../src/test/scala/com/spotify/scio/coders/CoderTest.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala index 6c08dbc720..dc8fb93175 100644 --- a/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala +++ b/scio-test/src/test/scala/com/spotify/scio/coders/CoderTest.scala @@ -207,14 +207,13 @@ final class CoderTest extends AnyFlatSpec with Matchers { ) val cw = new ClassWrapper() - try { cw.runWithImplicit throw new Throwable("Is expected to throw when passing implicit from outer class") } catch { case _: NullPointerException => - // In this case outer field is called "$cw" and it is hard to wrap it with proper exception - // so we allow it to fail with NullPointerException + // In this case outer field is called "$cw" and it is hard to wrap it with proper exception + // so we allow it to fail with NullPointerException } { From 9f4445840ebdce9fa905dcc25668f13e0f87a085 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Tue, 31 Jan 2023 12:36:04 -0500 Subject: [PATCH 07/12] Simplify java_opts --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ad1a7e3238..2131b92166 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,16 +33,13 @@ jobs: scala: - 2.12.17 - 2.13.8 + java_opts: [] include: - scala: 2.13.8 java: 11 coverage: true - java: 17 java_opts: '-J--add-opens=java.base/java.util=ALL-UNNAMED -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED' - - java: 11 - java_opts: '' - - java: 8 - java_opts: '' exclude: - scala: 2.12.17 java: 8 From 9058a28cde0540df8c783b934dbdd68984482c95 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Tue, 31 Jan 2023 12:37:46 -0500 Subject: [PATCH 08/12] fixup config --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2131b92166..1735a96a51 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -33,7 +33,8 @@ jobs: scala: - 2.12.17 - 2.13.8 - java_opts: [] + java_opts: + - '' include: - scala: 2.13.8 java: 11 From f84a86938128ca3693fe5137f04fee43219a2c70 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Tue, 31 Jan 2023 12:50:12 -0500 Subject: [PATCH 09/12] Set .jvmopts in script --- .github/workflows/tests.yml | 10 ++++------ scripts/set_java_17_opts.sh | 8 ++++++++ 2 files changed, 12 insertions(+), 6 deletions(-) create mode 100755 scripts/set_java_17_opts.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1735a96a51..8d3210d119 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,11 +18,13 @@ jobs: with: distribution: temurin java-version: ${{matrix.java}} + - if: ${{ matrix.java == 17 }} + run: scripts/set_java_17_opts.sh - if: ${{ !matrix.coverage }} - run: sbt ${{matrix.java_opts}} "++${{matrix.scala}} test" + run: sbt "++${{matrix.scala}} test" - if: ${{ matrix.coverage }} run: | - sbt ${{matrix.java_opts}} coverage "++${{matrix.scala}} test" coverageReport + sbt coverage "++${{matrix.scala}} test" coverageReport bash <(curl -s https://codecov.io/bash) strategy: matrix: @@ -33,14 +35,10 @@ jobs: scala: - 2.12.17 - 2.13.8 - java_opts: - - '' include: - scala: 2.13.8 java: 11 coverage: true - - java: 17 - java_opts: '-J--add-opens=java.base/java.util=ALL-UNNAMED -J--add-opens=java.base/java.lang.invoke=ALL-UNNAMED' exclude: - scala: 2.12.17 java: 8 diff --git a/scripts/set_java_17_opts.sh b/scripts/set_java_17_opts.sh new file mode 100755 index 0000000000..b1d7518252 --- /dev/null +++ b/scripts/set_java_17_opts.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +cat <>.jvmopts +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +EOF From 31eb3b446c71753a72ce311c4657e58e71ae01b6 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Mon, 6 Feb 2023 15:43:57 -0500 Subject: [PATCH 10/12] Keep it simple, fork tests for java 17 --- .github/workflows/release.yml | 4 ++-- .github/workflows/tests.yml | 2 -- build.sbt | 19 ++++++++++++------- scripts/set_java_17_opts.sh | 8 -------- 4 files changed, 14 insertions(+), 19 deletions(-) delete mode 100755 scripts/set_java_17_opts.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b731b1137d..2b12470686 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,11 +12,11 @@ jobs: fetch-depth: 0 - name: cache SBT uses: coursier/cache-action@v6 - - name: Java 11 setup + - name: Java 17 setup uses: actions/setup-java@v3 with: distribution: temurin - java-version: 11 + java-version: 17 - uses: olafurpg/setup-gpg@v3 - name: Publish ${{ github.ref }} run: sbt ci-release diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8d3210d119..b04a809c73 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,8 +18,6 @@ jobs: with: distribution: temurin java-version: ${{matrix.java}} - - if: ${{ matrix.java == 17 }} - run: scripts/set_java_17_opts.sh - if: ${{ !matrix.coverage }} run: sbt "++${{matrix.scala}} test" - if: ${{ matrix.coverage }} diff --git a/build.sbt b/build.sbt index 3aa7a7f6fe..aa305aee26 100644 --- a/build.sbt +++ b/build.sbt @@ -215,6 +215,7 @@ val commonSettings = Def // this setting is not derived in sbt-tpolecat // https://github.com/typelevel/sbt-tpolecat/issues/36 inTask(doc)(TpolecatPlugin.projectSettings), + scalacOptions ++= Seq("-release", "8"), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint:unchecked"), Compile / doc / javacOptions := Seq("-source", "1.8"), excludeDependencies ++= Seq( @@ -453,16 +454,20 @@ def splitTests(tests: Seq[TestDefinition], filter: Seq[String], forkOptions: For } } -lazy val java17Settings = Seq(Test / javaOptions ++= (sys.props("java.version") match { +lazy val java17Settings = sys.props("java.version") match { case v if v.startsWith("17.") => + println("Setting java 17 opts") Seq( - "--add-opens", - "java.base/java.util=ALL-UNNAMED", - "--add-opens", - "java.base/java.lang.invoke=ALL-UNNAMED" + Test / fork := true, + Test / javaOptions ++= Seq( + "--add-opens", + "java.base/java.util=ALL-UNNAMED", + "--add-opens", + "java.base/java.lang.invoke=ALL-UNNAMED" + ) ) - case _ => Seq.empty -})) + case _ => Seq() +} lazy val root: Project = Project("scio", file(".")) .settings(commonSettings) diff --git a/scripts/set_java_17_opts.sh b/scripts/set_java_17_opts.sh deleted file mode 100755 index b1d7518252..0000000000 --- a/scripts/set_java_17_opts.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -e - -cat <>.jvmopts ---add-opens=java.base/java.util=ALL-UNNAMED ---add-opens=java.base/java.lang.invoke=ALL-UNNAMED -EOF From 3540d04f05098389e8f733e45d47db7e64410a4e Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Mon, 6 Feb 2023 16:05:31 -0500 Subject: [PATCH 11/12] remove scalacOptions --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index aa305aee26..4711d1b11d 100644 --- a/build.sbt +++ b/build.sbt @@ -215,7 +215,6 @@ val commonSettings = Def // this setting is not derived in sbt-tpolecat // https://github.com/typelevel/sbt-tpolecat/issues/36 inTask(doc)(TpolecatPlugin.projectSettings), - scalacOptions ++= Seq("-release", "8"), javacOptions ++= Seq("-source", "1.8", "-target", "1.8", "-Xlint:unchecked"), Compile / doc / javacOptions := Seq("-source", "1.8"), excludeDependencies ++= Seq( From 1a2696d56b04a8d71e1abc8694b531792b7dc44b Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Mon, 6 Feb 2023 16:30:13 -0500 Subject: [PATCH 12/12] remove debug --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index 4711d1b11d..3f3c4b3b52 100644 --- a/build.sbt +++ b/build.sbt @@ -455,7 +455,6 @@ def splitTests(tests: Seq[TestDefinition], filter: Seq[String], forkOptions: For lazy val java17Settings = sys.props("java.version") match { case v if v.startsWith("17.") => - println("Setting java 17 opts") Seq( Test / fork := true, Test / javaOptions ++= Seq(