8000 [SPARK-47645][BUILD][CORE][SQL][YARN] Make Spark build with `-release… · sweisdb/spark@3f56e1b · GitHub
[go: up one dir, main page]

Skip to content

Commit 3f56e1b

Browse files
LuciferYangsweisdb
authored andcommitted
[SPARK-47645][BUILD][CORE][SQL][YARN] Make Spark build with -release instead of -target
### What changes were proposed in this pull request? This pr makes the following changes to allow Spark to build with `-release` instead of `-target`: 1. Use `MethodHandle` instead of direct calls to `sun.security.action.GetBooleanAction` and `sun.util.calendar.ZoneInfo`, because they are not `exports` APIs. 2. `Channels.newReader` is used instead of ``,StreamDecoder.forDecoder because `StreamDecoder.forDecoder` is also not `exports` APIs. ```java public static Reader newReader(ReadableByteChannel ch, CharsetDecoder dec, int minBufferCap) { Objects.requireNonNull(ch, "ch"); return StreamDecoder.forDecoder(ch, dec.reset(), minBufferCap); } ``` 3. Adjusted the import of `java.io._` in `yarn/Client.scala` to fix the compilation error: ``` Error: ] /home/runner/work/spark/spark/resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:20: object FileSystem is not a member of package java.io ``` 4. Replaced `-target` with `-release` in `pom.xml` and `SparkBuild.scala`, and removed the `-source` option, because using `-release` is sufficient. 5. Upgrade `scala-maven-plugin` from 4.7.1 to 4.8.1 to fix the error `[ERROR] -release cannot be less than -target` when executing `build/mvn clean install -DskipTests -Djava.version=21` ### Why are the changes needed? After Scala 2.13.9, the compile option `-target` has been deprecated, it is recommended to use `-release`: - scala/scala#9982 ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass GitHub Actions ### Was this patch authored or co-authored using generative AI tooling? No Closes apache#45716 from LuciferYang/scala-maven-plugin-491. Authored-by: yangjie01 <yangjie01@baidu.com> Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
1 parent 4f8b94f commit 3f56e1b

File tree

7 files changed

+40
-29
lines changed

7 files changed

+40
-29
lines changed

core/src/main/scala/org/apache/spark/serializer/SerializationDebugger.scala

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,16 @@
1818
package org.apache.spark.serializer
1919

2020
import java.io._
21+
import java.lang.invoke.MethodHandles
2122
import java.lang.reflect.{Field, Method}
22-
import java.security.AccessController
23+
import java.security.{AccessController, PrivilegedAction}
2324

2425
import scala.annotation.tailrec
2526
import scala.collection.mutable
2627
import scala.util.control.NonFatal
2728

2829
import org.apache.spark.internal.Logging
30+
import org.apache.spark.util.SparkClassUtils
2931

3032
private[spark] object SerializationDebugger extends Logging {
3133

@@ -68,8 +70,13 @@ private[spark] object SerializationDebugger extends Logging {
6870
}
6971

7072
private[serializer] var enableDebugging: Boolean = {
71-
!AccessController.doPrivileged(new sun.security.action.GetBooleanAction(
72-
"sun.io.serialization.extendedDebugInfo")).booleanValue()
73+
val lookup = MethodHandles.lookup()
74+
val clazz = SparkClassUtils.classForName("sun.security.action.GetBooleanAction")
75+
val constructor = clazz.getConstructor(classOf[String])
76+
val mh = lookup.unreflectConstructor(constructor)
77+
val action = mh.invoke("sun.io.serialization.extendedDebugInfo")
78+
.asInstanceOf[PrivilegedAction[Boolean]]
79+
!AccessController.doPrivileged(action).booleanValue()
7380
}
7481

7582
private class SerializationDebugger {

pom.xml

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,7 @@
114114
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
115115
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
116116
<java.version>17</java.version>
117-
<maven.compiler.source>${java.version}</maven.compiler.source>
118-
<maven.compiler.target>${java.version}</maven.compiler.target>
117+
<maven.compiler.release>${java.version}</maven.compiler.release>
119118
<maven.version>3.9.6</maven.version>
120119
<exec-maven-plugin.version>3.1.0</exec-maven-plugin.version>
121120
<sbt.project.name>spark</sbt.project.name>
@@ -175,8 +174,7 @@
175174
<scala.version>2.13.13</scala.version>
176175
<scala.binary.version>2.13</scala.binary.version>
177176
<scalatest-maven-plugin.version>2.2.0</scalatest-maven-plugin.version>
178-
<!-- don't upgrade scala-maven-plugin to version 4.7.2 or higher, see SPARK-45144 for details -->
179-
<scala-maven-plugin.version>4.7.1</scala-maven-plugin.version>
177+
<scala-maven-plugin.version>4.8.1</scala-maven-plugin.version>
180178
<maven.scaladoc.skip>false</maven.scaladoc.skip>
181179
<versions-maven-plugin.version>2.16.2</versions-maven-plugin.version>
182180
<!-- for now, not running scalafmt as part of default verify pipeline -->
@@ -3060,7 +3058,8 @@
30603058
<arg>-deprecation</arg>
30613059
<arg>-feature</arg>
30623060
<arg>-explaintypes</arg>
3063-
<arg>-target:17</arg>
3061+
<arg>-release</arg>
3062+
<arg>17</arg>
30643063
<arg>-Wconf:cat=deprecation:wv,any:e</arg>
30653064
<arg>-Wunused:imports</arg>
30663065
<arg>-Wconf:cat=scaladoc:wv</arg>
@@ -3092,9 +3091,7 @@
30923091
<jvmArg>-XX:ReservedCodeCacheSize=${CodeCacheSize}</jvmArg>
30933092
</jvmArgs>
30943093
<javacArgs>
3095-
<javacArg>-source</javacArg>
3096-
<javacArg>${java.version}</javacArg>
3097-
<javacArg>-target</javacArg>
3094+
<javacArg>--release</javacArg>
30983095
<javacArg>${java.version}</javacArg>
30993096
<javacArg>-Xlint:all,-serial,-path,-try</javacArg>
31003097
</javacArgs>
@@ -3105,8 +3102,7 @@
31053102
<artifactId>maven-compiler-plugin</artifactId>
31063103
<version>3.12.1</version>
31073104
<configuration>
3108-
<source>${java.version}</source>
3109-
<target>${java.version}</target>
3105+
<release>${java.version}</release>
31103106
<skipMain>true</skipMain> <!-- skip compile -->
31113107
<skip>true</skip> <!-- skip testCompile -->
31123108
</configuration>

project/SparkBuild.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -311,18 +311,17 @@ object SparkBuild extends PomBuild {
311311

312312
(Compile / javacOptions) ++= Seq(
313313
"-encoding", UTF_8.name(),
314-
"-source", javaVersion.value
314+
"--release", javaVersion.value
315315
),
316316
// This -target and Xlint:unchecked options cannot be set in the Compile configuration scope since
317317
// `javadoc` doesn't play nicely with them; see https://github.com/sbt/sbt/issues/355#issuecomment-3817629
318318
// for additional discussion and explanation.
319319
(Compile / compile / javacOptions) ++= Seq(
320-
"-target", javaVersion.value,
321320
"-Xlint:unchecked"
322321
),
323322

324323
(Compile / scalacOptions) ++= Seq(
325-
s"-target:${javaVersion.value}",
324+
"-release", javaVersion.value,
326325
"-sourcepath", (ThisBuild / baseDirectory).value.getAbsolutePath // Required for relative source links in scaladoc
327326
),
328327

resource-managers/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.spark.deploy.yarn
1919

20-
import java.io.{FileSystem => _, _}
20+
import java.io.{File, FileFilter, FileNotFoundException, FileOutputStream, InterruptedIOException, IOException, OutputStreamWriter}
2121
import java.net.{InetAddress, UnknownHostException, URI, URL}
2222
import java.nio.ByteBuffer
2323
import java.nio.charset.StandardCharsets

sql/api/src/main/scala/org/apache/spark/sql/catalyst/util/SparkDateTimeUtils.scala

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717
package org.apache.spark.sql.catalyst.util
1818

19+
import java.lang.invoke.{MethodHandles, MethodType}
1920
import java.sql.{Date, Timestamp}
2021
import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZonedDateTime, ZoneId, ZoneOffset}
2122
import java.util.TimeZone
@@ -24,14 +25,13 @@ import java.util.regex.Pattern
2425

2526
import scala.util.control.NonFatal
2627

27-
import sun.util.calendar.ZoneInfo
28-
2928
import org.apache.spark.QueryContext
3029
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
3130
import org.apache.spark.sql.catalyst.util.RebaseDateTime.{rebaseGregorianToJulianDays, rebaseGregorianToJulianMicros, rebaseJulianToGregorianDays, rebaseJulianToGregorianMicros}
3231
import org.apache.spark.sql.errors.ExecutionErrors
3332
import org.apache.spark.sql.types.{DateType, TimestampType}
3433
import org.apache.spark.unsafe.types.UTF8String
34+
import org.apache.spark.util.SparkClassUtils
3535

3636
trait SparkDateTimeUtils {
3737

@@ -197,6 +197,15 @@ trait SparkDateTimeUtils {
197197
rebaseJulianToGregorianDays(julianDays)
198198
}
199199

200+
private val zoneInfoClassName = "sun.util.calendar.ZoneInfo"
201+
private val getOffsetsByWallHandle = {
202+
val lookup = MethodHandles.lookup()
203+
val classType = SparkClassUtils.classForName(zoneInfoClassName)
204+
val methodName = "getOffsetsByWall"
205+
val methodType = MethodType.methodType(classOf[Int], classOf[Long], classOf[Array[Int]])
206+
lookup.findVirtual(classType, methodName, methodType)
207+
}
208+
200209
/**
201210
* Converts days since the epoch 1970-01-01 in Proleptic Gregorian calendar to a local date
202211
* at the default JVM time zone in the hybrid calendar (Julian + Gregorian). It rebases the given
@@ -215,8 +224,10 @@ trait SparkDateTimeUtils {
215224
val rebasedDays = rebaseGregorianToJulianDays(days)
216225
val localMillis = Math.multiplyExact(rebasedDays, MILLIS_PER_DAY)
217226
val timeZoneOffset = TimeZone.getDefault match {
218-
case zoneInfo: ZoneInfo => zoneInfo.getOffsetsByWall(localMillis, null)
219-
case timeZone: TimeZone => timeZone.getOffset(localMillis - timeZone.getRawOffset)
227+
case zoneInfo: TimeZone if zoneInfo.getClass.getName == zoneInfoClassName =>
228+
getOffsetsByWallHandle.invoke(zoneInfo, localMillis, null).asInstanceOf[Int]
229+
10000 case timeZone: TimeZone =>
230+
timeZone.getOffset(localMillis - timeZone.getRawOffset)
220231
}
221232
new Date(localMillis - timeZoneOffset)
222233
}

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/CreateJacksonParser.scala

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,13 @@
1717

1818
package org.apache.spark.sql.catalyst.json
1919

20-
import java.io.{ByteArrayInputStream, InputStream, InputStreamReader}
20+
import java.io.{ByteArrayInputStream, InputStream, InputStreamReader, Reader}
2121
import java.nio.channels.Channels
2222
import java.nio.charset.Charset
2323
import java.nio.charset.StandardCharsets
2424

2525
import com.fasterxml.jackson.core.{JsonFactory, JsonParser}
2626
import org.apache.hadoop.io.Text
27-
import sun.nio.cs.StreamDecoder
2827

2928
import org.apache.spark.sql.catalyst.InternalRow
3029
import org.apache.spark.unsafe.types.UTF8String
@@ -58,13 +57,13 @@ object CreateJacksonParser extends Serializable {
5857
// a reader with specific encoding.
5958
// The method creates a reader for an array with given encoding and sets size of internal
6059
// decoding buffer according to size of input array.
61-
private def getStreamDecoder(enc: String, in: Array[Byte], length: Int): StreamDecoder = {
60+
private def getStreamDecoder(enc: String, in: Array[Byte], length: Int): Reader = {
6261
val bais = new ByteArrayInputStream(in, 0, length)
6362
val byteChannel = Channels.newChannel(bais)
6463
val decodingBufferSize = Math.min(length, 8192)
6564
val decoder = Charset.forName(enc).newDecoder()
6665

67-
StreamDecoder.forDecoder(byteChannel, decoder, decodingBufferSize)
66+
Channels.newReader(byteChannel, decoder, decodingBufferSize)
6867
}
6968

7069
def text(enc: String, jsonFactory: JsonFactory, record: Text): JsonParser = {

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/xml/CreateXmlParser.scala

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,13 @@
1717

1818
package org.apache.spark.sql.catalyst.xml
1919

20-
import java.io.{ByteArrayInputStream, InputStream, InputStreamReader, StringReader}
20+
import java.io.{ByteArrayInputStream, InputStream, InputStreamReader, Reader, StringReader}
2121
import java.nio.channels.Channels
2222
import java.nio.charset.{Charset, StandardCharsets}
2323
import javax.xml.stream.{EventFilter, XMLEventReader, XMLInputFactory, XMLStreamConstants}
2424
import javax.xml.stream.events.XMLEvent
2525

2626
import org.apache.hadoop.io.Text
27-
import sun.nio.cs.StreamDecoder
2827

2928
import org.apache.spark.sql.catalyst.InternalRow
3029
import org.apache.spark.unsafe.types.UTF8String
@@ -75,13 +74,13 @@ object CreateXmlParser extends Serializable {
7574
// a reader with specific encoding.
7675
// The method creates a reader for an array with given encoding and sets size of internal
7776
// decoding buffer according to size of input array.
78-
private def getStreamDecoder(enc: String, in: Array[Byte], length: Int): StreamDecoder = {
77+
private def getStreamDecoder(enc: String, in: Array[Byte], length: Int): Reader = {
7978
val bais = new ByteArrayInputStream(in, 0, length)
8079
val byteChannel = Channels.newChannel(bais)
8180
val decodingBufferSize = Math.min(length, 8192)
8281
val decoder = Charset.forName(enc).newDecoder()
8382

84-
StreamDecoder.forDecoder(byteChannel, decoder, decodingBufferSize)
83+
Channels.newReader(byteChannel, decoder, decodingBufferSize)
8584
}
8685

8786
def text(enc: String, xmlInputFactory: XMLInputFactory, record: Text): XMLEventReader = {

0 commit comments

Comments
 (0)
0