已开启
Fix CVE-2022-31777, CVE-2023-22946, CVE-2023-32007, CVE-2025-54920, CVE-2025-55039 #85
chenyanpan创建于 8月17日
Fix CVE-2022-31777, CVE-2023-22946, CVE-2023-32007, CVE-2025-54920, CVE-2025-55039 #85
已开启
共 11 个文件变更+8093-1
| @@ -0,0 +1,51 @@ | |||
| 1 | +From 14bc8ec1d0ce5fcafd6b85b02cba2a4880b6a41d Mon Sep 17 00:00:00 2001 | ||
| 2 | +From: Sean Owen <srowen@gmail.com> | ||
| 3 | +Date: Fri, 17 Jun 2022 09:36:49 -0700 | ||
| 4 | +Subject: [PATCH] [SPARK-39505][UI] Escape log content rendered in UI | ||
| 5 | + | ||
| 6 | +### What changes were proposed in this pull request? | ||
| 7 | + | ||
| 8 | +Escape log content rendered to the UI. | ||
| 9 | + | ||
| 10 | +### Why are the changes needed? | ||
| 11 | + | ||
| 12 | +Log content may contain reserved characters or other code in the log and be misinterpreted in the UI as HTML. | ||
| 13 | + | ||
| 14 | +### Does this PR introduce _any_ user-facing change? | ||
| 15 | + | ||
| 16 | +No | ||
| 17 | + | ||
| 18 | +### How was this patch tested? | ||
| 19 | + | ||
| 20 | +Existing tests | ||
| 21 | + | ||
| 22 | +Closes #36902 from srowen/LogViewEscape. | ||
| 23 | + | ||
| 24 | +Authored-by: Sean Owen <srowen@gmail.com> | ||
| 25 | +Signed-off-by: Dongjoon Hyun <dongjoon@apache.org> | ||
| 26 | +--- | ||
| 27 | + .../src/main/resources/org/apache/spark/ui/static/log-view.js | 4 ++-- | ||
| 28 | + 1 file changed, 2 insertions(+), 2 deletions(-) | ||
| 29 | + | ||
| 30 | +diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js | ||
| 31 | +index b5c43e5788bc..163662bf3bb5 100644 | ||
| 32 | +--- a/core/src/main/resources/org/apache/spark/ui/static/log-view.js | ||
| 33 | ++++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js | ||
| 34 | +@@ -82,7 +82,7 @@ function loadMore() { | ||
| 35 | + if (retStartByte == 0) { | ||
| 36 | + disableMoreButton(); | ||
| 37 | + } | ||
| 38 | +- $("pre", ".log-content").prepend(cleanData); | ||
| 39 | ++ $("pre", ".log-content").prepend(document.createTextNode(cleanData)); | ||
| 40 | + | ||
| 41 | + curLogLength = curLogLength + (startByte - retStartByte); | ||
| 42 | + startByte = retStartByte; | ||
| 43 | +@@ -112,7 +112,7 @@ function loadNew() { | ||
| 44 | + var retLogLength = dataInfo[2]; | ||
| 45 | + | ||
| 46 | + var cleanData = data.substring(newlineIndex + 1); | ||
| 47 | +- $("pre", ".log-content").append(cleanData); | ||
| 48 | ++ $("pre", ".log-content").append(document.createTextNode(cleanData)); | ||
| 49 | + | ||
| 50 | + curLogLength = curLogLength + (retEndByte - retStartByte); | ||
| 51 | + endByte = retEndByte; | ||
| @@ -0,0 +1,537 @@ | |||
| 1 | +From 78a5825fe266c0884d2dd18cbca9625fa258d7f7 Mon Sep 17 00:00:00 2001 | ||
| 2 | +From: Dongjoon Hyun <dongjoon@apache.org> | ||
| 3 | +Date: Mon, 11 Jul 2022 15:13:02 +0000 | ||
| 4 | +Subject: [PATCH] Preparing Spark release v3.2.2-rc1 | ||
| 5 | + | ||
| 6 | +--- | ||
| 7 | + assembly/pom.xml | 2 +- | ||
| 8 | + common/kvstore/pom.xml | 2 +- | ||
| 9 | + common/network-common/pom.xml | 2 +- | ||
| 10 | + common/network-shuffle/pom.xml | 2 +- | ||
| 11 | + common/network-yarn/pom.xml | 2 +- | ||
| 12 | + common/sketch/pom.xml | 2 +- | ||
| 13 | + common/tags/pom.xml | 2 +- | ||
| 14 | + common/unsafe/pom.xml | 2 +- | ||
| 15 | + core/pom.xml | 2 +- | ||
| 16 | + docs/_config.yml | 2 +- | ||
| 17 | + examples/pom.xml | 2 +- | ||
| 18 | + external/avro/pom.xml | 2 +- | ||
| 19 | + external/docker-integration-tests/pom.xml | 2 +- | ||
| 20 | + external/kafka-0-10-assembly/pom.xml | 2 +- | ||
| 21 | + external/kafka-0-10-sql/pom.xml | 2 +- | ||
| 22 | + external/kafka-0-10-token-provider/pom.xml | 2 +- | ||
| 23 | + external/kafka-0-10/pom.xml | 2 +- | ||
| 24 | + external/kinesis-asl-assembly/pom.xml | 2 +- | ||
| 25 | + external/kinesis-asl/pom.xml | 2 +- | ||
| 26 | + external/spark-ganglia-lgpl/pom.xml | 2 +- | ||
| 27 | + graphx/pom.xml | 2 +- | ||
| 28 | + hadoop-cloud/pom.xml | 2 +- | ||
| 29 | + launcher/pom.xml | 2 +- | ||
| 30 | + mllib-local/pom.xml | 2 +- | ||
| 31 | + mllib/pom.xml | 2 +- | ||
| 32 | + pom.xml | 2 +- | ||
| 33 | + python/pyspark/version.py | 2 +- | ||
| 34 | + repl/pom.xml | 2 +- | ||
| 35 | + resource-managers/kubernetes/core/pom.xml | 2 +- | ||
| 36 | + resource-managers/kubernetes/integration-tests/pom.xml | 2 +- | ||
| 37 | + resource-managers/mesos/pom.xml | 2 +- | ||
| 38 | + resource-managers/yarn/pom.xml | 2 +- | ||
| 39 | + sql/catalyst/pom.xml | 2 +- | ||
| 40 | + sql/core/pom.xml | 2 +- | ||
| 41 | + sql/hive-thriftserver/pom.xml | 2 +- | ||
| 42 | + sql/hive/pom.xml | 2 +- | ||
| 43 | + streaming/pom.xml | 2 +- | ||
| 44 | + tools/pom.xml | 2 +- | ||
| 45 | + 38 files changed, 38 insertions(+), 38 deletions(-) | ||
| 46 | + | ||
| 47 | +diff --git a/assembly/pom.xml b/assembly/pom.xml | ||
| 48 | +index 9584884b003f..7248e549a9cd 100644 | ||
| 49 | +--- a/assembly/pom.xml | ||
| 50 | ++++ b/assembly/pom.xml | ||
| 51 | +@@ -21,7 +21,7 @@ | ||
| 52 | + <parent> | ||
| 53 | + <groupId>org.apache.spark</groupId> | ||
| 54 | + <artifactId>spark-parent_2.12</artifactId> | ||
| 55 | +- <version>3.2.2-SNAPSHOT</version> | ||
| 56 | ++ <version>3.2.2</version> | ||
| 57 | + <relativePath>../pom.xml</relativePath> | ||
| 58 | + </parent> | ||
| 59 | + | ||
| 60 | +diff --git a/common/kvstore/pom.xml b/common/kvstore/pom.xml | ||
| 61 | +index 167e69f9e6ed..42f54b5f1109 100644 | ||
| 62 | +--- a/common/kvstore/pom.xml | ||
| 63 | ++++ b/common/kvstore/pom.xml | ||
| 64 | +@@ -22,7 +22,7 @@ | ||
| 65 | + <parent> | ||
| 66 | + <groupId>org.apache.spark</groupId> | ||
| 67 | + <artifactId>spark-parent_2.12</artifactId> | ||
| 68 | +- <version>3.2.2-SNAPSHOT</version> | ||
| 69 | ++ <version>3.2.2</version> | ||
| 70 | + <relativePath>../../pom.xml</relativePath> | ||
| 71 | + </parent> | ||
| 72 | + | ||
| 73 | +diff --git a/common/network-common/pom.xml b/common/network-common/pom.xml | ||
| 74 | +index eaf1c1e12e22..d74828a18b00 100644 | ||
| 75 | +--- a/common/network-common/pom.xml | ||
| 76 | ++++ b/common/network-common/pom.xml | ||
| 77 | +@@ -22,7 +22,7 @@ | ||
| 78 | + <parent> | ||
| 79 | + <groupId>org.apache.spark</groupId> | ||
| 80 | + <artifactId>spark-parent_2.12</artifactId> | ||
| 81 | +- <version>3.2.2-SNAPSHOT</version> | ||
| 82 | ++ <version>3.2.2</version> | ||
| 83 | + <relativePath>../../pom.xml</relativePath> | ||
| 84 | + </parent> | ||
| 85 | + | ||
| 86 | +diff --git a/common/network-shuffle/pom.xml b/common/network-shuffle/pom.xml | ||
| 87 | +index 811e5035c465..7e73e2e1bc44 100644 | ||
| 88 | +--- a/common/network-shuffle/pom.xml | ||
| 89 | ++++ b/common/network-shuffle/pom.xml | ||
| 90 | +@@ -22,7 +22,7 @@ | ||
| 91 | + <parent> | ||
| 92 | + <groupId>org.apache.spark</groupId> | ||
| 93 | + <artifactId>spark-parent_2.12</artifactId> | ||
| 94 | +- <version>3.2.2-SNAPSHOT</version> | ||
| 95 | ++ <version>3.2.2</version> | ||
| 96 | + <relativePath>../../pom.xml</relativePath> | ||
| 97 | + </parent> | ||
| 98 | + | ||
| 99 | +diff --git a/common/network-yarn/pom.xml b/common/network-yarn/pom.xml | ||
| 100 | +index 23513f604394..f7e6a344d7d3 100644 | ||
| @@ -0,0 +1,51 @@ | |||
| 1 | +From 37dbf1cc99ab888342d04b0bf41619ff597ddab9 Mon Sep 17 00:00:00 2001 | ||
| 2 | +From: Sean Owen <srowen@gmail.com> | ||
| 3 | +Date: Fri, 17 Jun 2022 09:36:49 -0700 | ||
| 4 | +Subject: [PATCH] [SPARK-39505][UI] Escape log content rendered in UI | ||
| 5 | + | ||
| 6 | +### What changes were proposed in this pull request? | ||
| 7 | + | ||
| 8 | +Escape log content rendered to the UI. | ||
| 9 | + | ||
| 10 | +### Why are the changes needed? | ||
| 11 | + | ||
| 12 | +Log content may contain reserved characters or other code in the log and be misinterpreted in the UI as HTML. | ||
| 13 | + | ||
| 14 | +### Does this PR introduce _any_ user-facing change? | ||
| 15 | + | ||
| 16 | +No | ||
| 17 | + | ||
| 18 | +### How was this patch tested? | ||
| 19 | + | ||
| 20 | +Existing tests | ||
| 21 | + | ||
| 22 | +Closes #36902 from srowen/LogViewEscape. | ||
| 23 | + | ||
| 24 | +Authored-by: Sean Owen <srowen@gmail.com> | ||
| 25 | +Signed-off-by: Dongjoon Hyun <dongjoon@apache.org> | ||
| 26 | +--- | ||
| 27 | + .../src/main/resources/org/apache/spark/ui/static/log-view.js | 4 ++-- | ||
| 28 | + 1 file changed, 2 insertions(+), 2 deletions(-) | ||
| 29 | + | ||
| 30 | +diff --git a/core/src/main/resources/org/apache/spark/ui/static/log-view.js b/core/src/main/resources/org/apache/spark/ui/static/log-view.js | ||
| 31 | +index 2f416d8210e1..92df087f4bd9 100644 | ||
| 32 | +--- a/core/src/main/resources/org/apache/spark/ui/static/log-view.js | ||
| 33 | ++++ b/core/src/main/resources/org/apache/spark/ui/static/log-view.js | ||
| 34 | +@@ -85,7 +85,7 @@ function loadMore() { | ||
| 35 | + if (retStartByte == 0) { | ||
| 36 | + disableMoreButton(); | ||
| 37 | + } | ||
| 38 | +- $("pre", ".log-content").prepend(cleanData); | ||
| 39 | ++ $("pre", ".log-content").prepend(document.createTextNode(cleanData)); | ||
| 40 | + | ||
| 41 | + curLogLength = curLogLength + (startByte - retStartByte); | ||
| 42 | + startByte = retStartByte; | ||
| 43 | +@@ -115,7 +115,7 @@ function loadNew() { | ||
| 44 | + var retLogLength = dataInfo[2]; | ||
| 45 | + | ||
| 46 | + var cleanData = data.substring(newlineIndex + 1); | ||
| 47 | +- $("pre", ".log-content").append(cleanData); | ||
| 48 | ++ $("pre", ".log-content").append(document.createTextNode(cleanData)); | ||
| 49 | + | ||
| 50 | + curLogLength = curLogLength + (retEndByte - retStartByte); | ||
| 51 | + endByte = retEndByte; | ||
| @@ -0,0 +1,88 @@ | |||
| 1 | +From 909da96e1471886a01a9e1def93630c4fd40e74a Mon Sep 17 00:00:00 2001 | ||
| 2 | +From: Peter Toth <peter.toth@gmail.com> | ||
| 3 | +Date: Tue, 10 Jan 2023 18:05:52 +0900 | ||
| 4 | +Subject: [PATCH] [SPARK-41958][CORE] Disallow arbitrary custom classpath with | ||
| 5 | + proxy user in cluster mode | ||
| 6 | + | ||
| 7 | +### What changes were proposed in this pull request? | ||
| 8 | + | ||
| 9 | +This PR proposes to disallow arbitrary custom classpath with proxy user in cluster mode by default. | ||
| 10 | + | ||
| 11 | +### Why are the changes needed? | ||
| 12 | + | ||
| 13 | +To avoid arbitrary classpath in spark cluster. | ||
| 14 | + | ||
| 15 | +### Does this PR introduce _any_ user-facing change? | ||
| 16 | + | ||
| 17 | +Yes. User should reenable this feature by `spark.submit.proxyUser.allowCustomClasspathInClusterMode`. | ||
| 18 | + | ||
| 19 | +### How was this patch tested? | ||
| 20 | + | ||
| 21 | +Manually tested. | ||
| 22 | + | ||
| 23 | +Closes #39474 from Ngone51/dev. | ||
| 24 | + | ||
| 25 | +Lead-authored-by: Peter Toth <peter.toth@gmail.com> | ||
| 26 | +Co-authored-by: Yi Wu <yi.wu@databricks.com> | ||
| 27 | +Signed-off-by: Hyukjin Kwon <gurwls223@apache.org> | ||
| 28 | +--- | ||
| 29 | + .../org/apache/spark/deploy/SparkSubmit.scala | 15 +++++++++++++++ | ||
| 30 | + .../apache/spark/internal/config/package.scala | 7 +++++++ | ||
| 31 | + 2 files changed, 22 insertions(+) | ||
| 32 | + | ||
| 33 | +diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | ||
| 34 | +index 65e4367b33af..a701b0ea6070 100644 | ||
| 35 | +--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | ||
| 36 | ++++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala | ||
| 37 | +@@ -306,6 +306,10 @@ private[spark] class SparkSubmit extends Logging { | ||
| 38 | + val isKubernetesClient = clusterManager == KUBERNETES && deployMode == CLIENT | ||
| 39 | + val isKubernetesClusterModeDriver = isKubernetesClient && | ||
| 40 | + sparkConf.getBoolean("spark.kubernetes.submitInDriver", false) | ||
| 41 | ++ val isCustomClasspathInClusterModeDisallowed = | ||
| 42 | ++ !sparkConf.get(ALLOW_CUSTOM_CLASSPATH_BY_PROXY_USER_IN_CLUSTER_MODE) && | ||
| 43 | ++ args.proxyUser != null && | ||
| 44 | ++ (isYarnCluster || isMesosCluster || isStandAloneCluster || isKubernetesCluster) | ||
| 45 | + | ||
| 46 | + if (!isMesosCluster && !isStandAloneCluster) { | ||
| 47 | + // Resolve maven dependencies if there are any and add classpath to jars. Add them to py-files | ||
| 48 | +@@ -887,6 +891,13 @@ private[spark] class SparkSubmit extends Logging { | ||
| 49 | + | ||
| 50 | + sparkConf.set("spark.app.submitTime", System.currentTimeMillis().toString) | ||
| 51 | + | ||
| 52 | ++ if (childClasspath.nonEmpty && isCustomClasspathInClusterModeDisallowed) { | ||
| 53 | ++ childClasspath.clear() | ||
| 54 | ++ logWarning(s"Ignore classpath ${childClasspath.mkString(", ")} with proxy user specified " + | ||
| 55 | ++ s"in Cluster mode when ${ALLOW_CUSTOM_CLASSPATH_BY_PROXY_USER_IN_CLUSTER_MODE.key} is " + | ||
| 56 | ++ s"disabled") | ||
| 57 | ++ } | ||
| 58 | ++ | ||
| 59 | + (childArgs.toSeq, childClasspath.toSeq, sparkConf, childMainClass) | ||
| 60 | + } | ||
| 61 | + | ||
| 62 | +@@ -940,6 +951,10 @@ private[spark] class SparkSubmit extends Logging { | ||
| 63 | + logInfo(s"Classpath elements:\n${childClasspath.mkString("\n")}") | ||
| 64 | + logInfo("\n") | ||
| 65 | + } | ||
| 66 | ++ assert(!(args.deployMode == "cluster" && args.proxyUser != null && childClasspath.nonEmpty) || | ||
| 67 | ++ sparkConf.get(ALLOW_CUSTOM_CLASSPATH_BY_PROXY_USER_IN_CLUSTER_MODE), | ||
| 68 | ++ s"Classpath of spark-submit should not change in cluster mode if proxy user is specified " + | ||
| 69 | ++ s"when ${ALLOW_CUSTOM_CLASSPATH_BY_PROXY_USER_IN_CLUSTER_MODE.key} is disabled") | ||
| 70 | + val loader = getSubmitClassLoader(sparkConf) | ||
| 71 | + for (jar <- childClasspath) { | ||
| 72 | + addJarToClasspath(jar, loader) | ||
| 73 | +diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala | ||
| 74 | +index eb6ac8b765b0..be210cfe59b3 100644 | ||
| 75 | +--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala | ||
| 76 | ++++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala | ||
| 77 | +@@ -2461,4 +2461,11 @@ package object config { | ||
| 78 | + .version("3.4.0") | ||
| 79 | + .timeConf(TimeUnit.MILLISECONDS) | ||
| 80 | + .createWithDefaultString("5s") | ||
| 81 | ++ | ||
| 82 | ++ private[spark] val ALLOW_CUSTOM_CLASSPATH_BY_PROXY_USER_IN_CLUSTER_MODE = | ||
| 83 | ++ ConfigBuilder("spark.submit.proxyUser.allowCustomClasspathInClusterMode") | ||
| 84 | ++ .internal() | ||
| 85 | ++ .version("3.4.0") | ||
| 86 | ++ .booleanConf | ||
| 87 | ++ .createWithDefault(false) | ||
| 88 | + } | ||
| @@ -0,0 +1,55 @@ | |||
| 1 | +From 1d524a88f6e93e9971a09f70eb2804dca51d578c Mon Sep 17 00:00:00 2001 | ||
| 2 | +From: Hyukjin Kwon <gurwls223@apache.org> | ||
| 3 | +Date: Fri, 22 Apr 2022 19:01:05 +0900 | ||
| 4 | +Subject: [PATCH] [SPARK-38992][CORE] Avoid using bash -c in | ||
| 5 | + ShellBasedGroupsMappingProvider | ||
| 6 | + | ||
| 7 | +### What changes were proposed in this pull request? | ||
| 8 | + | ||
| 9 | +This PR proposes to avoid using `bash -c` in `ShellBasedGroupsMappingProvider`. This could allow users a command injection. | ||
| 10 | + | ||
| 11 | +### Why are the changes needed? | ||
| 12 | + | ||
| 13 | +For a security purpose. | ||
| 14 | + | ||
| 15 | +### Does this PR introduce _any_ user-facing change? | ||
| 16 | + | ||
| 17 | +Virtually no. | ||
| 18 | + | ||
| 19 | +### How was this patch tested? | ||
| 20 | + | ||
| 21 | +Manually tested. | ||
| 22 | + | ||
| 23 | +Closes #36315 from HyukjinKwon/SPARK-38992. | ||
| 24 | + | ||
| 25 | +Authored-by: Hyukjin Kwon <gurwls223@apache.org> | ||
| 26 | +Signed-off-by: Hyukjin Kwon <gurwls223@apache.org> | ||
| 27 | +(cherry picked from commit c83618e4e5fc092829a1f2a726f12fb832e802cc) | ||
| 28 | +Signed-off-by: Hyukjin Kwon <gurwls223@apache.org> | ||
| 29 | +--- | ||
| 30 | + .../spark/security/ShellBasedGroupsMappingProvider.scala | 5 +++-- | ||
| 31 | + 1 file changed, 3 insertions(+), 2 deletions(-) | ||
| 32 | + | ||
| 33 | +diff --git a/core/src/main/scala/org/apache/spark/security/ShellBasedGroupsMappingProvider.scala b/core/src/main/scala/org/apache/spark/security/ShellBasedGroupsMappingProvider.scala | ||
| 34 | +index f71dd08246b2..7ef8ef165e3a 100644 | ||
| 35 | +--- a/core/src/main/scala/org/apache/spark/security/ShellBasedGroupsMappingProvider.scala | ||
| 36 | ++++ b/core/src/main/scala/org/apache/spark/security/ShellBasedGroupsMappingProvider.scala | ||
| 37 | +@@ -30,6 +30,8 @@ import org.apache.spark.util.Utils | ||
| 38 | + private[spark] class ShellBasedGroupsMappingProvider extends GroupMappingServiceProvider | ||
| 39 | + with Logging { | ||
| 40 | + | ||
| 41 | ++ private lazy val idPath = Utils.executeAndGetOutput("which" :: "id" :: Nil).stripLineEnd | ||
| 42 | ++ | ||
| 43 | + override def getGroups(username: String): Set[String] = { | ||
| 44 | + val userGroups = getUnixGroups(username) | ||
| 45 | + logDebug("User: " + username + " Groups: " + userGroups.mkString(",")) | ||
| 46 | +@@ -38,8 +40,7 @@ private[spark] class ShellBasedGroupsMappingProvider extends GroupMappingService | ||
| 47 | + | ||
| 48 | + // shells out a "bash -c id -Gn username" to get user groups | ||
| 49 | + private def getUnixGroups(username: String): Set[String] = { | ||
| 50 | +- val cmdSeq = Seq("bash", "-c", "id -Gn " + username) | ||
| 51 | + // we need to get rid of the trailing "\n" from the result of command execution | ||
| 52 | +- Utils.executeAndGetOutput(cmdSeq).stripLineEnd.split(" ").toSet | ||
| 53 | ++ Utils.executeAndGetOutput(idPath :: "-Gn" :: username :: Nil).stripLineEnd.split(" ").toSet | ||
| 54 | + } | ||
| 55 | + } | ||
| @@ -0,0 +1,88 @@ | |||
| 1 | +From 8435ae3ddc96e063bf4e08f67b30fe35f90aa271 Mon Sep 17 00:00:00 2001 | ||
| 2 | +From: PJ Fanning <pjfanning@users.noreply.github.com> | ||
| 3 | +Date: Mon, 30 Jun 2025 10:55:38 +0800 | ||
| 4 | +Subject: [PATCH] [SPARK-52381][CORE][4.0] JsonProtocol: Only accept subclasses | ||
| 5 | + of SparkListenerEvent | ||
| 6 | + | ||
| 7 | +JsonProtocol tidy up. Only parse JSON relating to Spark events. | ||
| 8 | +https://issues.apache.org/jira/browse/SPARK-52381 | ||
| 9 | + | ||
| 10 | +Tidier code and https://lists.apache.org/thread/9zwkdo85wcdfppgqvbhjly8wdgf595yp | ||
| 11 | + | ||
| 12 | +No | ||
| 13 | + | ||
| 14 | +Unit test | ||
| 15 | + | ||
| 16 | +No | ||
| 17 | + | ||
| 18 | +Closes #51312 from pjfanning/SPARK-52381-br4.0. | ||
| 19 | + | ||
| 20 | +Authored-by: PJ Fanning <pjfanning@users.noreply.github.com> | ||
| 21 | +Signed-off-by: yangjie01 <yangjie01@baidu.com> | ||
| 22 | +--- | ||
| 23 | + .../org/apache/spark/util/JsonProtocol.scala | 10 +++++-- | ||
| 24 | + .../apache/spark/util/JsonProtocolSuite.scala | 30 +++++++++++++++++++ | ||
| 25 | + 2 files changed, 38 insertions(+), 2 deletions(-) | ||
| 26 | + | ||
| 27 | +diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala | ||
| 28 | +index 377caf776deb..3b4bc242b466 100644 | ||
| 29 | +--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala | ||
| 30 | ++++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala | ||
| 31 | +@@ -874,8 +874,14 @@ private[spark] object JsonProtocol extends JsonUtils { | ||
| 32 | + case `stageExecutorMetrics` => stageExecutorMetricsFromJson(json) | ||
| 33 | + case `blockUpdate` => blockUpdateFromJson(json) | ||
| 34 | + case `resourceProfileAdded` => resourceProfileAddedFromJson(json) | ||
| 35 | +- case other => mapper.readValue(json.toString, Utils.classForName(other)) | ||
| 36 | +- .asInstanceOf[SparkListenerEvent] | ||
| 37 | ++ case other => | ||
| 38 | ++ val otherClass = Utils.classForName(other) | ||
| 39 | ++ if (classOf[SparkListenerEvent].isAssignableFrom(otherClass)) { | ||
| 40 | ++ mapper.readValue(json.toString, otherClass) | ||
| 41 | ++ .asInstanceOf[SparkListenerEvent] | ||
| 42 | ++ } else { | ||
| 43 | ++ throw new SparkException(s"Unknown event type: $other") | ||
| 44 | ++ } | ||
| 45 | + } | ||
| 46 | + } | ||
| 47 | + | ||
| 48 | +diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 49 | +index 8105df64705a..8af8c8579232 100644 | ||
| 50 | +--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 51 | ++++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 52 | +@@ -874,6 +874,36 @@ class JsonProtocolSuite extends SparkFunSuite { | ||
| 53 | + val jobFailedEvent = JsonProtocol.sparkEventFromJson(exJobFailureNoStackJson) | ||
| 54 | + testEvent(jobFailedEvent, exJobFailureExpectedJson) | ||
| 55 | + } | ||
| 56 | ++ | ||
| 57 | ++ test("SPARK-52381: handle class not found") { | ||
| 58 | ++ val unknownJson = | ||
| 59 | ++ """{ | ||
| 60 | ++ | "Event" : "com.example.UnknownEvent", | ||
| 61 | ++ | "foo" : "foo" | ||
| 62 | ++ |}""".stripMargin | ||
| 63 | ++ try { | ||
| 64 | ++ JsonProtocol.sparkEventFromJson(unknownJson) | ||
| 65 | ++ fail("Expected ClassNotFoundException for unknown event type") | ||
| 66 | ++ } catch { | ||
| 67 | ++ case e: ClassNotFoundException => | ||
| 68 | ++ } | ||
| 69 | ++ } | ||
| 70 | ++ | ||
| 71 | ++ test("SPARK-52381: only read classes that extend SparkListenerEvent") { | ||
| 72 | ++ val unknownJson = | ||
| 73 | ++ """{ | ||
| 74 | ++ | "Event" : "org.apache.spark.SparkException", | ||
| 75 | ++ | "foo" : "foo" | ||
| 76 | ++ |}""".stripMargin | ||
| 77 | ++ try { | ||
| 78 | ++ JsonProtocol.sparkEventFromJson(unknownJson) | ||
| 79 | ++ fail("Expected SparkException for unknown event type") | ||
| 80 | ++ } catch { | ||
| 81 | ++ case e: SparkException => | ||
| 82 | ++ assert(e.getMessage.startsWith("Unknown event type")) | ||
| 83 | ++ } | ||
| 84 | ++ } | ||
| 85 | ++ | ||
| 86 | + } | ||
| 87 | + | ||
| 88 | + | ||
| @@ -0,0 +1,130 @@ | |||
| 1 | +From a618545f789ca8e9abbc33359dbf8b65404d9dde Mon Sep 17 00:00:00 2001 | ||
| 2 | +From: PJ Fanning <pjfanning@users.noreply.github.com> | ||
| 3 | +Date: Sat, 28 Jun 2025 19:04:10 +0800 | ||
| 4 | +Subject: [PATCH 1/2] [SPARK-52381][CORE] JsonProtocol: Only accept subclasses | ||
| 5 | + of SparkListenerEvent | ||
| 6 | + | ||
| 7 | +### What changes were proposed in this pull request? | ||
| 8 | + | ||
| 9 | +JsonProtocol tidy up. Only parse JSON relating to Spark events. | ||
| 10 | +https://issues.apache.org/jira/browse/SPARK-52381 | ||
| 11 | + | ||
| 12 | +### Why are the changes needed? | ||
| 13 | + | ||
| 14 | +Tidier code and https://lists.apache.org/thread/9zwkdo85wcdfppgqvbhjly8wdgf595yp | ||
| 15 | + | ||
| 16 | +### Does this PR introduce _any_ user-facing change? | ||
| 17 | + | ||
| 18 | +No | ||
| 19 | + | ||
| 20 | +### How was this patch tested? | ||
| 21 | + | ||
| 22 | +Unit test | ||
| 23 | + | ||
| 24 | +### Was this patch authored or co-authored using generative AI tooling? | ||
| 25 | + | ||
| 26 | +No | ||
| 27 | + | ||
| 28 | +Closes #51061 from pjfanning/SPARK-52381. | ||
| 29 | + | ||
| 30 | +Authored-by: PJ Fanning <pjfanning@users.noreply.github.com> | ||
| 31 | +Signed-off-by: yangjie01 <yangjie01@baidu.com> | ||
| 32 | +--- | ||
| 33 | + .../org/apache/spark/util/JsonProtocol.scala | 10 +++++-- | ||
| 34 | + .../apache/spark/util/JsonProtocolSuite.scala | 30 +++++++++++++++++++ | ||
| 35 | + 2 files changed, 38 insertions(+), 2 deletions(-) | ||
| 36 | + | ||
| 37 | +diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala | ||
| 38 | +index d5d7c449f23d..7b99714d7676 100644 | ||
| 39 | +--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala | ||
| 40 | ++++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala | ||
| 41 | +@@ -957,8 +957,14 @@ private[spark] object JsonProtocol extends JsonUtils { | ||
| 42 | + case `stageExecutorMetrics` => stageExecutorMetricsFromJson(json) | ||
| 43 | + case `blockUpdate` => blockUpdateFromJson(json) | ||
| 44 | + case `resourceProfileAdded` => resourceProfileAddedFromJson(json) | ||
| 45 | +- case other => mapper.readValue(json.toString, Utils.classForName(other)) | ||
| 46 | +- .asInstanceOf[SparkListenerEvent] | ||
| 47 | ++ case other => | ||
| 48 | ++ val otherClass = Utils.classForName(other) | ||
| 49 | ++ if (classOf[SparkListenerEvent].isAssignableFrom(otherClass)) { | ||
| 50 | ++ mapper.readValue(json.toString, otherClass) | ||
| 51 | ++ .asInstanceOf[SparkListenerEvent] | ||
| 52 | ++ } else { | ||
| 53 | ++ throw new SparkException(s"Unknown event type: $other") | ||
| 54 | ++ } | ||
| 55 | + } | ||
| 56 | + } | ||
| 57 | + | ||
| 58 | +diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 59 | +index cfba4c02c944..8c3c18d818a0 100644 | ||
| 60 | +--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 61 | ++++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 62 | +@@ -995,6 +995,36 @@ class JsonProtocolSuite extends SparkFunSuite { | ||
| 63 | + // stages that have completed even before the job start event is emitted. | ||
| 64 | + testEvent(jobStart, sparkEventToJsonString(jobStart)) | ||
| 65 | + } | ||
| 66 | ++ | ||
| 67 | ++ test("SPARK-52381: handle class not found") { | ||
| 68 | ++ val unknownJson = | ||
| 69 | ++ """{ | ||
| 70 | ++ | "Event" : "com.example.UnknownEvent", | ||
| 71 | ++ | "foo" : "foo" | ||
| 72 | ++ |}""".stripMargin | ||
| 73 | ++ try { | ||
| 74 | ++ jsonProtocol.sparkEventFromJson(unknownJson) | ||
| 75 | ++ fail("Expected ClassNotFoundException for unknown event type") | ||
| 76 | ++ } catch { | ||
| 77 | ++ case e: ClassNotFoundException => | ||
| 78 | ++ } | ||
| 79 | ++ } | ||
| 80 | ++ | ||
| 81 | ++ test("SPARK-52381: only read classes that extend SparkListenerEvent") { | ||
| 82 | ++ val unknownJson = | ||
| 83 | ++ """{ | ||
| 84 | ++ | "Event" : "org.apache.spark.SparkException", | ||
| 85 | ++ | "foo" : "foo" | ||
| 86 | ++ |}""".stripMargin | ||
| 87 | ++ try { | ||
| 88 | ++ jsonProtocol.sparkEventFromJson(unknownJson) | ||
| 89 | ++ fail("Expected SparkException for unknown event type") | ||
| 90 | ++ } catch { | ||
| 91 | ++ case e: SparkException => | ||
| 92 | ++ assert(e.getMessage.startsWith("Unknown event type")) | ||
| 93 | ++ } | ||
| 94 | ++ } | ||
| 95 | ++ | ||
| 96 | + } | ||
| 97 | + | ||
| 98 | + | ||
| 99 | + | ||
| 100 | +From e59f2b3e75f07b16a69de89e5bf05e9acb6d3cd4 Mon Sep 17 00:00:00 2001 | ||
| 101 | +From: PJ Fanning <pjfanning@users.noreply.github.com> | ||
| 102 | +Date: Sat, 28 Jun 2025 21:05:11 +0100 | ||
| 103 | +Subject: [PATCH 2/2] compile issue | ||
| 104 | + | ||
| 105 | +--- | ||
| 106 | + .../test/scala/org/apache/spark/util/JsonProtocolSuite.scala | 4 ++-- | ||
| 107 | + 1 file changed, 2 insertions(+), 2 deletions(-) | ||
| 108 | + | ||
| 109 | +diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 110 | +index 8c3c18d818a0..bd3158910bb6 100644 | ||
| 111 | +--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 112 | ++++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala | ||
| 113 | +@@ -1003,7 +1003,7 @@ class JsonProtocolSuite extends SparkFunSuite { | ||
| 114 | + | "foo" : "foo" | ||
| 115 | + |}""".stripMargin | ||
| 116 | + try { | ||
| 117 | +- jsonProtocol.sparkEventFromJson(unknownJson) | ||
| 118 | ++ JsonProtocol.sparkEventFromJson(unknownJson) | ||
| 119 | + fail("Expected ClassNotFoundException for unknown event type") | ||
| 120 | + } catch { | ||
| 121 | + case e: ClassNotFoundException => | ||
| 122 | +@@ -1017,7 +1017,7 @@ class JsonProtocolSuite extends SparkFunSuite { | ||
| 123 | + | "foo" : "foo" | ||
| 124 | + |}""".stripMargin | ||
| 125 | + try { | ||
| 126 | +- jsonProtocol.sparkEventFromJson(unknownJson) | ||
| 127 | ++ JsonProtocol.sparkEventFromJson(unknownJson) | ||
| 128 | + fail("Expected SparkException for unknown event type") | ||
| 129 | + } catch { | ||
| 130 | + case e: SparkException => | ||
| @@ -4,7 +4,7 @@ | |||
| 4 | Summary: A unified analytics engine for large-scale data processing. | 4 | Summary: A unified analytics engine for large-scale data processing. |
| 5 | Name: spark | 5 | Name: spark |
| 6 | Version: 3.5.0 | 6 | Version: 3.5.0 |
| 7 | -Release: 6 | 7 | +Release: 7 |
| 8 | License: Apache 2.0 | 8 | License: Apache 2.0 |
| 9 | URL: http://spark.apache.org/ | 9 | URL: http://spark.apache.org/ |
| 10 | Source0: https://github.com/apache/spark/archive/v%{version}.tar.gz | 10 | Source0: https://github.com/apache/spark/archive/v%{version}.tar.gz |
| @@ -24,6 +24,16 @@ Patch1002: 1002-Added-support-for-building-the-riscv64-protoc-gen-gr.patch | |||
| 24 | Patch1003: 1003-Added-support-for-building-the-loongarch64-protoc-bi.patch | 24 | Patch1003: 1003-Added-support-for-building-the-loongarch64-protoc-bi.patch |
| 25 | Patch1004: 1004-Add-protoc-java-support-for-loongarch64.patch | 25 | Patch1004: 1004-Add-protoc-java-support-for-loongarch64.patch |
| 26 | Patch1005: 1005-Added-support-for-building-the-loongarch64-protoc-ge.patch | 26 | Patch1005: 1005-Added-support-for-building-the-loongarch64-protoc-ge.patch |
| 27 | +Patch1006: backport-CVE-2022-31777.patch | ||
| 28 | +Patch1007: backport-CVE-2022-31777-2.patch | ||
| 29 | +Patch1008: backport-CVE-2022-31777-3.patch | ||
| 30 | +Patch1009: backport-CVE-2023-22946.patch | ||
| 31 | +Patch1010: backport-CVE-2023-32007.patch | ||
| 32 | +Patch1011: backport-CVE-2025-54920.patch | ||
| 33 | +Patch1012: backport-CVE-2025-54920-2.patch | ||
| 34 | +Patch1013: backport-CVE-2025-55039.patch | ||
| 35 | +Patch1014: backport-CVE-2025-55039-2.patch | ||
| 36 | +Patch1015: backport-CVE-2025-55039-3.patch | ||
| 27 | 37 | ||
| 28 | %ifarch riscv64 loongarch64 | 38 | %ifarch riscv64 loongarch64 |
| 29 | BuildRequires: protobuf-devel protobuf-compiler | 39 | BuildRequires: protobuf-devel protobuf-compiler |
| @@ -91,6 +101,16 @@ tar -mxf %{SOURCE6} -C %{_builddir}/%{name}-%{version}/%{name}-prep_dir | |||
| 91 | pushd %{_builddir}/%{name}-%{version}/%{name}-prep_dir/grpc-java-1.56.0 | 101 | pushd %{_builddir}/%{name}-%{version}/%{name}-prep_dir/grpc-java-1.56.0 |
| 92 | %patch1002 -p1 | 102 | %patch1002 -p1 |
| 93 | %patch1005 -p1 | 103 | %patch1005 -p1 |
| 104 | +%patch -P 1006 -p1 | ||
| 105 | +%patch -P 1007 -p1 | ||
| 106 | +%patch -P 1008 -p1 | ||
| 107 | +%patch -P 1009 -p1 | ||
| 108 | +%patch -P 1010 -p1 | ||
| 109 | +%patch -P 1011 -p1 | ||
| 110 | +%patch -P 1012 -p1 | ||
| 111 | +%patch -P 1013 -p1 | ||
| 112 | +%patch -P 1014 -p1 | ||
| 113 | +%patch -P 1015 -p1 | ||
| 94 | sed -i "s,@HOME@,${HOME},g" build.gradle | 114 | sed -i "s,@HOME@,${HOME},g" build.gradle |
| 95 | sed -i 's|https\\://services.gradle.org/distributions|file://%{_tmppath}/source|g' gradle/wrapper/gradle-wrapper.properties | 115 | sed -i 's|https\\://services.gradle.org/distributions|file://%{_tmppath}/source|g' gradle/wrapper/gradle-wrapper.properties |
| 96 | %ifarch riscv64 | 116 | %ifarch riscv64 |
| @@ -125,6 +145,9 @@ cp -rf ../%{name}-%{version} %{buildroot}/opt/apache-%{name}-%{version} | |||
| 125 | 145 | ||
| 126 | 146 | ||
| 127 | %changelog | 147 | %changelog |
| 148 | +* Tue Aug 18 2026 chenyanpan <chenyanpan@xfusion.com> - 3.5.0-7 | ||
| 149 | +- Fix CVE-2022-31777 CVE-2023-22946 CVE-2023-32007 CVE-2025-54920 CVE-2025-55039 | ||
| 150 | + | ||
| 128 | * Wed Oct 15 2025 Wenlong Zhang <zhangwenlong@loongson.cn> - 3.5.0-6 | 151 | * Wed Oct 15 2025 Wenlong Zhang <zhangwenlong@loongson.cn> - 3.5.0-6 |
| 129 | - fix build error on loongarch64 | 152 | - fix build error on loongarch64 |
| 130 | 153 | ||