From b8fae3322fed430d19f40141705cd79f438515a9 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 11 Jul 2022 17:09:04 +0800 Subject: [PATCH 1/2] Shade Jackson --- build.gradle | 2 ++ spark-3.2/build.gradle | 5 ++--- spark-3.3/build.gradle | 5 ++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index af213c9b..783358a8 100644 --- a/build.gradle +++ b/build.gradle @@ -72,6 +72,7 @@ subprojects { force "com.fasterxml.jackson.core:jackson-annotations:$jackson_version" force "com.fasterxml.jackson.core:jackson-core:$jackson_version" force "com.fasterxml.jackson.core:jackson-databind:$jackson_version" + force "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version" force "com.fasterxml.jackson.module:jackson-module-paranamer:$jackson_version" force "com.fasterxml.jackson.module:jackson-module-scala_$scala_binary_version:$jackson_version" @@ -164,6 +165,7 @@ project(':clickhouse-core') { api "org.apache.commons:commons-lang3:$commons_lang3_version" api "com.fasterxml.jackson.core:jackson-databind:$jackson_version" + api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version" api "com.fasterxml.jackson.module:jackson-module-scala_$scala_binary_version:$jackson_version" api("com.google.guava:guava:$guava_version") { diff --git a/spark-3.2/build.gradle b/spark-3.2/build.gradle index 24b10f70..f86f10d3 100644 --- a/spark-3.2/build.gradle +++ b/spark-3.2/build.gradle @@ -42,8 +42,6 @@ project(":clickhouse-spark-runtime-${spark_binary_version}_$scala_binary_version exclude group: "org.scala-lang", module: "scala-library" exclude group: "org.slf4j", module: "slf4j-api" exclude group: "org.apache.commons", module: "commons-lang3" - exclude group: "com.fasterxml.jackson.core", module: "jackson-databind" - exclude group: "com.fasterxml.jackson.module", module: "jackson-module-scala_$scala_binary_version" } } @@ -55,7 +53,9 @@ project(":clickhouse-spark-runtime-${spark_binary_version}_$scala_binary_version exclude "META-INF/native/**" exclude "META-INF/native-image/**" + relocate "com.fasterxml.jackson", "xenon.relocated.com.fasterxml.jackson" relocate "com.google", "xenon.relocated.com.google" + relocate "com.thoughtworks.paranamer", "xenon.relocated.com.thoughtworks.paranamer" relocate "io.grpc", "xenon.relocated.io.grpc" relocate "io.perfmark", "xenon.relocated.io.perfmark" @@ -82,7 +82,6 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") { testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version" testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version:tests" - testImplementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version" if (scala_binary_version == "2.12") { testImplementation "org.apache.kyuubi:kyuubi-spark-connector-tpcds_${scala_binary_version}:1.6.0-SNAPSHOT" diff --git a/spark-3.3/build.gradle b/spark-3.3/build.gradle index caa0829f..17fda4f8 100644 --- a/spark-3.3/build.gradle +++ b/spark-3.3/build.gradle @@ -42,8 +42,6 @@ project(":clickhouse-spark-runtime-${spark_binary_version}_$scala_binary_version exclude group: "org.scala-lang", module: "scala-library" exclude group: "org.slf4j", module: "slf4j-api" exclude group: "org.apache.commons", module: "commons-lang3" - exclude group: "com.fasterxml.jackson.core", module: "jackson-databind" - exclude group: "com.fasterxml.jackson.module", module: "jackson-module-scala_$scala_binary_version" } } @@ -55,7 +53,9 @@ project(":clickhouse-spark-runtime-${spark_binary_version}_$scala_binary_version exclude "META-INF/native/**" exclude "META-INF/native-image/**" + relocate "com.fasterxml.jackson", "xenon.relocated.com.fasterxml.jackson" relocate "com.google", "xenon.relocated.com.google" + relocate "com.thoughtworks.paranamer", "xenon.relocated.com.thoughtworks.paranamer" relocate "io.grpc", "xenon.relocated.io.grpc" relocate "io.perfmark", "xenon.relocated.io.perfmark" @@ -82,7 +82,6 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") { testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version" testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version:tests" - testImplementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version" if (scala_binary_version == "2.12") { testImplementation "org.apache.kyuubi:kyuubi-spark-connector-tpcds_${scala_binary_version}:1.6.0-SNAPSHOT" From 731f69efd630d0dda0aa4cd22ee2713f75d9e246 Mon Sep 17 00:00:00 2001 From: Cheng Pan Date: Mon, 11 Jul 2022 17:29:45 +0800 Subject: [PATCH 2/2] Fix it --- spark-3.2/build.gradle | 4 +++ .../spark/sql/clickhouse/TestUtils.scala | 30 +++++++++++++++++++ .../cluster/ClickHouseClusterUDFSuite.scala | 2 +- spark-3.3/build.gradle | 4 +++ .../spark/sql/clickhouse/TestUtils.scala | 30 +++++++++++++++++++ .../cluster/ClickHouseClusterUDFSuite.scala | 2 +- 6 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala create mode 100644 spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala diff --git a/spark-3.2/build.gradle b/spark-3.2/build.gradle index f86f10d3..510d7e8f 100644 --- a/spark-3.2/build.gradle +++ b/spark-3.2/build.gradle @@ -83,6 +83,10 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") { testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version" testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version:tests" + testImplementation "com.fasterxml.jackson.core:jackson-databind:$jackson_version" + testImplementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version" + testImplementation "com.fasterxml.jackson.module:jackson-module-scala_$scala_binary_version:$jackson_version" + if (scala_binary_version == "2.12") { testImplementation "org.apache.kyuubi:kyuubi-spark-connector-tpcds_${scala_binary_version}:1.6.0-SNAPSHOT" } diff --git a/spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala b/spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala new file mode 100644 index 00000000..8107a884 --- /dev/null +++ b/spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.clickhouse + +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} +import com.fasterxml.jackson.module.scala.ClassTagExtensions + +object TestUtils { + + @transient lazy val om: ObjectMapper with ClassTagExtensions = { + val _om = new ObjectMapper() with ClassTagExtensions + _om.findAndRegisterModules() + _om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + _om + } + + def toJson(value: Any): String = om.writeValueAsString(value) +} diff --git a/spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala b/spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala index d07b298a..a5316446 100644 --- a/spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala +++ b/spark-3.2/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala @@ -14,12 +14,12 @@ package org.apache.spark.sql.clickhouse.cluster -import xenon.clickhouse.JsonProtocol.om import xenon.clickhouse.base.ClickHouseClusterMixIn import xenon.clickhouse.Logging import java.lang.{Long => JLong} import org.apache.spark.sql.clickhouse.BaseSparkSuite +import org.apache.spark.sql.clickhouse.TestUtils.om class ClickHouseClusterUDFSuite extends BaseSparkSuite with ClickHouseClusterMixIn diff --git a/spark-3.3/build.gradle b/spark-3.3/build.gradle index 17fda4f8..960201a4 100644 --- a/spark-3.3/build.gradle +++ b/spark-3.3/build.gradle @@ -83,6 +83,10 @@ project(":clickhouse-spark-it-${spark_binary_version}_$scala_binary_version") { testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version" testImplementation "org.apache.spark:spark-sql_$scala_binary_version:$spark_version:tests" + testImplementation "com.fasterxml.jackson.core:jackson-databind:$jackson_version" + testImplementation "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jackson_version" + testImplementation "com.fasterxml.jackson.module:jackson-module-scala_$scala_binary_version:$jackson_version" + if (scala_binary_version == "2.12") { testImplementation "org.apache.kyuubi:kyuubi-spark-connector-tpcds_${scala_binary_version}:1.6.0-SNAPSHOT" } diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala new file mode 100644 index 00000000..8107a884 --- /dev/null +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/TestUtils.scala @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.clickhouse + +import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper} +import com.fasterxml.jackson.module.scala.ClassTagExtensions + +object TestUtils { + + @transient lazy val om: ObjectMapper with ClassTagExtensions = { + val _om = new ObjectMapper() with ClassTagExtensions + _om.findAndRegisterModules() + _om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + _om + } + + def toJson(value: Any): String = om.writeValueAsString(value) +} diff --git a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala index d07b298a..a5316446 100644 --- a/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala +++ b/spark-3.3/clickhouse-spark-it/src/test/scala/org/apache/spark/sql/clickhouse/cluster/ClickHouseClusterUDFSuite.scala @@ -14,12 +14,12 @@ package org.apache.spark.sql.clickhouse.cluster -import xenon.clickhouse.JsonProtocol.om import xenon.clickhouse.base.ClickHouseClusterMixIn import xenon.clickhouse.Logging import java.lang.{Long => JLong} import org.apache.spark.sql.clickhouse.BaseSparkSuite +import org.apache.spark.sql.clickhouse.TestUtils.om class ClickHouseClusterUDFSuite extends BaseSparkSuite with ClickHouseClusterMixIn