Skip to content

Commit ea75fa8

Browse files
pan3793pan3793
committed
[KYUUBI #7174] Kyuubi Spark Extention supports Spark 4.0
### Why are the changes needed? Port Kyuubi Spark Extension from 3.5 to 4.0. Note: the `KyuubiEnsureRequirements` does not handle DSv2 bucketing properly now. ### How was this patch tested? UTs are ported too. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #7174 from pan3793/spark4-ext. Closes #7174 874cc61 [Cheng Pan] kyuubi-extension-spark-4-0 Authored-by: Cheng Pan <chengpan@apache.org> Signed-off-by: Cheng Pan <chengpan@apache.org>
1 parent cd04743 commit ea75fa8

File tree

48 files changed

+6646
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+6646
-1
lines changed

‎.github/workflows/style.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ jobs:
6868
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-3-3 -Pspark-3.3
6969
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4
7070
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-3-5,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.5
71+
build/mvn clean install -pl extensions/spark/kyuubi-extension-spark-4-0 -Pspark-4.0 -Pscala-2.13
7172
7273
- name: Scalastyle with maven
7374
id: scalastyle-check

‎dev/kyuubi-codecov/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,11 @@
218218
<profile>
219219
<id>spark-4.0</id>
220220
<dependencies>
221+
<dependency>
222+
<groupId>org.apache.kyuubi</groupId>
223+
<artifactId>kyuubi-extension-spark-4-0_${scala.binary.version}</artifactId>
224+
<version>${project.version}</version>
225+
</dependency>
221226
<dependency>
222227
<groupId>org.apache.kyuubi</groupId>
223228
<artifactId>kyuubi-spark-connector-hive_${scala.binary.version}</artifactId>

‎dev/reformat‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ set -x
2020

2121
KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)"
2222

23-
PROFILES="-Pflink-provided,hive-provided,spark-provided,spark-3.5,spark-3.4,spark-3.3,tpcds,kubernetes-it"
23+
PROFILES="-Pflink-provided,hive-provided,spark-provided,spark-4.0,spark-3.5,spark-3.4,spark-3.3,tpcds,kubernetes-it"
2424

2525
# python style checks rely on `black` in path
2626
if ! command -v black &> /dev/null

‎extensions/spark/kyuubi-extension-spark-4-0/pom.xml‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.kyuubi</groupId>
23+
<artifactId>kyuubi-parent</artifactId>
24+
<version>1.11.0-SNAPSHOT</version>
25+
<relativePath>../../../pom.xml</relativePath>
26+
</parent>
27+
28+
<artifactId>kyuubi-extension-spark-4-0_${scala.binary.version}</artifactId>
29+
<packaging>jar</packaging>
30+
<name>Kyuubi Dev Spark Extensions (for Spark 4.0)</name>
31+
<url>https://kyuubi.apache.org/</url>
32+
33+
<dependencies>
34+
<dependency>
35+
<groupId>org.scala-lang</groupId>
36+
<artifactId>scala-library</artifactId>
37+
<scope>provided</scope>
38+
</dependency>
39+
40+
<dependency>
41+
<groupId>org.apache.spark</groupId>
42+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
43+
<scope>provided</scope>
44+
</dependency>
45+
46+
<dependency>
47+
<groupId>org.apache.spark</groupId>
48+
<artifactId>spark-hive_${scala.binary.version}</artifactId>
49+
<scope>provided</scope>
50+
</dependency>
51+
52+
<dependency>
53+
<groupId>org.apache.hadoop</groupId>
54+
<artifactId>hadoop-client-api</artifactId>
55+
<scope>provided</scope>
56+
</dependency>
57+
58+
<dependency>
59+
<groupId>org.apache.kyuubi</groupId>
60+
<artifactId>kyuubi-download</artifactId>
61+
<version>${project.version}</version>
62+
<type>pom</type>
63+
<scope>test</scope>
64+
</dependency>
65+
66+
<dependency>
67+
<groupId>org.apache.kyuubi</groupId>
68+
<artifactId>kyuubi-util-scala_${scala.binary.version}</artifactId>
69+
<version>${project.version}</version>
70+
<type>test-jar</type>
71+
<scope>test</scope>
72+
</dependency>
73+
74+
<dependency>
75+
<groupId>org.apache.spark</groupId>
76+
<artifactId>spark-core_${scala.binary.version}</artifactId>
77+
<type>test-jar</type>
78+
<scope>test</scope>
79+
</dependency>
80+
81+
<dependency>
82+
<groupId>org.apache.spark</groupId>
83+
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
84+
<type>test-jar</type>
85+
<scope>test</scope>
86+
</dependency>
87+
88+
<dependency>
89+
<groupId>org.scalatestplus</groupId>
90+
<artifactId>scalacheck-1-17_${scala.binary.version}</artifactId>
91+
<scope>test</scope>
92+
</dependency>
93+
94+
<dependency>
95+
<groupId>org.apache.spark</groupId>
96+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
97+
<version>${spark.version}</version>
98+
<type>test-jar</type>
99+
<scope>test</scope>
100+
</dependency>
101+
102+
<dependency>
103+
<groupId>org.apache.hadoop</groupId>
104+
<artifactId>hadoop-client-runtime</artifactId>
105+
<scope>test</scope>
106+
</dependency>
107+
108+
<dependency>
109+
<groupId>javax.servlet</groupId>
110+
<artifactId>javax.servlet-api</artifactId>
111+
<scope>test</scope>
112+
</dependency>
113+
114+
<dependency>
115+
<groupId>jakarta.servlet</groupId>
116+
<artifactId>jakarta.servlet-api</artifactId>
117+
<scope>test</scope>
118+
</dependency>
119+
120+
<dependency>
121+
<groupId>org.apache.logging.log4j</groupId>
122+
<artifactId>log4j-slf4j-impl</artifactId>
123+
<scope>test</scope>
124+
</dependency>
125+
</dependencies>
126+
127+
<build>
128+
129+
<plugins>
130+
<plugin>
131+
<groupId>org.codehaus.mojo</groupId>
132+
<artifactId>build-helper-maven-plugin</artifactId>
133+
<executions>
134+
<execution>
135+
<id>regex-property</id>
136+
<goals>
137+
<goal>regex-property</goal>
138+
</goals>
139+
<configuration>
140+
<name>spark.home</name>
141+
<value>${project.basedir}/../../../externals/kyuubi-download/target/${spark.archive.name}</value>
142+
<regex>(.+)\.tgz</regex>
143+
<replacement>$1</replacement>
144+
</configuration>
145+
</execution>
146+
</executions>
147+
</plugin>
148+
<plugin>
149+
<groupId>org.scalatest</groupId>
150+
<artifactId>scalatest-maven-plugin</artifactId>
151+
<configuration>
152+
<environmentVariables>
153+
<!--
154+
Some tests run Spark in local-cluster mode.
155+
This mode uses SPARK_HOME and SPARK_SCALA_VERSION to build command to launch a Spark Standalone Cluster.
156+
-->
157+
<SPARK_HOME>${spark.home}</SPARK_HOME>
158+
<SPARK_SCALA_VERSION>${scala.binary.version}</SPARK_SCALA_VERSION>
159+
</environmentVariables>
160+
</configuration>
161+
</plugin>
162+
<plugin>
163+
<groupId>org.antlr</groupId>
164+
<artifactId>antlr4-maven-plugin</artifactId>
165+
<configuration>
166+
<visitor>true</visitor>
167+
<sourceDirectory>${project.basedir}/src/main/antlr4</sourceDirectory>
168+
</configuration>
169+
</plugin>
170+
171+
<plugin>
172+
<groupId>org.apache.maven.plugins</groupId>
173+
<artifactId>maven-shade-plugin</artifactId>
174+
<configuration>
175+
<shadedArtifactAttached>false</shadedArtifactAttached>
176+
<artifactSet>
177+
<includes>
178+
<include>org.apache.kyuubi:*</include>
179+
</includes>
180+
</artifactSet>
181+
</configuration>
182+
<executions>
183+
<execution>
184+
<goals>
185+
<goal>shade</goal>
186+
</goals>
187+
<phase>package</phase>
188+
</execution>
189+
</executions>
190+
</plugin>
191+
</plugins>
192+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
193+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
194+
</build>
195+
</project>

0 commit comments

Comments
 (0)