scSenicourt · Jun 10, 2026 · Jun 10, 2026 · Jun 10, 2026 · Jun 10, 2026 · Jun 10, 2026
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
@@ -26,11 +26,11 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up JDK 11
+    - name: Set up JDK 17
       uses: actions/setup-java@v3
       with:
-        java-version: '11'
-        distribution: 'adopt'
+        java-version: '17'
+        distribution: 'temurin'
         cache: maven
     - name: Setup Node.js environment
       uses: actions/setup-node@v3
@@ -41,4 +41,33 @@ jobs:
 
     # Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
     - name: Update dependency graph
+      if: github.event_name == 'push'
       uses: advanced-security/maven-dependency-submission-action@v3
+
+  compatibility-build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - profile: spark-3.5
+            java: '11'
+          - profile: spark-4.0
+            java: '17'
+          - profile: spark-4.2
+            java: '17'
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up JDK ${{ matrix.java }}
+      uses: actions/setup-java@v3
+      with:
+        java-version: ${{ matrix.java }}
+        distribution: 'temurin'
+        cache: maven
+    - name: Setup Node.js environment
+      uses: actions/setup-node@v3
+      with:
+        node-version: '18'
+    - name: Build ${{ matrix.profile }} compatibility package
+      run: mvn -B package --file pom.xml -P${{ matrix.profile }} -DskipTests -Ddockerfile.skip -Dgpg.skip=true
diff --git a/BUILDING.md b/BUILDING.md
@@ -6,7 +6,7 @@ is installed on the build machine.
 ### Prerequisites
 
 You need the following tools installed on your machine:
-* JDK 11 or later — but not too new (Java 16 is currently not supported)
+* JDK 11 for Spark 3.x builds and JDK 17 for Spark 4.x builds
 * Apache Maven (install via package manager download from https://maven.apache.org/download.cgi)
 * npm (install via package manager or download from https://www.npmjs.com/get-npm)
 * Docker (the creation of the Docker image can be skipped via `-Ddockerfile.skip`)
@@ -25,6 +25,8 @@ Building Flowman with the default settings (i.e. newest supported Spark and Hado
 mvn clean install
 ```
 
+The default open source build currently targets Spark 4.1, Scala 2.13, Hadoop 3.4, and Java 17.
+
 ### Main Artifacts
 
 The main artifacts will be a Docker image 'dimajix/flowman' and additionally a tar.gz file containing a runnable 
@@ -82,11 +84,11 @@ the `main` branch really builds clean with all unittests passing on Linux.
 
 ### Build for Custom Spark / Hadoop Version
 
-Per default, Flowman will be built for fairly recent versions of Spark (3.0.2 as of this writing) and Hadoop (3.2.0). 
-But of course you can also build for a different version by either using a profile
+Per default, Flowman will be built for the newest supported open source Spark and Hadoop versions. But of course you can
+also build for a different version by using a profile:
 
 ```shell
-mvn install -Pspark2.4 -Phadoop2.7 -DskipTests
+mvn install -Pspark-3.5 -Phadoop-3.3 -DskipTests
 ```
 
 This will always select the latest bugfix version within the minor version. You can also specify versions explicitly 
@@ -105,9 +107,13 @@ using the correct version. The following profiles are available:
 * spark-3.3
 * spark-3.4
 * spark-3.5
+* spark-4.0
+* spark-4.1
+* spark-4.2 (preview)
 * hadoop-3.1
 * hadoop-3.2
 * hadoop-3.3
+* hadoop-3.4
 * EMR-6.12
 * synapse-3.3
 * CDP-7.1-spark-3.2
@@ -116,11 +122,33 @@ using the correct version. The following profiles are available:
 With these profiles it is easy to build Flowman to match your environment. 
 
 
+### Supported Spark, Scala and Java Combinations
+
+The main open source compatibility targets are:
+
+* Spark 3.5 with Scala 2.12 and Java 11
+* Spark 4.0 with Scala 2.13 and Java 17
+* Spark 4.1 with Scala 2.13 and Java 17
+* Spark 4.2 preview with Scala 2.13 and Java 17
+
+To validate the main package variants locally, run the package phase with the matching JDK:
+
+```shell
+JAVA_HOME=/path/to/jdk-11 mvn clean package -Pspark-3.5 -DskipTests
+JAVA_HOME=/path/to/jdk-17 mvn clean package -Pspark-4.0 -DskipTests
+JAVA_HOME=/path/to/jdk-17 mvn clean package -Pspark-4.1 -DskipTests
+JAVA_HOME=/path/to/jdk-17 mvn clean package -Pspark-4.2 -DskipTests
+```
+
+The Spark 4.2 preview distribution does not include the Delta Lake plugin. Delta 4.3.0 is built for Spark 4.1 and is
+not binary-compatible with Spark 4.2-preview5.
+
+
 ### Building for specific Java Version
 
-If nothing else is set on the command line, Flowman will now be built for Java 11 (except when building for Spark 2.4, EMR
-or Azure Synapse, where Java 1.8 is used). If you are still stuck on Java 1.8, you can simply override the Java version 
-by specifying the property `java.version`
+If nothing else is set on the command line, Flowman will now be built for Java 17. The `spark-3.5` profile overrides this
+to Java 11, and the older distribution profiles may use Java 8. You can override the Java target by specifying the
+property `java.version`, but the selected Spark runtime still needs to support that Java version.
 
 ```shell
 mvn install -Djava.version=1.8
@@ -172,6 +200,24 @@ mvn clean install -Pspark-3.4 -Phadoop-3.3 -Dhadoop.version=3.3.4 -DskipTests
 mvn clean install -Pspark-3.5 -Phadoop-3.3 -Dhadoop.version=3.3.4 -DskipTests
 ```
 
+#### Spark 4.0 and Hadoop 3.4
+
+```shell
+mvn clean install -Pspark-4.0 -Phadoop-3.4 -DskipTests
+```
+
+#### Spark 4.1 and Hadoop 3.4
+
+```shell
+mvn clean install -Pspark-4.1 -Phadoop-3.4 -DskipTests
+```
+
+#### Spark 4.2 preview and Hadoop 3.5
+
+```shell
+mvn clean install -Pspark-4.2 -DskipTests
+```
+
 
 ### Building for Cloudera
 

diff --git a/docker/conf/default-namespace.yml b/docker/conf/default-namespace.yml
@@ -50,11 +50,10 @@ plugins:
   - flowman-avro
   - flowman-aws
   - flowman-azure
-  - flowman-delta
+  @flowman.plugin.delta@
   - flowman-kafka
   - flowman-mariadb
   - flowman-mysql
-  - flowman-mssqlserver
   - flowman-oracle
   - flowman-postgresql
   - flowman-swagger

diff --git a/docker/conf/history-server.yml b/docker/conf/history-server.yml
@@ -17,4 +17,3 @@ connections:
 plugins:
   - flowman-mariadb
   - flowman-mysql
-  - flowman-mssqlserver
diff --git a/docker/pom.xml b/docker/pom.xml
@@ -80,7 +80,6 @@
                                     <directory>.</directory>
                                     <includes>
                                         <include>bin/*</include>
-                                        <include>conf/*</include>
                                         <include>libexec/*</include>
                                     </includes>
                                     <filtering>false</filtering>
@@ -96,6 +95,29 @@
                             </resources>
                         </configuration>
                     </execution>
+                    <execution>
+                        <id>copy-config</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <delimiters>
+                                <delimiter>@</delimiter>
+                            </delimiters>
+                            <useDefaultDelimiters>false</useDefaultDelimiters>
+                            <outputDirectory>${basedir}/target/build</outputDirectory>
+                            <resources>
+                                <resource>
+                                    <directory>.</directory>
+                                    <includes>
+                                        <include>conf/*</include>
+                                    </includes>
+                                    <filtering>true</filtering>
+                                </resource>
+                            </resources>
+                        </configuration>
+                    </execution>
                 </executions>
             </plugin>
             <plugin>

diff --git a/docs/setup/building.md b/docs/setup/building.md
@@ -31,7 +31,7 @@ is installed on the build machine.
 
 When you decide against downloading a prebuilt Flowman distribution, you can simply build it yourself with Maven. As
 a prerequisite, you need
-* Java (1.8 for Spark ≤ 2.4 and 11 for Spark ≥ 3.0)
+* Java (11 for Spark 3.x builds and 17 for Spark 4.x builds)
 * Apache Maven (install via package manager download from https://maven.apache.org/download.cgi)
 * npm (install via package manager or download from https://www.npmjs.com/get-npm)
 * Docker (the creation of the Docker image can be skipped via `-Ddockerfile.skip`)
@@ -49,6 +49,8 @@ Building Flowman with the default settings (i.e. Hadoop and Spark version) is as
 mvn clean install
 ```
 
+The default open source build currently targets Spark 4.1, Scala 2.13, Hadoop 3.4, and Java 17.
+
 ### Skip Tests
 
 In case you don't want to run tests, you can simply append `-DskipTests`
@@ -100,10 +102,10 @@ You might also want to skip unit tests (the HBase plugin is currently failing un
 
 ### Build for Custom Spark / Hadoop Version
 
-Per default, Flowman will be built for fairly recent versions of Spark (3.0.2 as of this writing) and Hadoop (3.2.0). 
-But of course, you can also build for a different version by either using a profile
+Per default, Flowman will be built for the newest supported open source Spark and Hadoop versions. But of course, you can
+also build for a different version by using a profile
 
-    mvn install -Pspark2.4 -Phadoop2.7 -DskipTests
+    mvn install -Pspark-3.5 -Phadoop-3.3 -DskipTests
 
 This will always select the latest bug fix version within the minor version. You can also specify versions explicitly
 as follows:    
@@ -119,9 +121,13 @@ using the correct version. The following profiles are available:
 * spark-3.3
 * spark-3.4
 * spark-3.5
+* spark-4.0
+* spark-4.1
+* spark-4.2 (preview)
 * hadoop-3.1
 * hadoop-3.2
 * hadoop-3.3
+* hadoop-3.4
 * EMR-6.12
 * synapse-3.3
 * CDP-7.1-spark-3.2
@@ -130,10 +136,33 @@ using the correct version. The following profiles are available:
 With these profiles it is easy to build Flowman to match your environment. 
 
 
+### Supported Spark, Scala and Java Combinations
+
+The main open source compatibility targets are:
+
+* Spark 3.5 with Scala 2.12 and Java 11
+* Spark 4.0 with Scala 2.13 and Java 17
+* Spark 4.1 with Scala 2.13 and Java 17
+* Spark 4.2 preview with Scala 2.13 and Java 17
+
+To validate the main package variants locally, run the package phase with the matching JDK:
+
+```shell
+JAVA_HOME=/path/to/jdk-11 mvn clean package -Pspark-3.5 -DskipTests
+JAVA_HOME=/path/to/jdk-17 mvn clean package -Pspark-4.0 -DskipTests
+JAVA_HOME=/path/to/jdk-17 mvn clean package -Pspark-4.1 -DskipTests
+JAVA_HOME=/path/to/jdk-17 mvn clean package -Pspark-4.2 -DskipTests
+```
+
+The Spark 4.2 preview distribution does not include the Delta Lake plugin. Delta 4.3.0 is built for Spark 4.1 and is
+not binary-compatible with Spark 4.2-preview5.
+
+
 ### Building for specific Java Version
 
-If nothing else is set on the command line, Flowman will now build for Java 11. If you are still stuck on Java 1.8, 
-you can simply override the Java version by specifying the property `java.version`
+If nothing else is set on the command line, Flowman will now build for Java 17. The `spark-3.5` profile overrides this
+to Java 11, and the older distribution profiles may use Java 8. You can override the Java target by specifying the
+property `java.version`, but the selected Spark runtime still needs to support that Java version.
 
 ```shell
 mvn install -Djava.version=1.8
@@ -177,6 +206,21 @@ mvn clean install -Pspark-3.4 -Phadoop-3.3 -Phadoop.version=3.3.4 -DskipTests
 mvn clean install -Pspark-3.5 -Phadoop-3.3 -Phadoop.version=3.3.4 -DskipTests
 ```
 
+#### Spark 4.0 and Hadoop 3.4
+```shell
+mvn clean install -Pspark-4.0 -Phadoop-3.4 -DskipTests
+```
+
+#### Spark 4.1 and Hadoop 3.4
+```shell
+mvn clean install -Pspark-4.1 -Phadoop-3.4 -DskipTests
+```
+
+#### Spark 4.2 preview and Hadoop 3.5
+```shell
+mvn clean install -Pspark-4.2 -DskipTests
+```
+
 
 ### Building for Cloudera
 

diff --git a/flowman-common-jersey/pom.xml b/flowman-common-jersey/pom.xml
@@ -16,6 +16,7 @@
     </parent>
 
     <build>
+        <sourceDirectory>src/main/scala</sourceDirectory>
         <plugins>
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
@@ -25,9 +26,48 @@
                 <groupId>org.scalatest</groupId>
                 <artifactId>scalatest-maven-plugin</artifactId>
             </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>build-helper-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>add-source</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.basedir}/src/main/spark-${spark-source.version}</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                    <execution>
+                        <id>add-site-source</id>
+                        <phase>pre-site</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                        </goals>
+                        <configuration>
+                            <sources>
+                                <source>${project.basedir}/src/main/spark-${spark-source.version}</source>
+                            </sources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
         </plugins>
     </build>
 
+    <profiles>
+        <profile>
+            <id>spark-4.2</id>
+            <properties>
+                <spark-source.version>4.2</spark-source.version>
+            </properties>
+        </profile>
+    </profiles>
+
     <dependencies>
         <dependency>
             <groupId>com.dimajix.flowman</groupId>