[HUDI-4357] Support flink 1.15.x (#6050)

2022-07-06 13:42:58 +08:00
parent b18c32379f
commit 7eeaff9ee0
91 changed files with 7799 additions and 82 deletions
--- a/pom.xml
+++ b/pom.xml
@@ -123,15 +123,23 @@
    <spark2.version>2.4.4</spark2.version>
    <spark3.version>3.2.1</spark3.version>
    <sparkbundle.version></sparkbundle.version>
+    <flink1.15.version>1.15.0</flink1.15.version>
    <flink1.14.version>1.14.4</flink1.14.version>
    <flink1.13.version>1.13.6</flink1.13.version>
-    <flink.version>${flink1.14.version}</flink.version>
-    <hudi.flink.module>hudi-flink1.14.x</hudi.flink.module>
-    <flink.bundle.version>1.14</flink.bundle.version>
+    <flink.version>${flink1.13.version}</flink.version>
+    <hudi.flink.module>hudi-flink1.13.x</hudi.flink.module>
+    <flink.bundle.version>1.13</flink.bundle.version>
+    <flink.format.parquet.version>1.12.2</flink.format.parquet.version>
    <flink.runtime.artifactId>flink-runtime</flink.runtime.artifactId>
    <flink.table.runtime.artifactId>flink-table-runtime_${scala.binary.version}</flink.table.runtime.artifactId>
    <flink.table.planner.artifactId>flink-table-planner_${scala.binary.version}</flink.table.planner.artifactId>
-    <flink.format.parquet.version>1.12.2</flink.format.parquet.version>
+    <flink.parquet.artifactId>flink-parquet</flink.parquet.artifactId>
+    <flink.statebackend.rocksdb.artifactId>flink-statebackend-rocksdb</flink.statebackend.rocksdb.artifactId>
+    <flink.test.utils.artifactId>flink-test-utils</flink.test.utils.artifactId>
+    <flink.streaming.java.artifactId>flink-streaming-java</flink.streaming.java.artifactId>
+    <flink.clients.artifactId>flink-clients</flink.clients.artifactId>
+    <flink.connector.kafka.artifactId>flink-connector-kafka</flink.connector.kafka.artifactId>
+    <flink.hadoop.compatibility.artifactId>flink-hadoop-compatibility_${scala.binary.version}</flink.hadoop.compatibility.artifactId>
    <spark31.version>3.1.3</spark31.version>
    <spark32.version>3.2.1</spark32.version>
    <hudi.spark.module>hudi-spark2</hudi.spark.module>
@@ -644,19 +652,19 @@
      <!-- Flink -->
      <dependency>
        <groupId>org.apache.flink</groupId>
-        <artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
+        <artifactId>${flink.streaming.java.artifactId}</artifactId>
        <version>${flink.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.flink</groupId>
-        <artifactId>flink-clients_${scala.binary.version}</artifactId>
+        <artifactId>${flink.clients.artifactId}</artifactId>
        <version>${flink.version}</version>
        <scope>provided</scope>
      </dependency>
      <dependency>
        <groupId>org.apache.flink</groupId>
-        <artifactId>flink-connector-kafka_${scala.binary.version}</artifactId>
+        <artifactId>${flink.connector.kafka.artifactId}</artifactId>
        <version>${flink.version}</version>
        <scope>provided</scope>
      </dependency>
@@ -1759,8 +1767,43 @@
      </activation>
    </profile>

+    <profile>
+      <id>flink1.15</id>
+      <properties>
+        <flink.version>${flink1.15.version}</flink.version>
+        <flink.table.runtime.artifactId>flink-table-runtime</flink.table.runtime.artifactId>
+        <flink.parquet.artifactId>flink-parquet</flink.parquet.artifactId>
+        <flink.statebackend.rocksdb.artifactId>flink-statebackend-rocksdb</flink.statebackend.rocksdb.artifactId>
+        <flink.test.utils.artifactId>flink-test-utils</flink.test.utils.artifactId>
+        <flink.streaming.java.artifactId>flink-streaming-java</flink.streaming.java.artifactId>
+        <flink.clients.artifactId>flink-clients</flink.clients.artifactId>
+        <flink.connector.kafka.artifactId>flink-connector-kafka</flink.connector.kafka.artifactId>
+        <!-- 1.15 only supports scala2.12 -->
+        <flink.hadoop.compatibility.artifactId>flink-hadoop-compatibility_2.12</flink.hadoop.compatibility.artifactId>
+        <hudi.flink.module>hudi-flink1.15.x</hudi.flink.module>
+        <flink.bundle.version>1.15</flink.bundle.version>
+      </properties>
+      <activation>
+        <property>
+          <name>flink1.15</name>
+        </property>
+      </activation>
+    </profile>
    <profile>
      <id>flink1.14</id>
+      <properties>
+        <flink.version>${flink1.14.version}</flink.version>
+        <flink.table.runtime.artifactId>flink-table-runtime_${scala.binary.version}</flink.table.runtime.artifactId>
+        <flink.parquet.artifactId>flink-parquet_${scala.binary.version}</flink.parquet.artifactId>
+        <flink.statebackend.rocksdb.artifactId>flink-statebackend-rocksdb_${scala.binary.version}</flink.statebackend.rocksdb.artifactId>
+        <flink.test.utils.artifactId>flink-test-utils_${scala.binary.version}</flink.test.utils.artifactId>
+        <flink.streaming.java.artifactId>flink-streaming-java_${scala.binary.version}</flink.streaming.java.artifactId>
+        <flink.clients.artifactId>flink-clients_${scala.binary.version}</flink.clients.artifactId>
+        <flink.connector.kafka.artifactId>flink-connector-kafka_${scala.binary.version}</flink.connector.kafka.artifactId>
+        <flink.hadoop.compatibility.artifactId>flink-hadoop-compatibility_${scala.binary.version}</flink.hadoop.compatibility.artifactId>
+        <hudi.flink.module>hudi-flink1.14.x</hudi.flink.module>
+        <flink.bundle.version>1.14</flink.bundle.version>
+      </properties>
      <activation>
        <activeByDefault>true</activeByDefault>
        <property>
@@ -1773,10 +1816,18 @@
    <profile>
      <id>flink1.13</id>
      <properties>
+        <flink.scala.bnary.version>2.11</flink.scala.bnary.version>
        <flink.version>${flink1.13.version}</flink.version>
        <flink.runtime.artifactId>flink-runtime_${scala.binary.version}</flink.runtime.artifactId>
        <flink.table.runtime.artifactId>flink-table-runtime-blink_${scala.binary.version}</flink.table.runtime.artifactId>
        <flink.table.planner.artifactId>flink-table-planner-blink_${scala.binary.version}</flink.table.planner.artifactId>
+        <flink.parquet.artifactId>flink-parquet_${scala.binary.version}</flink.parquet.artifactId>
+        <flink.statebackend.rocksdb.artifactId>flink-statebackend-rocksdb_${scala.binary.version}</flink.statebackend.rocksdb.artifactId>
+        <flink.test.utils.artifactId>flink-test-utils_${scala.binary.version}</flink.test.utils.artifactId>
+        <flink.streaming.java.artifactId>flink-streaming-java_${scala.binary.version}</flink.streaming.java.artifactId>
+        <flink.clients.artifactId>flink-clients_${scala.binary.version}</flink.clients.artifactId>
+        <flink.connector.kafka.artifactId>flink-connector-kafka_${scala.binary.version}</flink.connector.kafka.artifactId>
+        <flink.hadoop.compatibility.artifactId>flink-hadoop-compatibility_${scala.binary.version}</flink.hadoop.compatibility.artifactId>
        <hudi.flink.module>hudi-flink1.13.x</hudi.flink.module>
        <flink.bundle.version>1.13</flink.bundle.version>
        <skipITs>true</skipITs>