diff --git a/bin/build-sync.sh b/bin/build-sync.sh
new file mode 100755
index 0000000..054df55
--- /dev/null
+++ b/bin/build-sync.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+mvn -pl service-common clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml
+mvn -pl utils/sync clean package -D skipTests -s ~/.m2/settings-development.xml
+ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/utils/sync/target/sync-1.0.0-SNAPSHOT.jar
diff --git a/pom.xml b/pom.xml
index 7e8e5b3..3391879 100644
--- a/pom.xml
+++ b/pom.xml
@@ -9,6 +9,7 @@
1.0.0-SNAPSHOT
pom
+ service-common
service-configuration
service-gateway
service-queue
@@ -32,6 +33,8 @@
service-scheduler
service-launcher
service-command
+ utils/executor
+ utils/sync
@@ -39,7 +42,7 @@
8
UTF-8
- b1e11
+ b2b12
2.6.8
2021.0.3
diff --git a/service-common/pom.xml b/service-common/pom.xml
new file mode 100644
index 0000000..cbe5fd1
--- /dev/null
+++ b/service-common/pom.xml
@@ -0,0 +1,41 @@
+
+
+ 4.0.0
+
+ com.lanyuanxiaoyao
+ hudi-service
+ 1.0.0-SNAPSHOT
+
+
+ service-common
+
+
+
+
+
+ cn.hutool
+ hutool-all
+
+
+
+ io.github.dragons96
+ sql-builder
+ 0.0.5.3
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-source-plugin
+
+
+ org.apache.maven.plugins
+ maven-deploy-plugin
+
+
+
+
\ No newline at end of file
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java
new file mode 100644
index 0000000..ca17da3
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java
@@ -0,0 +1,211 @@
+package com.lanyuanxiaoyao.service.common;
+
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import java.time.format.DateTimeFormatter;
+import java.util.function.BiFunction;
+
+/**
+ * 常量
+ *
+ * @author ZhangJiacheng
+ * @version 0.0.1
+ * @date 2021-12-03
+ */
+public interface Constants {
+ // String DATABASE_NAME = "hudi_collect_build";
+ // String DATABASE_NAME = "hudi_collect_build_2";
+ String DATABASE_NAME = "hudi_collect_build_b12";
+
+ String API_HEADER_NAME = "Api-Version";
+ String API_VERSION = "1";
+
+ /**
+ * 联合主键
+ */
+ String UNION_KEY_NAME = "_key";
+ /**
+ * 源端最后操作时间
+ */
+ String LATEST_OPERATION_TIMESTAMP_KEY_NAME = "latest_op_ts";
+ /**
+ * 记录下游入库时间
+ */
+ String UPDATE_TIMESTAMP_KEY_NAME = "update_ts";
+ /**
+ * Hudi 删除标记字段
+ */
+ String HUDI_DELETE_KEY_NAME = "_hoodie_is_deleted";
+
+ String PULSAR_SUBSCRIPTION_NAME_PREFIX = "Hudi_Sync_Pulsar_Reader";
+
+ String VERSION_UPDATE_KEY = "versionUpdate";
+ String VERSION_KEY = "version";
+
+ String DELETE = "D";
+ String INSERT = "I";
+ String UPDATE = "U";
+ String DDL = "ddl";
+ String UNKNOWN = "unknown";
+
+ String CITY_ID = "CITY_ID";
+ String INCLUDE = "INCLUDE";
+ String EXCLUDE = "EXCLUDE";
+
+ String JOB_ID = "job-id";
+ String SERVICE_MODE = "service-mode";
+ String FLINK_JOB = "flink-job";
+ String TABLE_META = "table-meta";
+ String TABLE_META_LIST = "table-meta-list";
+ String MESSAGE_ID = "message-id";
+ String INSTANTS = "instants";
+ String BETA = "beta";
+ String CLUSTER = "cluster";
+
+ String COW = "COPY_ON_WRITE";
+ String MOR = "MERGE_ON_READ";
+
+ String FLINK_JOB_OPTION = "-" + FLINK_JOB;
+ String TABLE_META_OPTION = "-" + TABLE_META;
+ String TABLE_META_LIST_OPTION = "-" + TABLE_META_LIST;
+ String INSTANTS_OPTION = "-" + INSTANTS;
+ String BETA_OPTION = "-" + BETA;
+ String CLUSTER_OPTION = "-" + CLUSTER;
+
+ String SPRING_SECURITY_AUTHORITY = "Anonymous";
+ String SPRING_SECURITY_USERNAME = "AxhEbscwsJDbYMH2";
+ String SPRING_SECURITY_PASSWORD = "{noop}cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4";
+ String SPRING_SECURITY_PASSWORD_PLAIN = "cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4";
+
+ String VICTORIA_USERNAME = "EsCFVuNkiDWv7PKmcF";
+ String VICTORIA_PASSWORD = "Abf%x9ocS^iKr3tgrd";
+
+ String SCHEMA_NAME = "schema";
+ String TABLE_NAME = "table";
+ String DATA_TIME = "data-time";
+ String DATA_PARENT_PATH = "data-parent-path";
+
+ String METRICS_PREFIX = "metrics_hudi";
+ String METRICS_YARN_PREFIX = METRICS_PREFIX + "_yarn";
+ String METRICS_YARN_JOB = METRICS_YARN_PREFIX + "_job";
+ String METRICS_YARN_TABLE = METRICS_YARN_PREFIX + "_table";
+ String METRICS_SYNC_PREFIX = METRICS_PREFIX + "_sync";
+ String METRICS_SYNC_SOURCE_LATENCY = METRICS_SYNC_PREFIX + "_source_latency";
+ String METRICS_SYNC_LATENCY = METRICS_SYNC_PREFIX + "_latency";
+ String METRICS_SYNC_FLINK_JOB_ID = METRICS_SYNC_PREFIX + "_flink_job_id";
+
+ String METRICS_SYNC_SOURCE_MESSAGE_RECEIVE = METRICS_SYNC_PREFIX + "_source_message_receive";
+ String METRICS_SYNC_SOURCE_MESSAGE_SIZE_RECEIVE_BYTES = METRICS_SYNC_PREFIX + "_source_message_receive_bytes";
+ String METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE = METRICS_SYNC_PREFIX + "_source_operation_type_receive";
+ String METRICS_SYNC_SOURCE_CHANGE_FILTER = METRICS_SYNC_PREFIX + "_source_change_filter";
+ String METRICS_SYNC_SOURCE_CHANGE_PARTITION = METRICS_SYNC_PREFIX + "_source_change_partition";
+ String METRICS_SYNC_SOURCE_BACK_LOGS = METRICS_SYNC_PREFIX + "_source_back_logs";
+
+ String METRICS_LABEL_FLINK_JOB_ID = "flink_job_id";
+ String METRICS_LABEL_FLINK_JOB_NAME = "flink_job_name";
+ String METRICS_LABEL_FLINK_NATIVE_JOB_ID = "flink_native_job_id";
+ String METRICS_LABEL_FLINK_NATIVE_TASK_NAME = "flink_native_task_name";
+ String METRICS_LABEL_FLINK_PARALLEL_ID = "flink_parallel_id";
+ String METRICS_LABEL_RUN_TYPE = "run_type";
+ String METRICS_LABEL_EXECUTOR_VERSION = "executor_version";
+ String METRICS_LABEL_CLUSTER = "cluster";
+
+ String METRICS_RUN_TYPE_SYNC = "sync";
+ String METRICS_RUN_TYPE_COMPACTION = "compaction";
+
+ String METRICS_LABEL_SCHEMA = "schema";
+ String METRICS_LABEL_TABLE = "table";
+ String METRICS_LABEL_STATUS = "status";
+ String METRICS_LABEL_TOPIC = "topic";
+ String METRICS_LABEL_BATCH_ID = "batch_id";
+ String METRICS_LABEL_ALIAS = "alias";
+ String METRICS_LABEL_APPLICATION_ID = "application_id";
+
+ String METRICS_STATUS_RUNNING = "running";
+ String METRICS_STATUS_STOPPED = "stopped";
+
+ String METRICS_LABEL_TYPE = "type";
+
+ String LOKI_PUSH_URL = "loki_push_url";
+
+ DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ String OPERATION_DONE = "操作完成";
+ String OPERATION_CANCEL = "操作取消";
+ String FETCHING_DATA = "Fetching Data";
+
+ String COMPACTION_STATUS_SCHEDULE = "SCHEDULE";
+ String COMPACTION_STATUS_START = "START";
+ String COMPACTION_STATUS_FINISH = "FINISH";
+ String COMPACTION_STATUS_FAILURE = "FAILURE";
+
+ long SECOND = 1000;
+ long HALF_MINUTE = 30 * SECOND;
+ long MINUTE = 60 * SECOND;
+ long HALF_HOUR = 30 * MINUTE;
+ long HOUR = 60 * MINUTE;
+
+ long KB = 1024;
+ long MB = 1024 * KB;
+ long GB = 1024 * MB;
+ long TB = 1024 * GB;
+
+ String TAG_SPLIT = ";";
+ String TAG_OPERATOR = "=";
+
+ String EVENT = "event";
+ String FROM_COMMAND_UTIL = "command util";
+ String FROM_COMPACTOR = "compactor";
+
+ int COMMAND_RENDER_WIDTH = 500;
+
+ String LOG_FLINK_JOB_ID_LABEL = "LOG_FLINK_JOB_ID_LABEL";
+ String LOG_FLINK_JOB_ID = "flink_job_id";
+ String LOG_ALIAS_LABEL = "LOG_ALIAS_LABEL";
+ String LOG_ALIAS = "alias";
+ String LOG_JOB_ID_LABEL = "LOG_JOB_ID_LABEL";
+ String LOG_JOB_ID = "job_id";
+
+ String LOG_POINT_PREFIX = "LOP-";
+ String LOG_POINT_MESSAGE_ID_EMPTY = LOG_POINT_PREFIX + "000001";
+ String LOG_POINT_CHECKPOINT_INITIAL = LOG_POINT_PREFIX + "000002";
+ String LOG_POINT_CHECKPOINT_INITIAL_MESSAGE_ID = LOG_POINT_PREFIX + "000003";
+ String LOG_POINT_PULSAR_SOURCE_BOOTSTRAP_MESSAGE_ID = LOG_POINT_PREFIX + "000004";
+ String LOG_POINT_PULSAR_SOURCE_GET_MESSAGE_ID_ERROR = LOG_POINT_PREFIX + "000005";
+ String LOG_POINT_FIELD_TYPE_NOT_FOUND = LOG_POINT_PREFIX + "000006";
+
+ String TAGS_NO_COMPACT = "NO_COMPACT";
+ String TAGS_PULSAR_BACKUP = "PULSAR_BACKUP";
+ String TAGS_NO_PRE_COMBINE = "NO_PRE_COMBINE";
+ String TAGS_PRE_COMBINE = "PRE_COMBINE";
+ String TAGS_NO_IGNORE_FAILED = "NO_IGNORE_FAILED";
+ String TAGS_DISABLE_CHAINING = "DISABLE_CHAINING";
+ String TAGS_TRACE_LATEST_OP_TS = "TRACE_LATEST_OP_TS";
+ String TAGS_SOURCE_READER = "SOURCE_READER";
+ String TAGS_USE_TEST_JAR = "USE_TEST_JAR";
+ String TAGS_ODS = "ODS";
+ String TAGS_ODS_FOCUS = "ODS_FOCUS";
+
+ String COMPACTION_QUEUE_PRE = "compaction-queue-pre";
+ String COMPACTION_QUEUE_B1 = "compaction-queue-b1";
+ String COMPACTION_QUEUE_B5 = "compaction-queue-b5";
+ String COMPACTION_QUEUE_A4 = "compaction-queue-a4";
+ String COMPACTION_QUEUE_B12 = "compaction-queue-b12";
+
+ String CLUSTER_B1 = "b1";
+ String CLUSTER_B5 = "b5";
+ String CLUSTER_A4 = "a4";
+ String CLUSTER_B12 = "b12";
+
+ String SCHEDULE_JOB_FAIL_COUNT = "SCHEDULE_JOB_FAIL_COUNT";
+
+ String SCHEDULE_RECOMMEND = "schedule_recommend";
+ String SCHEDULE_FORCE = "schedule_force";
+
+ BiFunction FIELD_COVERT = (tableMeta, field) -> {
+ if (TableMeta.SourceType.TELEPG.equals(tableMeta.getSourceType())) {
+ return field.toLowerCase();
+ } else {
+ return field;
+ }
+ };
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/SQLConstants.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/SQLConstants.java
new file mode 100644
index 0000000..35340bd
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/SQLConstants.java
@@ -0,0 +1,1713 @@
+package com.lanyuanxiaoyao.service.common;
+
+import club.kingon.sql.builder.entry.Alias;
+
+/**
+ * SQL
+ */
+@SuppressWarnings({"unused", "SpellCheckingInspection"})
+public interface SQLConstants {
+ /**
+ * hudi_collect_build
+ */
+ interface HudiCollectBuild {
+ /**
+ * Schema 名称
+ */
+ String _name_ = Constants.DATABASE_NAME;
+
+ /**
+ * tb_app_collect_table_info
+ */
+ interface TbAppCollectTableInfo {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_collect_table_info";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_collect_table_info", "tacti");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tacti.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 alias 原始值 alias 表别名
+ */
+ String ALIAS_O = "alias";
+ /**
+ * 字段 alias 别名值 tacti.alias 表别名
+ */
+ String ALIAS_A = _alias_.getAlias() + "." + ALIAS_O;
+ /**
+ * 字段 flink_job_id 原始值 flink_job_id Flink Job Id
+ */
+ String FLINK_JOB_ID_O = "flink_job_id";
+ /**
+ * 字段 flink_job_id 别名值 tacti.flink_job_id Flink Job Id
+ */
+ String FLINK_JOB_ID_A = _alias_.getAlias() + "." + FLINK_JOB_ID_O;
+ /**
+ * 字段 hudi_job_id 原始值 hudi_job_id hudi 配置
+ */
+ String HUDI_JOB_ID_O = "hudi_job_id";
+ /**
+ * 字段 hudi_job_id 别名值 tacti.hudi_job_id hudi 配置
+ */
+ String HUDI_JOB_ID_A = _alias_.getAlias() + "." + HUDI_JOB_ID_O;
+ /**
+ * 字段 sync_yarn_job_id 原始值 sync_yarn_job_id 同步 yarn 配置
+ */
+ String SYNC_YARN_JOB_ID_O = "sync_yarn_job_id";
+ /**
+ * 字段 sync_yarn_job_id 别名值 tacti.sync_yarn_job_id 同步 yarn 配置
+ */
+ String SYNC_YARN_JOB_ID_A = _alias_.getAlias() + "." + SYNC_YARN_JOB_ID_O;
+ /**
+ * 字段 compaction_yarn_job_id 原始值 compaction_yarn_job_id 压缩 yarn 配置
+ */
+ String COMPACTION_YARN_JOB_ID_O = "compaction_yarn_job_id";
+ /**
+ * 字段 compaction_yarn_job_id 别名值 tacti.compaction_yarn_job_id 压缩 yarn 配置
+ */
+ String COMPACTION_YARN_JOB_ID_A = _alias_.getAlias() + "." + COMPACTION_YARN_JOB_ID_O;
+ /**
+ * 字段 config_id 原始值 config_id 全局配置
+ */
+ String CONFIG_ID_O = "config_id";
+ /**
+ * 字段 config_id 别名值 tacti.config_id 全局配置
+ */
+ String CONFIG_ID_A = _alias_.getAlias() + "." + CONFIG_ID_O;
+ /**
+ * 字段 src_db 原始值 src_db 源数据库
+ */
+ String SRC_DB_O = "src_db";
+ /**
+ * 字段 src_db 别名值 tacti.src_db 源数据库
+ */
+ String SRC_DB_A = _alias_.getAlias() + "." + SRC_DB_O;
+ /**
+ * 字段 src_type 原始值 src_type 数据源类型
+ */
+ String SRC_TYPE_O = "src_type";
+ /**
+ * 字段 src_type 别名值 tacti.src_type 数据源类型
+ */
+ String SRC_TYPE_A = _alias_.getAlias() + "." + SRC_TYPE_O;
+ /**
+ * 字段 src_schema 原始值 src_schema 源库
+ */
+ String SRC_SCHEMA_O = "src_schema";
+ /**
+ * 字段 src_schema 别名值 tacti.src_schema 源库
+ */
+ String SRC_SCHEMA_A = _alias_.getAlias() + "." + SRC_SCHEMA_O;
+ /**
+ * 字段 src_table 原始值 src_table 源表
+ */
+ String SRC_TABLE_O = "src_table";
+ /**
+ * 字段 src_table 别名值 tacti.src_table 源表
+ */
+ String SRC_TABLE_A = _alias_.getAlias() + "." + SRC_TABLE_O;
+ /**
+ * 字段 src_pulsar_addr 原始值 src_pulsar_addr pulsar 地址
+ */
+ String SRC_PULSAR_ADDR_O = "src_pulsar_addr";
+ /**
+ * 字段 src_pulsar_addr 别名值 tacti.src_pulsar_addr pulsar 地址
+ */
+ String SRC_PULSAR_ADDR_A = _alias_.getAlias() + "." + SRC_PULSAR_ADDR_O;
+ /**
+ * 字段 src_topic 原始值 src_topic pulsar topic
+ */
+ String SRC_TOPIC_O = "src_topic";
+ /**
+ * 字段 src_topic 别名值 tacti.src_topic pulsar topic
+ */
+ String SRC_TOPIC_A = _alias_.getAlias() + "." + SRC_TOPIC_O;
+ /**
+ * 字段 tgt_db 原始值 tgt_db 目标库
+ */
+ String TGT_DB_O = "tgt_db";
+ /**
+ * 字段 tgt_db 别名值 tacti.tgt_db 目标库
+ */
+ String TGT_DB_A = _alias_.getAlias() + "." + TGT_DB_O;
+ /**
+ * 字段 tgt_table 原始值 tgt_table 目标表
+ */
+ String TGT_TABLE_O = "tgt_table";
+ /**
+ * 字段 tgt_table 别名值 tacti.tgt_table 目标表
+ */
+ String TGT_TABLE_A = _alias_.getAlias() + "." + TGT_TABLE_O;
+ /**
+ * 字段 tgt_hdfs_path 原始值 tgt_hdfs_path 目标 HDFS 路径
+ */
+ String TGT_HDFS_PATH_O = "tgt_hdfs_path";
+ /**
+ * 字段 tgt_hdfs_path 别名值 tacti.tgt_hdfs_path 目标 HDFS 路径
+ */
+ String TGT_HDFS_PATH_A = _alias_.getAlias() + "." + TGT_HDFS_PATH_O;
+ /**
+ * 字段 tgt_table_type 原始值 tgt_table_type 表类型
+ */
+ String TGT_TABLE_TYPE_O = "tgt_table_type";
+ /**
+ * 字段 tgt_table_type 别名值 tacti.tgt_table_type 表类型
+ */
+ String TGT_TABLE_TYPE_A = _alias_.getAlias() + "." + TGT_TABLE_TYPE_O;
+ /**
+ * 字段 status 原始值 status 记录状态
+ */
+ String STATUS_O = "status";
+ /**
+ * 字段 status 别名值 tacti.status 记录状态
+ */
+ String STATUS_A = _alias_.getAlias() + "." + STATUS_O;
+ /**
+ * 字段 filter_field 原始值 filter_field 过滤字段
+ */
+ String FILTER_FIELD_O = "filter_field";
+ /**
+ * 字段 filter_field 别名值 tacti.filter_field 过滤字段
+ */
+ String FILTER_FIELD_A = _alias_.getAlias() + "." + FILTER_FIELD_O;
+ /**
+ * 字段 filter_values 原始值 filter_values 过滤值
+ */
+ String FILTER_VALUES_O = "filter_values";
+ /**
+ * 字段 filter_values 别名值 tacti.filter_values 过滤值
+ */
+ String FILTER_VALUES_A = _alias_.getAlias() + "." + FILTER_VALUES_O;
+ /**
+ * 字段 filter_type 原始值 filter_type 过滤类型
+ */
+ String FILTER_TYPE_O = "filter_type";
+ /**
+ * 字段 filter_type 别名值 tacti.filter_type 过滤类型
+ */
+ String FILTER_TYPE_A = _alias_.getAlias() + "." + FILTER_TYPE_O;
+ /**
+ * 字段 bucket_number 原始值 bucket_number hash 值
+ */
+ String BUCKET_NUMBER_O = "bucket_number";
+ /**
+ * 字段 bucket_number 别名值 tacti.bucket_number hash 值
+ */
+ String BUCKET_NUMBER_A = _alias_.getAlias() + "." + BUCKET_NUMBER_O;
+ /**
+ * 字段 partition_field 原始值 partition_field 分区字段
+ */
+ String PARTITION_FIELD_O = "partition_field";
+ /**
+ * 字段 partition_field 别名值 tacti.partition_field 分区字段
+ */
+ String PARTITION_FIELD_A = _alias_.getAlias() + "." + PARTITION_FIELD_O;
+ /**
+ * 字段 update_time 原始值 update_time
+ */
+ String UPDATE_TIME_O = "update_time";
+ /**
+ * 字段 update_time 别名值 tacti.update_time
+ */
+ String UPDATE_TIME_A = _alias_.getAlias() + "." + UPDATE_TIME_O;
+ /**
+ * 字段 priority 原始值 priority 优先级
+ */
+ String PRIORITY_O = "priority";
+ /**
+ * 字段 priority 别名值 tacti.priority 优先级
+ */
+ String PRIORITY_A = _alias_.getAlias() + "." + PRIORITY_O;
+ /**
+ * 字段 schedule_id 原始值 schedule_id 压缩调度计划
+ */
+ String SCHEDULE_ID_O = "schedule_id";
+ /**
+ * 字段 schedule_id 别名值 tacti.schedule_id 压缩调度计划
+ */
+ String SCHEDULE_ID_A = _alias_.getAlias() + "." + SCHEDULE_ID_O;
+ /**
+ * 字段 hive_db 原始值 hive_db
+ */
+ String HIVE_DB_O = "hive_db";
+ /**
+ * 字段 hive_db 别名值 tacti.hive_db
+ */
+ String HIVE_DB_A = _alias_.getAlias() + "." + HIVE_DB_O;
+ /**
+ * 字段 hive_table 原始值 hive_table
+ */
+ String HIVE_TABLE_O = "hive_table";
+ /**
+ * 字段 hive_table 别名值 tacti.hive_table
+ */
+ String HIVE_TABLE_A = _alias_.getAlias() + "." + HIVE_TABLE_O;
+ /**
+ * 字段 tags 原始值 tags 标签
+ */
+ String TAGS_O = "tags";
+ /**
+ * 字段 tags 别名值 tacti.tags 标签
+ */
+ String TAGS_A = _alias_.getAlias() + "." + TAGS_O;
+ /**
+ * 字段 version 原始值 version 表版本
+ */
+ String VERSION_O = "version";
+ /**
+ * 字段 version 别名值 tacti.version 表版本
+ */
+ String VERSION_A = _alias_.getAlias() + "." + VERSION_O;
+ }
+
+ /**
+ * tb_app_collect_table_version 跨天版本记录表
+ */
+ interface TbAppCollectTableVersion {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_collect_table_version";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_collect_table_version", "tactv");
+ /**
+ * 字段 flink_job_id 原始值 flink_job_id Flink Job ID
+ */
+ String FLINK_JOB_ID_O = "flink_job_id";
+ /**
+ * 字段 flink_job_id 别名值 tactv.flink_job_id Flink Job ID
+ */
+ String FLINK_JOB_ID_A = _alias_.getAlias() + "." + FLINK_JOB_ID_O;
+ /**
+ * 字段 alias 原始值 alias 表别名
+ */
+ String ALIAS_O = "alias";
+ /**
+ * 字段 alias 别名值 tactv.alias 表别名
+ */
+ String ALIAS_A = _alias_.getAlias() + "." + ALIAS_O;
+ /**
+ * 字段 version 原始值 version 版本
+ */
+ String VERSION_O = "version";
+ /**
+ * 字段 version 别名值 tactv.version 版本
+ */
+ String VERSION_A = _alias_.getAlias() + "." + VERSION_O;
+ /**
+ * 字段 op_ts 原始值 op_ts 操作时间
+ */
+ String OP_TS_O = "op_ts";
+ /**
+ * 字段 op_ts 别名值 tactv.op_ts 操作时间
+ */
+ String OP_TS_A = _alias_.getAlias() + "." + OP_TS_O;
+ /**
+ * 字段 create_time 原始值 create_time 记录创建时间
+ */
+ String CREATE_TIME_O = "create_time";
+ /**
+ * 字段 create_time 别名值 tactv.create_time 记录创建时间
+ */
+ String CREATE_TIME_A = _alias_.getAlias() + "." + CREATE_TIME_O;
+ /**
+ * 字段 update_time 原始值 update_time 记录创建时间
+ */
+ String UPDATE_TIME_O = "update_time";
+ /**
+ * 字段 update_time 别名值 tactv.update_time 记录创建时间
+ */
+ String UPDATE_TIME_A = _alias_.getAlias() + "." + UPDATE_TIME_O;
+ /**
+ * 字段 scheduled 原始值 scheduled
+ */
+ String SCHEDULED_O = "scheduled";
+ /**
+ * 字段 scheduled 别名值 tactv.scheduled
+ */
+ String SCHEDULED_A = _alias_.getAlias() + "." + SCHEDULED_O;
+ }
+
+ /**
+ * tb_app_flink_job_config
+ */
+ interface TbAppFlinkJobConfig {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_flink_job_config";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_flink_job_config", "tafjc");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tafjc.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 name 原始值 name 名称
+ */
+ String NAME_O = "name";
+ /**
+ * 字段 name 别名值 tafjc.name 名称
+ */
+ String NAME_A = _alias_.getAlias() + "." + NAME_O;
+ /**
+ * 字段 status 原始值 status 记录状态
+ */
+ String STATUS_O = "status";
+ /**
+ * 字段 status 别名值 tafjc.status 记录状态
+ */
+ String STATUS_A = _alias_.getAlias() + "." + STATUS_O;
+ /**
+ * 字段 comment 原始值 comment 注释
+ */
+ String COMMENT_O = "comment";
+ /**
+ * 字段 comment 别名值 tafjc.comment 注释
+ */
+ String COMMENT_A = _alias_.getAlias() + "." + COMMENT_O;
+ /**
+ * 字段 application_id 原始值 application_id 同步 yarn id
+ */
+ String APPLICATION_ID_O = "application_id";
+ /**
+ * 字段 application_id 别名值 tafjc.application_id 同步 yarn id
+ */
+ String APPLICATION_ID_A = _alias_.getAlias() + "." + APPLICATION_ID_O;
+ /**
+ * 字段 run_mode 原始值 run_mode 运行模式
+ */
+ String RUN_MODE_O = "run_mode";
+ /**
+ * 字段 run_mode 别名值 tafjc.run_mode 运行模式
+ */
+ String RUN_MODE_A = _alias_.getAlias() + "." + RUN_MODE_O;
+ /**
+ * 字段 one_in_one_yarn_job_id 原始值 one_in_one_yarn_job_id ONE_IN_ONE yarn 配置
+ */
+ String ONE_IN_ONE_YARN_JOB_ID_O = "one_in_one_yarn_job_id";
+ /**
+ * 字段 one_in_one_yarn_job_id 别名值 tafjc.one_in_one_yarn_job_id ONE_IN_ONE yarn 配置
+ */
+ String ONE_IN_ONE_YARN_JOB_ID_A = _alias_.getAlias() + "." + ONE_IN_ONE_YARN_JOB_ID_O;
+ }
+
+ /**
+ * tb_app_global_config 通用配置
+ */
+ interface TbAppGlobalConfig {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_global_config";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_global_config", "tagc");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tagc.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 metric_publish_url 原始值 metric_publish_url victoria 指标服务地址
+ */
+ String METRIC_PUBLISH_URL_O = "metric_publish_url";
+ /**
+ * 字段 metric_publish_url 别名值 tagc.metric_publish_url victoria 指标服务地址
+ */
+ String METRIC_PUBLISH_URL_A = _alias_.getAlias() + "." + METRIC_PUBLISH_URL_O;
+ /**
+ * 字段 metric_prometheus_url 原始值 metric_prometheus_url 指标推送 prometheus 服务地址
+ */
+ String METRIC_PROMETHEUS_URL_O = "metric_prometheus_url";
+ /**
+ * 字段 metric_prometheus_url 别名值 tagc.metric_prometheus_url 指标推送 prometheus 服务地址
+ */
+ String METRIC_PROMETHEUS_URL_A = _alias_.getAlias() + "." + METRIC_PROMETHEUS_URL_O;
+ /**
+ * 字段 metric_api_url 原始值 metric_api_url api 推送地址
+ */
+ String METRIC_API_URL_O = "metric_api_url";
+ /**
+ * 字段 metric_api_url 别名值 tagc.metric_api_url api 推送地址
+ */
+ String METRIC_API_URL_A = _alias_.getAlias() + "." + METRIC_API_URL_O;
+ /**
+ * 字段 metric_publish_delay 原始值 metric_publish_delay
+ */
+ String METRIC_PUBLISH_DELAY_O = "metric_publish_delay";
+ /**
+ * 字段 metric_publish_delay 别名值 tagc.metric_publish_delay
+ */
+ String METRIC_PUBLISH_DELAY_A = _alias_.getAlias() + "." + METRIC_PUBLISH_DELAY_O;
+ /**
+ * 字段 metric_publish_period 原始值 metric_publish_period
+ */
+ String METRIC_PUBLISH_PERIOD_O = "metric_publish_period";
+ /**
+ * 字段 metric_publish_period 别名值 tagc.metric_publish_period
+ */
+ String METRIC_PUBLISH_PERIOD_A = _alias_.getAlias() + "." + METRIC_PUBLISH_PERIOD_O;
+ /**
+ * 字段 metric_publish_timeout 原始值 metric_publish_timeout
+ */
+ String METRIC_PUBLISH_TIMEOUT_O = "metric_publish_timeout";
+ /**
+ * 字段 metric_publish_timeout 别名值 tagc.metric_publish_timeout
+ */
+ String METRIC_PUBLISH_TIMEOUT_A = _alias_.getAlias() + "." + METRIC_PUBLISH_TIMEOUT_O;
+ /**
+ * 字段 metric_publish_batch 原始值 metric_publish_batch
+ */
+ String METRIC_PUBLISH_BATCH_O = "metric_publish_batch";
+ /**
+ * 字段 metric_publish_batch 别名值 tagc.metric_publish_batch
+ */
+ String METRIC_PUBLISH_BATCH_A = _alias_.getAlias() + "." + METRIC_PUBLISH_BATCH_O;
+ /**
+ * 字段 checkpoint_root_path 原始值 checkpoint_root_path checkpoint 信息保存路径
+ */
+ String CHECKPOINT_ROOT_PATH_O = "checkpoint_root_path";
+ /**
+ * 字段 checkpoint_root_path 别名值 tagc.checkpoint_root_path checkpoint 信息保存路径
+ */
+ String CHECKPOINT_ROOT_PATH_A = _alias_.getAlias() + "." + CHECKPOINT_ROOT_PATH_O;
+ /**
+ * 字段 zk_url 原始值 zk_url
+ */
+ String ZK_URL_O = "zk_url";
+ /**
+ * 字段 zk_url 别名值 tagc.zk_url
+ */
+ String ZK_URL_A = _alias_.getAlias() + "." + ZK_URL_O;
+ }
+
+ /**
+ * tb_app_hudi_compaction_job 压缩调度任务队列
+ */
+ interface TbAppHudiCompactionJob {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_hudi_compaction_job";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_hudi_compaction_job", "tahcj");
+ /**
+ * 字段 flink_job_id 原始值 flink_job_id
+ */
+ String FLINK_JOB_ID_O = "flink_job_id";
+ /**
+ * 字段 flink_job_id 别名值 tahcj.flink_job_id
+ */
+ String FLINK_JOB_ID_A = _alias_.getAlias() + "." + FLINK_JOB_ID_O;
+ /**
+ * 字段 alias 原始值 alias 别名
+ */
+ String ALIAS_O = "alias";
+ /**
+ * 字段 alias 别名值 tahcj.alias 别名
+ */
+ String ALIAS_A = _alias_.getAlias() + "." + ALIAS_O;
+ /**
+ * 字段 comment 原始值 comment 备注
+ */
+ String COMMENT_O = "comment";
+ /**
+ * 字段 comment 别名值 tahcj.comment 备注
+ */
+ String COMMENT_A = _alias_.getAlias() + "." + COMMENT_O;
+ /**
+ * 字段 application_id 原始值 application_id
+ */
+ String APPLICATION_ID_O = "application_id";
+ /**
+ * 字段 application_id 别名值 tahcj.application_id
+ */
+ String APPLICATION_ID_A = _alias_.getAlias() + "." + APPLICATION_ID_O;
+ /**
+ * 字段 create_time 原始值 create_time 创建时间
+ */
+ String CREATE_TIME_O = "create_time";
+ /**
+ * 字段 create_time 别名值 tahcj.create_time 创建时间
+ */
+ String CREATE_TIME_A = _alias_.getAlias() + "." + CREATE_TIME_O;
+ /**
+ * 字段 update_time 原始值 update_time 更新时间
+ */
+ String UPDATE_TIME_O = "update_time";
+ /**
+ * 字段 update_time 别名值 tahcj.update_time 更新时间
+ */
+ String UPDATE_TIME_A = _alias_.getAlias() + "." + UPDATE_TIME_O;
+ }
+
+ /**
+ * tb_app_hudi_compaction_metrics 压缩计划执行具体指标
+ */
+ interface TbAppHudiCompactionMetrics {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_hudi_compaction_metrics";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_hudi_compaction_metrics", "tahcm");
+ /**
+ * 字段 flink_job_id 原始值 flink_job_id flink_job_id
+ */
+ String FLINK_JOB_ID_O = "flink_job_id";
+ /**
+ * 字段 flink_job_id 别名值 tahcm.flink_job_id flink_job_id
+ */
+ String FLINK_JOB_ID_A = _alias_.getAlias() + "." + FLINK_JOB_ID_O;
+ /**
+ * 字段 alias 原始值 alias 表别名
+ */
+ String ALIAS_O = "alias";
+ /**
+ * 字段 alias 别名值 tahcm.alias 表别名
+ */
+ String ALIAS_A = _alias_.getAlias() + "." + ALIAS_O;
+ /**
+ * 字段 cluster 原始值 cluster
+ */
+ String CLUSTER_O = "cluster";
+ /**
+ * 字段 cluster 别名值 tahcm.cluster
+ */
+ String CLUSTER_A = _alias_.getAlias() + "." + CLUSTER_O;
+ /**
+ * 字段 application_id 原始值 application_id
+ */
+ String APPLICATION_ID_O = "application_id";
+ /**
+ * 字段 application_id 别名值 tahcm.application_id
+ */
+ String APPLICATION_ID_A = _alias_.getAlias() + "." + APPLICATION_ID_O;
+ /**
+ * 字段 type 原始值 type 类型
+ */
+ String TYPE_O = "type";
+ /**
+ * 字段 type 别名值 tahcm.type 类型
+ */
+ String TYPE_A = _alias_.getAlias() + "." + TYPE_O;
+ /**
+ * 字段 source_schema 原始值 source_schema 库名
+ */
+ String SOURCE_SCHEMA_O = "source_schema";
+ /**
+ * 字段 source_schema 别名值 tahcm.source_schema 库名
+ */
+ String SOURCE_SCHEMA_A = _alias_.getAlias() + "." + SOURCE_SCHEMA_O;
+ /**
+ * 字段 source_table 原始值 source_table 表名
+ */
+ String SOURCE_TABLE_O = "source_table";
+ /**
+ * 字段 source_table 别名值 tahcm.source_table 表名
+ */
+ String SOURCE_TABLE_A = _alias_.getAlias() + "." + SOURCE_TABLE_O;
+ /**
+ * 字段 compaction_plan_instant 原始值 compaction_plan_instant 压缩计划时间点
+ */
+ String COMPACTION_PLAN_INSTANT_O = "compaction_plan_instant";
+ /**
+ * 字段 compaction_plan_instant 别名值 tahcm.compaction_plan_instant 压缩计划时间点
+ */
+ String COMPACTION_PLAN_INSTANT_A = _alias_.getAlias() + "." + COMPACTION_PLAN_INSTANT_O;
+ /**
+ * 字段 total_scan_time 原始值 total_scan_time 扫描日志文件时长
+ */
+ String TOTAL_SCAN_TIME_O = "total_scan_time";
+ /**
+ * 字段 total_scan_time 别名值 tahcm.total_scan_time 扫描日志文件时长
+ */
+ String TOTAL_SCAN_TIME_A = _alias_.getAlias() + "." + TOTAL_SCAN_TIME_O;
+ /**
+ * 字段 total_log_files_compacted 原始值 total_log_files_compacted 压缩文件个数
+ */
+ String TOTAL_LOG_FILES_COMPACTED_O = "total_log_files_compacted";
+ /**
+ * 字段 total_log_files_compacted 别名值 tahcm.total_log_files_compacted 压缩文件个数
+ */
+ String TOTAL_LOG_FILES_COMPACTED_A = _alias_.getAlias() + "." + TOTAL_LOG_FILES_COMPACTED_O;
+ /**
+ * 字段 total_log_files_size 原始值 total_log_files_size 压缩文件大小
+ */
+ String TOTAL_LOG_FILES_SIZE_O = "total_log_files_size";
+ /**
+ * 字段 total_log_files_size 别名值 tahcm.total_log_files_size 压缩文件大小
+ */
+ String TOTAL_LOG_FILES_SIZE_A = _alias_.getAlias() + "." + TOTAL_LOG_FILES_SIZE_O;
+ /**
+ * 字段 total_records_deleted 原始值 total_records_deleted 删除记录数
+ */
+ String TOTAL_RECORDS_DELETED_O = "total_records_deleted";
+ /**
+ * 字段 total_records_deleted 别名值 tahcm.total_records_deleted 删除记录数
+ */
+ String TOTAL_RECORDS_DELETED_A = _alias_.getAlias() + "." + TOTAL_RECORDS_DELETED_O;
+ /**
+ * 字段 total_records_updated 原始值 total_records_updated 更新记录数
+ */
+ String TOTAL_RECORDS_UPDATED_O = "total_records_updated";
+ /**
+ * 字段 total_records_updated 别名值 tahcm.total_records_updated 更新记录数
+ */
+ String TOTAL_RECORDS_UPDATED_A = _alias_.getAlias() + "." + TOTAL_RECORDS_UPDATED_O;
+ /**
+ * 字段 total_records_compacted 原始值 total_records_compacted 压缩记录数
+ */
+ String TOTAL_RECORDS_COMPACTED_O = "total_records_compacted";
+ /**
+ * 字段 total_records_compacted 别名值 tahcm.total_records_compacted 压缩记录数
+ */
+ String TOTAL_RECORDS_COMPACTED_A = _alias_.getAlias() + "." + TOTAL_RECORDS_COMPACTED_O;
+ /**
+ * 字段 update_time 原始值 update_time 记录更新时间
+ */
+ String UPDATE_TIME_O = "update_time";
+ /**
+ * 字段 update_time 别名值 tahcm.update_time 记录更新时间
+ */
+ String UPDATE_TIME_A = _alias_.getAlias() + "." + UPDATE_TIME_O;
+ }
+
+ /**
+ * tb_app_hudi_job_config
+ */
+ interface TbAppHudiJobConfig {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_hudi_job_config";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_hudi_job_config", "tahjc");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tahjc.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 source_tasks 原始值 source_tasks Pulsar 源端并行度
+ */
+ String SOURCE_TASKS_O = "source_tasks";
+ /**
+ * 字段 source_tasks 别名值 tahjc.source_tasks Pulsar 源端并行度
+ */
+ String SOURCE_TASKS_A = _alias_.getAlias() + "." + SOURCE_TASKS_O;
+ /**
+ * 字段 write_tasks 原始值 write_tasks 写并行度
+ */
+ String WRITE_TASKS_O = "write_tasks";
+ /**
+ * 字段 write_tasks 别名值 tahjc.write_tasks 写并行度
+ */
+ String WRITE_TASKS_A = _alias_.getAlias() + "." + WRITE_TASKS_O;
+ /**
+ * 字段 write_operation 原始值 write_operation 写操作类型（默认为 upsert）
+ */
+ String WRITE_OPERATION_O = "write_operation";
+ /**
+ * 字段 write_operation 别名值 tahjc.write_operation 写操作类型（默认为 upsert）
+ */
+ String WRITE_OPERATION_A = _alias_.getAlias() + "." + WRITE_OPERATION_O;
+ /**
+ * 字段 write_task_max_memory 原始值 write_task_max_memory 写任务最大缓存
+ */
+ String WRITE_TASK_MAX_MEMORY_O = "write_task_max_memory";
+ /**
+ * 字段 write_task_max_memory 别名值 tahjc.write_task_max_memory 写任务最大缓存
+ */
+ String WRITE_TASK_MAX_MEMORY_A = _alias_.getAlias() + "." + WRITE_TASK_MAX_MEMORY_O;
+ /**
+ * 字段 write_batch_size 原始值 write_batch_size 写任务单次操作大小
+ */
+ String WRITE_BATCH_SIZE_O = "write_batch_size";
+ /**
+ * 字段 write_batch_size 别名值 tahjc.write_batch_size 写任务单次操作大小
+ */
+ String WRITE_BATCH_SIZE_A = _alias_.getAlias() + "." + WRITE_BATCH_SIZE_O;
+ /**
+ * 字段 write_rate_limit 原始值 write_rate_limit 写任务速度限制（默认为10000TPS）
+ */
+ String WRITE_RATE_LIMIT_O = "write_rate_limit";
+ /**
+ * 字段 write_rate_limit 别名值 tahjc.write_rate_limit 写任务速度限制（默认为10000TPS）
+ */
+ String WRITE_RATE_LIMIT_A = _alias_.getAlias() + "." + WRITE_RATE_LIMIT_O;
+ /**
+ * 字段 compaction_strategy 原始值 compaction_strategy 压缩策略
+ */
+ String COMPACTION_STRATEGY_O = "compaction_strategy";
+ /**
+ * 字段 compaction_strategy 别名值 tahjc.compaction_strategy 压缩策略
+ */
+ String COMPACTION_STRATEGY_A = _alias_.getAlias() + "." + COMPACTION_STRATEGY_O;
+ /**
+ * 字段 compaction_tasks 原始值 compaction_tasks 压缩任务并行度
+ */
+ String COMPACTION_TASKS_O = "compaction_tasks";
+ /**
+ * 字段 compaction_tasks 别名值 tahjc.compaction_tasks 压缩任务并行度
+ */
+ String COMPACTION_TASKS_A = _alias_.getAlias() + "." + COMPACTION_TASKS_O;
+ /**
+ * 字段 compaction_delta_commits 原始值 compaction_delta_commits 压缩最小间隔 commit 数量
+ */
+ String COMPACTION_DELTA_COMMITS_O = "compaction_delta_commits";
+ /**
+ * 字段 compaction_delta_commits 别名值 tahjc.compaction_delta_commits 压缩最小间隔 commit 数量
+ */
+ String COMPACTION_DELTA_COMMITS_A = _alias_.getAlias() + "." + COMPACTION_DELTA_COMMITS_O;
+ /**
+ * 字段 compaction_delta_seconds 原始值 compaction_delta_seconds 压缩最小间隔时间（秒）
+ */
+ String COMPACTION_DELTA_SECONDS_O = "compaction_delta_seconds";
+ /**
+ * 字段 compaction_delta_seconds 别名值 tahjc.compaction_delta_seconds 压缩最小间隔时间（秒）
+ */
+ String COMPACTION_DELTA_SECONDS_A = _alias_.getAlias() + "." + COMPACTION_DELTA_SECONDS_O;
+ /**
+ * 字段 compaction_async_enabled 原始值 compaction_async_enabled 同步压缩是否开启（默认关闭，目前该参数无效）
+ */
+ String COMPACTION_ASYNC_ENABLED_O = "compaction_async_enabled";
+ /**
+ * 字段 compaction_async_enabled 别名值 tahjc.compaction_async_enabled 同步压缩是否开启（默认关闭，目前该参数无效）
+ */
+ String COMPACTION_ASYNC_ENABLED_A = _alias_.getAlias() + "." + COMPACTION_ASYNC_ENABLED_O;
+ /**
+ * 字段 compaction_max_memory 原始值 compaction_max_memory 压缩任务最大使用缓存大小
+ */
+ String COMPACTION_MAX_MEMORY_O = "compaction_max_memory";
+ /**
+ * 字段 compaction_max_memory 别名值 tahjc.compaction_max_memory 压缩任务最大使用缓存大小
+ */
+ String COMPACTION_MAX_MEMORY_A = _alias_.getAlias() + "." + COMPACTION_MAX_MEMORY_O;
+ /**
+ * 字段 keep_file_version 原始值 keep_file_version 保留数据文件版本
+ */
+ String KEEP_FILE_VERSION_O = "keep_file_version";
+ /**
+ * 字段 keep_file_version 别名值 tahjc.keep_file_version 保留数据文件版本
+ */
+ String KEEP_FILE_VERSION_A = _alias_.getAlias() + "." + KEEP_FILE_VERSION_O;
+ /**
+ * 字段 keep_commit_version 原始值 keep_commit_version 保留时间线版本
+ */
+ String KEEP_COMMIT_VERSION_O = "keep_commit_version";
+ /**
+ * 字段 keep_commit_version 别名值 tahjc.keep_commit_version 保留时间线版本
+ */
+ String KEEP_COMMIT_VERSION_A = _alias_.getAlias() + "." + KEEP_COMMIT_VERSION_O;
+ /**
+ * 字段 hive_sync_enable 原始值 hive_sync_enable hive 源数据同步启动（目前该参数不使用）
+ */
+ String HIVE_SYNC_ENABLE_O = "hive_sync_enable";
+ /**
+ * 字段 hive_sync_enable 别名值 tahjc.hive_sync_enable hive 源数据同步启动（目前该参数不使用）
+ */
+ String HIVE_SYNC_ENABLE_A = _alias_.getAlias() + "." + HIVE_SYNC_ENABLE_O;
+ /**
+ * 字段 hive_conn_id 原始值 hive_conn_id hive 连接信息标识（目前该参数不使用）
+ */
+ String HIVE_CONN_ID_O = "hive_conn_id";
+ /**
+ * 字段 hive_conn_id 别名值 tahjc.hive_conn_id hive 连接信息标识（目前该参数不使用）
+ */
+ String HIVE_CONN_ID_A = _alias_.getAlias() + "." + HIVE_CONN_ID_O;
+ /**
+ * 字段 configs 原始值 configs 自定义配置
+ */
+ String CONFIGS_O = "configs";
+ /**
+ * 字段 configs 别名值 tahjc.configs 自定义配置
+ */
+ String CONFIGS_A = _alias_.getAlias() + "." + CONFIGS_O;
+ /**
+ * 字段 status 原始值 status 逻辑删除标识
+ */
+ String STATUS_O = "status";
+ /**
+ * 字段 status 别名值 tahjc.status 逻辑删除标识
+ */
+ String STATUS_A = _alias_.getAlias() + "." + STATUS_O;
+ }
+
+ /**
+ * tb_app_hudi_sync_state
+ */
+ interface TbAppHudiSyncState {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_hudi_sync_state";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_hudi_sync_state", "tahss");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tahss.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 message_id 原始值 message_id pulsar 消息位移
+ */
+ String MESSAGE_ID_O = "message_id";
+ /**
+ * 字段 message_id 别名值 tahss.message_id pulsar 消息位移
+ */
+ String MESSAGE_ID_A = _alias_.getAlias() + "." + MESSAGE_ID_O;
+ /**
+ * 字段 source_database 原始值 source_database 源数据库
+ */
+ String SOURCE_DATABASE_O = "source_database";
+ /**
+ * 字段 source_database 别名值 tahss.source_database 源数据库
+ */
+ String SOURCE_DATABASE_A = _alias_.getAlias() + "." + SOURCE_DATABASE_O;
+ /**
+ * 字段 source_schema 原始值 source_schema 源库
+ */
+ String SOURCE_SCHEMA_O = "source_schema";
+ /**
+ * 字段 source_schema 别名值 tahss.source_schema 源库
+ */
+ String SOURCE_SCHEMA_A = _alias_.getAlias() + "." + SOURCE_SCHEMA_O;
+ /**
+ * 字段 source_table 原始值 source_table 源表
+ */
+ String SOURCE_TABLE_O = "source_table";
+ /**
+ * 字段 source_table 别名值 tahss.source_table 源表
+ */
+ String SOURCE_TABLE_A = _alias_.getAlias() + "." + SOURCE_TABLE_O;
+ /**
+ * 字段 source_start_time 原始值 source_start_time 同步启动时间
+ */
+ String SOURCE_START_TIME_O = "source_start_time";
+ /**
+ * 字段 source_start_time 别名值 tahss.source_start_time 同步启动时间
+ */
+ String SOURCE_START_TIME_A = _alias_.getAlias() + "." + SOURCE_START_TIME_O;
+ /**
+ * 字段 source_checkpoint_time 原始值 source_checkpoint_time 同步检查时间
+ */
+ String SOURCE_CHECKPOINT_TIME_O = "source_checkpoint_time";
+ /**
+ * 字段 source_checkpoint_time 别名值 tahss.source_checkpoint_time 同步检查时间
+ */
+ String SOURCE_CHECKPOINT_TIME_A = _alias_.getAlias() + "." + SOURCE_CHECKPOINT_TIME_O;
+ /**
+ * 字段 source_publish_time 原始值 source_publish_time 消息发布时间
+ */
+ String SOURCE_PUBLISH_TIME_O = "source_publish_time";
+ /**
+ * 字段 source_publish_time 别名值 tahss.source_publish_time 消息发布时间
+ */
+ String SOURCE_PUBLISH_TIME_A = _alias_.getAlias() + "." + SOURCE_PUBLISH_TIME_O;
+ /**
+ * 字段 source_op_time 原始值 source_op_time 消息业务时间
+ */
+ String SOURCE_OP_TIME_O = "source_op_time";
+ /**
+ * 字段 source_op_time 别名值 tahss.source_op_time 消息业务时间
+ */
+ String SOURCE_OP_TIME_A = _alias_.getAlias() + "." + SOURCE_OP_TIME_O;
+ /**
+ * 字段 source_application_id 原始值 source_application_id 同步程序 application id
+ */
+ String SOURCE_APPLICATION_ID_O = "source_application_id";
+ /**
+ * 字段 source_application_id 别名值 tahss.source_application_id 同步程序 application id
+ */
+ String SOURCE_APPLICATION_ID_A = _alias_.getAlias() + "." + SOURCE_APPLICATION_ID_O;
+ /**
+ * 字段 source_cluster 原始值 source_cluster 运行主机
+ */
+ String SOURCE_CLUSTER_O = "source_cluster";
+ /**
+ * 字段 source_cluster 别名值 tahss.source_cluster 运行主机
+ */
+ String SOURCE_CLUSTER_A = _alias_.getAlias() + "." + SOURCE_CLUSTER_O;
+ /**
+ * 字段 compaction_start_time 原始值 compaction_start_time 压缩启动时间
+ */
+ String COMPACTION_START_TIME_O = "compaction_start_time";
+ /**
+ * 字段 compaction_start_time 别名值 tahss.compaction_start_time 压缩启动时间
+ */
+ String COMPACTION_START_TIME_A = _alias_.getAlias() + "." + COMPACTION_START_TIME_O;
+ /**
+ * 字段 compaction_finish_time 原始值 compaction_finish_time 压缩完成时间
+ */
+ String COMPACTION_FINISH_TIME_O = "compaction_finish_time";
+ /**
+ * 字段 compaction_finish_time 别名值 tahss.compaction_finish_time 压缩完成时间
+ */
+ String COMPACTION_FINISH_TIME_A = _alias_.getAlias() + "." + COMPACTION_FINISH_TIME_O;
+ /**
+ * 字段 compaction_message 原始值 compaction_message 压缩异常信息
+ */
+ String COMPACTION_MESSAGE_O = "compaction_message";
+ /**
+ * 字段 compaction_message 别名值 tahss.compaction_message 压缩异常信息
+ */
+ String COMPACTION_MESSAGE_A = _alias_.getAlias() + "." + COMPACTION_MESSAGE_O;
+ /**
+ * 字段 compaction_application_id 原始值 compaction_application_id 压缩任务 yarn id
+ */
+ String COMPACTION_APPLICATION_ID_O = "compaction_application_id";
+ /**
+ * 字段 compaction_application_id 别名值 tahss.compaction_application_id 压缩任务 yarn id
+ */
+ String COMPACTION_APPLICATION_ID_A = _alias_.getAlias() + "." + COMPACTION_APPLICATION_ID_O;
+ /**
+ * 字段 compaction_status 原始值 compaction_status 压缩状态
+ */
+ String COMPACTION_STATUS_O = "compaction_status";
+ /**
+ * 字段 compaction_status 别名值 tahss.compaction_status 压缩状态
+ */
+ String COMPACTION_STATUS_A = _alias_.getAlias() + "." + COMPACTION_STATUS_O;
+ /**
+ * 字段 compaction_status_time 原始值 compaction_status_time 压缩事件时间
+ */
+ String COMPACTION_STATUS_TIME_O = "compaction_status_time";
+ /**
+ * 字段 compaction_status_time 别名值 tahss.compaction_status_time 压缩事件时间
+ */
+ String COMPACTION_STATUS_TIME_A = _alias_.getAlias() + "." + COMPACTION_STATUS_TIME_O;
+ /**
+ * 字段 compaction_cluster 原始值 compaction_cluster 运行主机
+ */
+ String COMPACTION_CLUSTER_O = "compaction_cluster";
+ /**
+ * 字段 compaction_cluster 别名值 tahss.compaction_cluster 运行主机
+ */
+ String COMPACTION_CLUSTER_A = _alias_.getAlias() + "." + COMPACTION_CLUSTER_O;
+ /**
+ * 字段 compaction_latest_op_ts 原始值 compaction_latest_op_ts 压缩提供的 latest_op_ts
+ */
+ String COMPACTION_LATEST_OP_TS_O = "compaction_latest_op_ts";
+ /**
+ * 字段 compaction_latest_op_ts 别名值 tahss.compaction_latest_op_ts 压缩提供的 latest_op_ts
+ */
+ String COMPACTION_LATEST_OP_TS_A = _alias_.getAlias() + "." + COMPACTION_LATEST_OP_TS_O;
+ /**
+ * 字段 update_batch_marker 原始值 update_batch_marker 更新批次号
+ */
+ String UPDATE_BATCH_MARKER_O = "update_batch_marker";
+ /**
+ * 字段 update_batch_marker 别名值 tahss.update_batch_marker 更新批次号
+ */
+ String UPDATE_BATCH_MARKER_A = _alias_.getAlias() + "." + UPDATE_BATCH_MARKER_O;
+ /**
+ * 字段 pulsar_back_log 原始值 pulsar_back_log pulsar 积压
+ */
+ String PULSAR_BACK_LOG_O = "pulsar_back_log";
+ /**
+ * 字段 pulsar_back_log 别名值 tahss.pulsar_back_log pulsar 积压
+ */
+ String PULSAR_BACK_LOG_A = _alias_.getAlias() + "." + PULSAR_BACK_LOG_O;
+ /**
+ * 字段 update_time 原始值 update_time 更新时间
+ */
+ String UPDATE_TIME_O = "update_time";
+ /**
+ * 字段 update_time 别名值 tahss.update_time 更新时间
+ */
+ String UPDATE_TIME_A = _alias_.getAlias() + "." + UPDATE_TIME_O;
+ /**
+ * 字段 check_source_total 原始值 check_source_total
+ */
+ String CHECK_SOURCE_TOTAL_O = "check_source_total";
+ /**
+ * 字段 check_source_total 别名值 tahss.check_source_total
+ */
+ String CHECK_SOURCE_TOTAL_A = _alias_.getAlias() + "." + CHECK_SOURCE_TOTAL_O;
+ /**
+ * 字段 check_source_total_time 原始值 check_source_total_time
+ */
+ String CHECK_SOURCE_TOTAL_TIME_O = "check_source_total_time";
+ /**
+ * 字段 check_source_total_time 别名值 tahss.check_source_total_time
+ */
+ String CHECK_SOURCE_TOTAL_TIME_A = _alias_.getAlias() + "." + CHECK_SOURCE_TOTAL_TIME_O;
+ /**
+ * 字段 check_target_total 原始值 check_target_total
+ */
+ String CHECK_TARGET_TOTAL_O = "check_target_total";
+ /**
+ * 字段 check_target_total 别名值 tahss.check_target_total
+ */
+ String CHECK_TARGET_TOTAL_A = _alias_.getAlias() + "." + CHECK_TARGET_TOTAL_O;
+ /**
+ * 字段 check_target_latest_op_ts 原始值 check_target_latest_op_ts
+ */
+ String CHECK_TARGET_LATEST_OP_TS_O = "check_target_latest_op_ts";
+ /**
+ * 字段 check_target_latest_op_ts 别名值 tahss.check_target_latest_op_ts
+ */
+ String CHECK_TARGET_LATEST_OP_TS_A = _alias_.getAlias() + "." + CHECK_TARGET_LATEST_OP_TS_O;
+ /**
+ * 字段 check_target_total_time 原始值 check_target_total_time
+ */
+ String CHECK_TARGET_TOTAL_TIME_O = "check_target_total_time";
+ /**
+ * 字段 check_target_total_time 别名值 tahss.check_target_total_time
+ */
+ String CHECK_TARGET_TOTAL_TIME_A = _alias_.getAlias() + "." + CHECK_TARGET_TOTAL_TIME_O;
+ }
+
+ /**
+ * tb_app_yarn_job_config_sync Yarn 任务配置
+ */
+ interface TbAppYarnJobConfigSync {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_yarn_job_config";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_yarn_job_config", "tayjc_sync");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tayjc_sync.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 job_manager_memory 原始值 job_manager_memory Job Manager 内存（MB）
+ */
+ String JOB_MANAGER_MEMORY_O = "job_manager_memory";
+ /**
+ * 字段 job_manager_memory 别名值 tayjc_sync.job_manager_memory Job Manager 内存（MB）
+ */
+ String JOB_MANAGER_MEMORY_A = _alias_.getAlias() + "." + JOB_MANAGER_MEMORY_O;
+ /**
+ * 字段 task_manager_memory 原始值 task_manager_memory Task Manager 内存（MB）
+ */
+ String TASK_MANAGER_MEMORY_O = "task_manager_memory";
+ /**
+ * 字段 task_manager_memory 别名值 tayjc_sync.task_manager_memory Task Manager 内存（MB）
+ */
+ String TASK_MANAGER_MEMORY_A = _alias_.getAlias() + "." + TASK_MANAGER_MEMORY_O;
+ /**
+ * 字段 status 原始值 status 逻辑删除标识
+ */
+ String STATUS_O = "status";
+ /**
+ * 字段 status 别名值 tayjc_sync.status 逻辑删除标识
+ */
+ String STATUS_A = _alias_.getAlias() + "." + STATUS_O;
+ }
+
+ /**
+ * tb_app_yarn_job_config_compaction Yarn 任务配置
+ */
+ interface TbAppYarnJobConfigCompaction {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_yarn_job_config";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_yarn_job_config", "tayjc_compaction");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tayjc_compaction.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 job_manager_memory 原始值 job_manager_memory Job Manager 内存（MB）
+ */
+ String JOB_MANAGER_MEMORY_O = "job_manager_memory";
+ /**
+ * 字段 job_manager_memory 别名值 tayjc_compaction.job_manager_memory Job Manager 内存（MB）
+ */
+ String JOB_MANAGER_MEMORY_A = _alias_.getAlias() + "." + JOB_MANAGER_MEMORY_O;
+ /**
+ * 字段 task_manager_memory 原始值 task_manager_memory Task Manager 内存（MB）
+ */
+ String TASK_MANAGER_MEMORY_O = "task_manager_memory";
+ /**
+ * 字段 task_manager_memory 别名值 tayjc_compaction.task_manager_memory Task Manager 内存（MB）
+ */
+ String TASK_MANAGER_MEMORY_A = _alias_.getAlias() + "." + TASK_MANAGER_MEMORY_O;
+ /**
+ * 字段 status 原始值 status 逻辑删除标识
+ */
+ String STATUS_O = "status";
+ /**
+ * 字段 status 别名值 tayjc_compaction.status 逻辑删除标识
+ */
+ String STATUS_A = _alias_.getAlias() + "." + STATUS_O;
+ }
+
+ /**
+ * tb_app_yarn_job_config Yarn 任务配置
+ */
+ interface TbAppYarnJobConfig {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".tb_app_yarn_job_config";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".tb_app_yarn_job_config", "tayjc");
+ /**
+ * 字段 id 原始值 id 唯一标识
+ */
+ String ID_O = "id";
+ /**
+ * 字段 id 别名值 tayjc.id 唯一标识
+ */
+ String ID_A = _alias_.getAlias() + "." + ID_O;
+ /**
+ * 字段 job_manager_memory 原始值 job_manager_memory Job Manager 内存（MB）
+ */
+ String JOB_MANAGER_MEMORY_O = "job_manager_memory";
+ /**
+ * 字段 job_manager_memory 别名值 tayjc.job_manager_memory Job Manager 内存（MB）
+ */
+ String JOB_MANAGER_MEMORY_A = _alias_.getAlias() + "." + JOB_MANAGER_MEMORY_O;
+ /**
+ * 字段 task_manager_memory 原始值 task_manager_memory Task Manager 内存（MB）
+ */
+ String TASK_MANAGER_MEMORY_O = "task_manager_memory";
+ /**
+ * 字段 task_manager_memory 别名值 tayjc.task_manager_memory Task Manager 内存（MB）
+ */
+ String TASK_MANAGER_MEMORY_A = _alias_.getAlias() + "." + TASK_MANAGER_MEMORY_O;
+ /**
+ * 字段 status 原始值 status 逻辑删除标识
+ */
+ String STATUS_O = "status";
+ /**
+ * 字段 status 别名值 tayjc.status 逻辑删除标识
+ */
+ String STATUS_A = _alias_.getAlias() + "." + STATUS_O;
+ }
+ }
+
+ /**
+ * iap-datahub
+ */
+ interface IapDatahub {
+ /**
+ * Schema 名称
+ */
+ String _name_ = "`iap-datahub`";
+
+ /**
+ * data_source 数据源定义，定义数据源名称、类型、连接信息、扩展属性等
+ */
+ interface DataSource {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".data_source";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".data_source", "ds");
+ /**
+ * 字段 ds_id 原始值 ds_id 数据源标识，主键自增
+ */
+ String DS_ID_O = "ds_id";
+ /**
+ * 字段 ds_id 别名值 ds.ds_id 数据源标识，主键自增
+ */
+ String DS_ID_A = _alias_.getAlias() + "." + DS_ID_O;
+ /**
+ * 字段 tenant_id 原始值 tenant_id 租户标识,外键关联租户表
+ */
+ String TENANT_ID_O = "tenant_id";
+ /**
+ * 字段 tenant_id 别名值 ds.tenant_id 租户标识,外键关联租户表
+ */
+ String TENANT_ID_A = _alias_.getAlias() + "." + TENANT_ID_O;
+ /**
+ * 字段 project_id 原始值 project_id 项目标识,外键关联项目表
+ */
+ String PROJECT_ID_O = "project_id";
+ /**
+ * 字段 project_id 别名值 ds.project_id 项目标识,外键关联项目表
+ */
+ String PROJECT_ID_A = _alias_.getAlias() + "." + PROJECT_ID_O;
+ /**
+ * 字段 group_id 原始值 group_id 分组标识,主键自增
+ */
+ String GROUP_ID_O = "group_id";
+ /**
+ * 字段 group_id 别名值 ds.group_id 分组标识,主键自增
+ */
+ String GROUP_ID_A = _alias_.getAlias() + "." + GROUP_ID_O;
+ /**
+ * 字段 scenario_id 原始值 scenario_id 场景标识，0默认 1UDAL实时同步 2oracle实时同步 3teledb实时同步 4telepg实时同步 5udal离线同步 6oracle离线同步 7teledb离线同步 8telepg离线同步 9ftp采集 10kafka采集 11pulsar采集 12api采集
+ */
+ String SCENARIO_ID_O = "scenario_id";
+ /**
+ * 字段 scenario_id 别名值 ds.scenario_id 场景标识，0默认 1UDAL实时同步 2oracle实时同步 3teledb实时同步 4telepg实时同步 5udal离线同步 6oracle离线同步 7teledb离线同步 8telepg离线同步 9ftp采集 10kafka采集 11pulsar采集 12api采集
+ */
+ String SCENARIO_ID_A = _alias_.getAlias() + "." + SCENARIO_ID_O;
+ /**
+ * 字段 ds_name 原始值 ds_name 数据源名称，项目内唯一
+ */
+ String DS_NAME_O = "ds_name";
+ /**
+ * 字段 ds_name 别名值 ds.ds_name 数据源名称，项目内唯一
+ */
+ String DS_NAME_A = _alias_.getAlias() + "." + DS_NAME_O;
+ /**
+ * 字段 ds_name_cn 原始值 ds_name_cn 数据源中文名称
+ */
+ String DS_NAME_CN_O = "ds_name_cn";
+ /**
+ * 字段 ds_name_cn 别名值 ds.ds_name_cn 数据源中文名称
+ */
+ String DS_NAME_CN_A = _alias_.getAlias() + "." + DS_NAME_CN_O;
+ /**
+ * 字段 ds_role 原始值 ds_role 数据源角色，类型为src、tgt
+ */
+ String DS_ROLE_O = "ds_role";
+ /**
+ * 字段 ds_role 别名值 ds.ds_role 数据源角色，类型为src、tgt
+ */
+ String DS_ROLE_A = _alias_.getAlias() + "." + DS_ROLE_O;
+ /**
+ * 字段 ds_type 原始值 ds_type 数据源类型，类型主要有mysql、oracle、pg、gp、udal、teledb、hdfs、hive、ftp，具体参考系统字典类型表ds_type
+ */
+ String DS_TYPE_O = "ds_type";
+ /**
+ * 字段 ds_type 别名值 ds.ds_type 数据源类型，类型主要有mysql、oracle、pg、gp、udal、teledb、hdfs、hive、ftp，具体参考系统字典类型表ds_type
+ */
+ String DS_TYPE_A = _alias_.getAlias() + "." + DS_TYPE_O;
+ /**
+ * 字段 schema_name 原始值 schema_name 数据库名，针对数据库类数据源，其他数据源可不填
+ */
+ String SCHEMA_NAME_O = "schema_name";
+ /**
+ * 字段 schema_name 别名值 ds.schema_name 数据库名，针对数据库类数据源，其他数据源可不填
+ */
+ String SCHEMA_NAME_A = _alias_.getAlias() + "." + SCHEMA_NAME_O;
+ /**
+ * 字段 ds_state 原始值 ds_state y正常状态，n冻结状态
+ */
+ String DS_STATE_O = "ds_state";
+ /**
+ * 字段 ds_state 别名值 ds.ds_state y正常状态，n冻结状态
+ */
+ String DS_STATE_A = _alias_.getAlias() + "." + DS_STATE_O;
+ /**
+ * 字段 connection 原始值 connection 数据源信息，标准JSON格式
+ */
+ String CONNECTION_O = "connection";
+ /**
+ * 字段 connection 别名值 ds.connection 数据源信息，标准JSON格式
+ */
+ String CONNECTION_A = _alias_.getAlias() + "." + CONNECTION_O;
+ /**
+ * 字段 ext_settings 原始值 ext_settings 数据源扩展设置，标准JSON格式
+ */
+ String EXT_SETTINGS_O = "ext_settings";
+ /**
+ * 字段 ext_settings 别名值 ds.ext_settings 数据源扩展设置，标准JSON格式
+ */
+ String EXT_SETTINGS_A = _alias_.getAlias() + "." + EXT_SETTINGS_O;
+ /**
+ * 字段 remark 原始值 remark 备注信息
+ */
+ String REMARK_O = "remark";
+ /**
+ * 字段 remark 别名值 ds.remark 备注信息
+ */
+ String REMARK_A = _alias_.getAlias() + "." + REMARK_O;
+ /**
+ * 字段 creator_id 原始值 creator_id 创建人标识，外键关联用户表
+ */
+ String CREATOR_ID_O = "creator_id";
+ /**
+ * 字段 creator_id 别名值 ds.creator_id 创建人标识，外键关联用户表
+ */
+ String CREATOR_ID_A = _alias_.getAlias() + "." + CREATOR_ID_O;
+ /**
+ * 字段 create_time 原始值 create_time 创建时间
+ */
+ String CREATE_TIME_O = "create_time";
+ /**
+ * 字段 create_time 别名值 ds.create_time 创建时间
+ */
+ String CREATE_TIME_A = _alias_.getAlias() + "." + CREATE_TIME_O;
+ /**
+ * 字段 modifier_id 原始值 modifier_id 修改人标识，外键关联用户表
+ */
+ String MODIFIER_ID_O = "modifier_id";
+ /**
+ * 字段 modifier_id 别名值 ds.modifier_id 修改人标识，外键关联用户表
+ */
+ String MODIFIER_ID_A = _alias_.getAlias() + "." + MODIFIER_ID_O;
+ /**
+ * 字段 modify_time 原始值 modify_time 修改时间
+ */
+ String MODIFY_TIME_O = "modify_time";
+ /**
+ * 字段 modify_time 别名值 ds.modify_time 修改时间
+ */
+ String MODIFY_TIME_A = _alias_.getAlias() + "." + MODIFY_TIME_O;
+ /**
+ * 字段 record_state 原始值 record_state 记录状态，y有效数据 n已删除数据
+ */
+ String RECORD_STATE_O = "record_state";
+ /**
+ * 字段 record_state 别名值 ds.record_state 记录状态，y有效数据 n已删除数据
+ */
+ String RECORD_STATE_A = _alias_.getAlias() + "." + RECORD_STATE_O;
+ }
+
+ /**
+ * data_source_table 数据表信息
+ */
+ interface DataSourceTable {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".data_source_table";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".data_source_table", "dst");
+ /**
+ * 字段 table_id 原始值 table_id 主键自增，数据表标识
+ */
+ String TABLE_ID_O = "table_id";
+ /**
+ * 字段 table_id 别名值 dst.table_id 主键自增，数据表标识
+ */
+ String TABLE_ID_A = _alias_.getAlias() + "." + TABLE_ID_O;
+ /**
+ * 字段 tenant_id 原始值 tenant_id 租户标识,外键关联租户表
+ */
+ String TENANT_ID_O = "tenant_id";
+ /**
+ * 字段 tenant_id 别名值 dst.tenant_id 租户标识,外键关联租户表
+ */
+ String TENANT_ID_A = _alias_.getAlias() + "." + TENANT_ID_O;
+ /**
+ * 字段 project_id 原始值 project_id 项目标识,外键关联项目表
+ */
+ String PROJECT_ID_O = "project_id";
+ /**
+ * 字段 project_id 别名值 dst.project_id 项目标识,外键关联项目表
+ */
+ String PROJECT_ID_A = _alias_.getAlias() + "." + PROJECT_ID_O;
+ /**
+ * 字段 ds_id 原始值 ds_id 数据源标识，外键关联数据源定义
+ */
+ String DS_ID_O = "ds_id";
+ /**
+ * 字段 ds_id 别名值 dst.ds_id 数据源标识，外键关联数据源定义
+ */
+ String DS_ID_A = _alias_.getAlias() + "." + DS_ID_O;
+ /**
+ * 字段 table_name 原始值 table_name 数据表名称
+ */
+ String TABLE_NAME_O = "table_name";
+ /**
+ * 字段 table_name 别名值 dst.table_name 数据表名称
+ */
+ String TABLE_NAME_A = _alias_.getAlias() + "." + TABLE_NAME_O;
+ /**
+ * 字段 table_type 原始值 table_type 数据源表类型，不同数据源，表类型不一样
+ */
+ String TABLE_TYPE_O = "table_type";
+ /**
+ * 字段 table_type 别名值 dst.table_type 数据源表类型，不同数据源，表类型不一样
+ */
+ String TABLE_TYPE_A = _alias_.getAlias() + "." + TABLE_TYPE_O;
+ /**
+ * 字段 primary_key 原始值 primary_key 主键字段，多字段按照逗号分隔
+ */
+ String PRIMARY_KEY_O = "primary_key";
+ /**
+ * 字段 primary_key 别名值 dst.primary_key 主键字段，多字段按照逗号分隔
+ */
+ String PRIMARY_KEY_A = _alias_.getAlias() + "." + PRIMARY_KEY_O;
+ /**
+ * 字段 table_sql 原始值 table_sql 建表语句
+ */
+ String TABLE_SQL_O = "table_sql";
+ /**
+ * 字段 table_sql 别名值 dst.table_sql 建表语句
+ */
+ String TABLE_SQL_A = _alias_.getAlias() + "." + TABLE_SQL_O;
+ /**
+ * 字段 table_comment 原始值 table_comment 表备注信息
+ */
+ String TABLE_COMMENT_O = "table_comment";
+ /**
+ * 字段 table_comment 别名值 dst.table_comment 表备注信息
+ */
+ String TABLE_COMMENT_A = _alias_.getAlias() + "." + TABLE_COMMENT_O;
+ /**
+ * 字段 ext_settings 原始值 ext_settings 表扩展属性
+ */
+ String EXT_SETTINGS_O = "ext_settings";
+ /**
+ * 字段 ext_settings 别名值 dst.ext_settings 表扩展属性
+ */
+ String EXT_SETTINGS_A = _alias_.getAlias() + "." + EXT_SETTINGS_O;
+ /**
+ * 字段 remark 原始值 remark 备注信息
+ */
+ String REMARK_O = "remark";
+ /**
+ * 字段 remark 别名值 dst.remark 备注信息
+ */
+ String REMARK_A = _alias_.getAlias() + "." + REMARK_O;
+ /**
+ * 字段 creator_id 原始值 creator_id 创建人标识，外键关联用户表
+ */
+ String CREATOR_ID_O = "creator_id";
+ /**
+ * 字段 creator_id 别名值 dst.creator_id 创建人标识，外键关联用户表
+ */
+ String CREATOR_ID_A = _alias_.getAlias() + "." + CREATOR_ID_O;
+ /**
+ * 字段 create_time 原始值 create_time 创建时间
+ */
+ String CREATE_TIME_O = "create_time";
+ /**
+ * 字段 create_time 别名值 dst.create_time 创建时间
+ */
+ String CREATE_TIME_A = _alias_.getAlias() + "." + CREATE_TIME_O;
+ /**
+ * 字段 modifier_id 原始值 modifier_id 修改人标识，外键关联用户表
+ */
+ String MODIFIER_ID_O = "modifier_id";
+ /**
+ * 字段 modifier_id 别名值 dst.modifier_id 修改人标识，外键关联用户表
+ */
+ String MODIFIER_ID_A = _alias_.getAlias() + "." + MODIFIER_ID_O;
+ /**
+ * 字段 modify_time 原始值 modify_time
+ */
+ String MODIFY_TIME_O = "modify_time";
+ /**
+ * 字段 modify_time 别名值 dst.modify_time
+ */
+ String MODIFY_TIME_A = _alias_.getAlias() + "." + MODIFY_TIME_O;
+ /**
+ * 字段 record_state 原始值 record_state 记录状态，y有效数据 n已删除数据
+ */
+ String RECORD_STATE_O = "record_state";
+ /**
+ * 字段 record_state 别名值 dst.record_state 记录状态，y有效数据 n已删除数据
+ */
+ String RECORD_STATE_A = _alias_.getAlias() + "." + RECORD_STATE_O;
+ }
+
+ /**
+ * data_source_table_field 数据表字段信息，包括字段名称、类型、长度、精度、备注等
+ */
+ interface DataSourceTableField {
+ /**
+ * 表原名
+ */
+ String _origin_ = _name_ + ".data_source_table_field";
+ /**
+ * SqlBuilder 别名对象 {@link Alias}
+ */
+ Alias _alias_ = Alias.of(_name_ + ".data_source_table_field", "dstf");
+ /**
+ * 字段 field_id 原始值 field_id 主键自增，字段标识
+ */
+ String FIELD_ID_O = "field_id";
+ /**
+ * 字段 field_id 别名值 dstf.field_id 主键自增，字段标识
+ */
+ String FIELD_ID_A = _alias_.getAlias() + "." + FIELD_ID_O;
+ /**
+ * 字段 table_id 原始值 table_id 数据源表标识，外键关联数据源表
+ */
+ String TABLE_ID_O = "table_id";
+ /**
+ * 字段 table_id 别名值 dstf.table_id 数据源表标识，外键关联数据源表
+ */
+ String TABLE_ID_A = _alias_.getAlias() + "." + TABLE_ID_O;
+ /**
+ * 字段 tenant_id 原始值 tenant_id 租户标识，外键关联租户表
+ */
+ String TENANT_ID_O = "tenant_id";
+ /**
+ * 字段 tenant_id 别名值 dstf.tenant_id 租户标识，外键关联租户表
+ */
+ String TENANT_ID_A = _alias_.getAlias() + "." + TENANT_ID_O;
+ /**
+ * 字段 project_id 原始值 project_id 项目标识，外键关联项目表
+ */
+ String PROJECT_ID_O = "project_id";
+ /**
+ * 字段 project_id 别名值 dstf.project_id 项目标识，外键关联项目表
+ */
+ String PROJECT_ID_A = _alias_.getAlias() + "." + PROJECT_ID_O;
+ /**
+ * 字段 field_name 原始值 field_name 字段名称，表内唯一
+ */
+ String FIELD_NAME_O = "field_name";
+ /**
+ * 字段 field_name 别名值 dstf.field_name 字段名称，表内唯一
+ */
+ String FIELD_NAME_A = _alias_.getAlias() + "." + FIELD_NAME_O;
+ /**
+ * 字段 field_seq 原始值 field_seq 字段序号
+ */
+ String FIELD_SEQ_O = "field_seq";
+ /**
+ * 字段 field_seq 别名值 dstf.field_seq 字段序号
+ */
+ String FIELD_SEQ_A = _alias_.getAlias() + "." + FIELD_SEQ_O;
+ /**
+ * 字段 field_type 原始值 field_type 字段类型
+ */
+ String FIELD_TYPE_O = "field_type";
+ /**
+ * 字段 field_type 别名值 dstf.field_type 字段类型
+ */
+ String FIELD_TYPE_A = _alias_.getAlias() + "." + FIELD_TYPE_O;
+ /**
+ * 字段 field_comment 原始值 field_comment 字段备注
+ */
+ String FIELD_COMMENT_O = "field_comment";
+ /**
+ * 字段 field_comment 别名值 dstf.field_comment 字段备注
+ */
+ String FIELD_COMMENT_A = _alias_.getAlias() + "." + FIELD_COMMENT_O;
+ /**
+ * 字段 primary_key 原始值 primary_key 是否主键，y是主键，n不是主键
+ */
+ String PRIMARY_KEY_O = "primary_key";
+ /**
+ * 字段 primary_key 别名值 dstf.primary_key 是否主键，y是主键，n不是主键
+ */
+ String PRIMARY_KEY_A = _alias_.getAlias() + "." + PRIMARY_KEY_O;
+ /**
+ * 字段 partition_key 原始值 partition_key 是否分区键，y是分区键，n不是分区键
+ */
+ String PARTITION_KEY_O = "partition_key";
+ /**
+ * 字段 partition_key 别名值 dstf.partition_key 是否分区键，y是分区键，n不是分区键
+ */
+ String PARTITION_KEY_A = _alias_.getAlias() + "." + PARTITION_KEY_O;
+ /**
+ * 字段 length 原始值 length 字段长度
+ */
+ String LENGTH_O = "length";
+ /**
+ * 字段 length 别名值 dstf.length 字段长度
+ */
+ String LENGTH_A = _alias_.getAlias() + "." + LENGTH_O;
+ /**
+ * 字段 scale 原始值 scale 字段精度
+ */
+ String SCALE_O = "scale";
+ /**
+ * 字段 scale 别名值 dstf.scale 字段精度
+ */
+ String SCALE_A = _alias_.getAlias() + "." + SCALE_O;
+ /**
+ * 字段 remark 原始值 remark 备注信息
+ */
+ String REMARK_O = "remark";
+ /**
+ * 字段 remark 别名值 dstf.remark 备注信息
+ */
+ String REMARK_A = _alias_.getAlias() + "." + REMARK_O;
+ /**
+ * 字段 creator_id 原始值 creator_id 创建人标识，外键关联用户表
+ */
+ String CREATOR_ID_O = "creator_id";
+ /**
+ * 字段 creator_id 别名值 dstf.creator_id 创建人标识，外键关联用户表
+ */
+ String CREATOR_ID_A = _alias_.getAlias() + "." + CREATOR_ID_O;
+ /**
+ * 字段 create_time 原始值 create_time 创建时间
+ */
+ String CREATE_TIME_O = "create_time";
+ /**
+ * 字段 create_time 别名值 dstf.create_time 创建时间
+ */
+ String CREATE_TIME_A = _alias_.getAlias() + "." + CREATE_TIME_O;
+ /**
+ * 字段 modifier_id 原始值 modifier_id 修改人标识，外键关联用户表
+ */
+ String MODIFIER_ID_O = "modifier_id";
+ /**
+ * 字段 modifier_id 别名值 dstf.modifier_id 修改人标识，外键关联用户表
+ */
+ String MODIFIER_ID_A = _alias_.getAlias() + "." + MODIFIER_ID_O;
+ /**
+ * 字段 modify_time 原始值 modify_time 修改时间
+ */
+ String MODIFY_TIME_O = "modify_time";
+ /**
+ * 字段 modify_time 别名值 dstf.modify_time 修改时间
+ */
+ String MODIFY_TIME_A = _alias_.getAlias() + "." + MODIFY_TIME_O;
+ /**
+ * 字段 record_state 原始值 record_state 记录状态，y有效数据，n已删除数据
+ */
+ String RECORD_STATE_O = "record_state";
+ /**
+ * 字段 record_state 别名值 dstf.record_state 记录状态，y有效数据，n已删除数据
+ */
+ String RECORD_STATE_A = _alias_.getAlias() + "." + RECORD_STATE_O;
+ }
+ }
+}
\ No newline at end of file
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/FlinkJob.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/FlinkJob.java
new file mode 100644
index 0000000..38174e4
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/FlinkJob.java
@@ -0,0 +1,123 @@
+package com.lanyuanxiaoyao.service.common.entity;
+
+import java.io.Serializable;
+
+/**
+ * Flink Job
+ *
+ * @author ZhangJiacheng
+ * @version 0.0.1
+ * @date 2021-12-08
+ */
+public class FlinkJob implements Serializable {
+ private Long id;
+ private String name;
+ private RunMode runMode;
+ private TableMeta.YarnMeta oneInOneSyncYarn;
+
+ public FlinkJob() {
+ }
+
+ public FlinkJob(Builder builder) {
+ this.id = builder.id;
+ this.name = builder.name;
+ this.runMode = builder.runMode;
+ this.oneInOneSyncYarn = builder.oneInOneSyncYarn;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public RunMode getRunMode() {
+ return runMode;
+ }
+
+ public void setRunMode(RunMode runMode) {
+ this.runMode = runMode;
+ }
+
+ public TableMeta.YarnMeta getOneInOneSyncYarn() {
+ return oneInOneSyncYarn;
+ }
+
+ public void setOneInOneSyncYarn(TableMeta.YarnMeta oneInOneSyncYarn) {
+ this.oneInOneSyncYarn = oneInOneSyncYarn;
+ }
+
+ @Override
+ public String toString() {
+ return "FlinkJob{" +
+ "id=" + id +
+ ", name='" + name + '\'' +
+ ", runMode=" + runMode +
+ ", oneInOneSyncYarn=" + oneInOneSyncYarn +
+ '}';
+ }
+
+ public enum RunMode {
+ /**
+ * 所有表放在同一个任务中
+ */
+ ALL_IN_ONE,
+ /**
+ * 每个表放在单独的任务中
+ */
+ ONE_IN_ONE,
+ /**
+ * 针对 ACCT 小表，将同一个表放在同一个子任务中
+ */
+ ALL_IN_ONE_BY_TABLE,
+ ALL_IN_ONE_BY_SCHEMA,
+ }
+
+ public static final class Builder {
+ private Long id;
+ private String name;
+ private RunMode runMode;
+ private TableMeta.YarnMeta oneInOneSyncYarn;
+
+ private Builder() {}
+
+ public Builder id(Long id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder runMode(RunMode runMode) {
+ this.runMode = runMode;
+ return this;
+ }
+
+ public Builder oneInOneSyncYarn(TableMeta.YarnMeta oneInOneSyncYarn) {
+ this.oneInOneSyncYarn = oneInOneSyncYarn;
+ return this;
+ }
+
+ public FlinkJob build() {
+ return new FlinkJob(this);
+ }
+ }
+
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/Record.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/Record.java
new file mode 100644
index 0000000..c73ea5e
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/Record.java
@@ -0,0 +1,197 @@
+package com.lanyuanxiaoyao.service.common.entity;
+
+import java.io.Serializable;
+import java.util.Map;
+
+/**
+ * 从 Pulsar 读取的消息封装
+ *
+ * @author lanyuanxiaoyao
+ * @version 0.0.1
+ * @date 2021-11-25
+ */
+public class Record implements Serializable {
+ private Source source;
+ private Statement statement;
+
+ public Record() {
+ }
+
+ public Source getSource() {
+ return source;
+ }
+
+ public void setSource(Source source) {
+ this.source = source;
+ }
+
+ public Statement getStatement() {
+ return statement;
+ }
+
+ public void setStatement(Statement statement) {
+ this.statement = statement;
+ }
+
+ @Override
+ public String toString() {
+ return "Record{" +
+ "source=" + source +
+ ", statement=" + statement +
+ '}';
+ }
+
+ public static class Source implements Serializable {
+ private String sourceId;
+ private String sourceType;
+ private String sourcePos;
+ private String currentTs;
+
+ public Source() {
+ }
+
+ public String getSourceId() {
+ return sourceId;
+ }
+
+ public void setSourceId(String sourceId) {
+ this.sourceId = sourceId;
+ }
+
+ public String getSourceType() {
+ return sourceType;
+ }
+
+ public void setSourceType(String sourceType) {
+ this.sourceType = sourceType;
+ }
+
+ public String getSourcePos() {
+ return sourcePos;
+ }
+
+ public void setSourcePos(String sourcePos) {
+ this.sourcePos = sourcePos;
+ }
+
+ public String getCurrentTs() {
+ return currentTs;
+ }
+
+ public void setCurrentTs(String currentTs) {
+ this.currentTs = currentTs;
+ }
+
+ @Override
+ public String toString() {
+ return "Source{" +
+ "sourceId='" + sourceId + '\'' +
+ ", sourceType='" + sourceType + '\'' +
+ ", sourcePos='" + sourcePos + '\'' +
+ ", currentTs='" + currentTs + '\'' +
+ '}';
+ }
+ }
+
+ public static class Statement implements Serializable {
+ private String schema;
+ private String table;
+ private String opStatement;
+ private String opType;
+ private String op;
+ private String opTs;
+ private String version;
+ private Map before;
+ private Map after;
+
+ public Statement() {
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public String getOpStatement() {
+ return opStatement;
+ }
+
+ public void setOpStatement(String opStatement) {
+ this.opStatement = opStatement;
+ }
+
+ public String getOpType() {
+ return opType;
+ }
+
+ public void setOpType(String opType) {
+ this.opType = opType;
+ }
+
+ public String getOp() {
+ return op;
+ }
+
+ public void setOp(String op) {
+ this.op = op;
+ }
+
+ public String getOpTs() {
+ return opTs;
+ }
+
+ public void setOpTs(String opTs) {
+ this.opTs = opTs;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public void setVersion(String version) {
+ this.version = version;
+ }
+
+ public Map getBefore() {
+ return before;
+ }
+
+ public void setBefore(Map before) {
+ this.before = before;
+ }
+
+ public Map getAfter() {
+ return after;
+ }
+
+ public void setAfter(Map after) {
+ this.after = after;
+ }
+
+ @Override
+ public String toString() {
+ return "Statement{" +
+ "schema='" + schema + '\'' +
+ ", table='" + table + '\'' +
+ ", opStatement='" + opStatement + '\'' +
+ ", opType='" + opType + '\'' +
+ ", op='" + op + '\'' +
+ ", opTs='" + opTs + '\'' +
+ ", version='" + version + '\'' +
+ ", before=" + before +
+ ", after=" + after +
+ '}';
+ }
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/RunMeta.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/RunMeta.java
new file mode 100644
index 0000000..273ead9
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/RunMeta.java
@@ -0,0 +1,121 @@
+package com.lanyuanxiaoyao.service.common.entity;
+
+import com.lanyuanxiaoyao.service.common.utils.NameHelper;
+import java.io.Serializable;
+
+/**
+ * 运行时参数
+ *
+ * @author ZhangJiacheng
+ * @date 2023-05-11
+ */
+@SuppressWarnings("FieldMayBeFinal")
+public class RunMeta implements Serializable {
+ private String cluster;
+ private Long flinkJobId;
+ private String alias;
+ private String flinkJobName;
+ private String host;
+ private String applicationId;
+ private String containerId;
+ private String containerPath;
+ private String runType;
+ private String executorVersion;
+ private String jvmPid;
+ private String applicationProxy;
+ private String subscriptionName;
+
+ public RunMeta() {
+ this.flinkJobName = System.getenv("flink_job_name");
+ this.host = System.getenv("NM_HOST");
+ this.applicationId = System.getenv("_APP_ID");
+ this.containerId = System.getenv("CONTAINER_ID");
+ this.containerPath = System.getenv("PWD");
+ this.runType = System.getenv("run_type");
+ this.executorVersion = System.getenv("executor_version");
+ this.jvmPid = System.getenv("JVM_PID");
+ this.applicationProxy = System.getenv("APPLICATION_WEB_PROXY_BASE");
+ }
+
+ public RunMeta(String cluster, Long flinkJobId) {
+ this();
+ this.cluster = cluster;
+ this.flinkJobId = flinkJobId;
+ }
+
+ public RunMeta(String cluster, Long flinkJobId, String alias) {
+ this(cluster, flinkJobId);
+ this.alias = alias;
+ this.subscriptionName = NameHelper.pulsarSubscriptionName(flinkJobId, alias);
+ }
+
+ public String getCluster() {
+ return cluster;
+ }
+
+ public Long getFlinkJobId() {
+ return flinkJobId;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public String getFlinkJobName() {
+ return flinkJobName;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public String getApplicationId() {
+ return applicationId;
+ }
+
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public String getContainerPath() {
+ return containerPath;
+ }
+
+ public String getRunType() {
+ return runType;
+ }
+
+ public String getExecutorVersion() {
+ return executorVersion;
+ }
+
+ public String getJvmPid() {
+ return jvmPid;
+ }
+
+ public String getApplicationProxy() {
+ return applicationProxy;
+ }
+
+ public String getSubscriptionName() {
+ return subscriptionName;
+ }
+
+ @Override
+ public String toString() {
+ return "RunMeta{" +
+ "cluster='" + cluster + '\'' +
+ ", flinkJobId='" + flinkJobId + '\'' +
+ ", flinkJobName='" + flinkJobName + '\'' +
+ ", host='" + host + '\'' +
+ ", applicationId='" + applicationId + '\'' +
+ ", containerId='" + containerId + '\'' +
+ ", containerPath='" + containerPath + '\'' +
+ ", runType='" + runType + '\'' +
+ ", executorVersion='" + executorVersion + '\'' +
+ ", jvmPid='" + jvmPid + '\'' +
+ ", applicationProxy='" + applicationProxy + '\'' +
+ ", subscriptionName='" + subscriptionName + '\'' +
+ '}';
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/SyncState.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/SyncState.java
new file mode 100644
index 0000000..861ec45
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/SyncState.java
@@ -0,0 +1,259 @@
+package com.lanyuanxiaoyao.service.common.entity;
+
+import java.io.Serializable;
+
+/**
+ * 同步压缩状态表类
+ *
+ * @author ZhangJiacheng
+ * @date 2023-04-24
+ */
+public class SyncState implements Serializable {
+ private Long flinkJobId;
+ private String alias;
+ private String messageId;
+ private Long sourceStartTime;
+ private Long sourceCheckpointTime;
+ private Long sourcePublishTime;
+ private Long sourceOperationTime;
+ private Long compactionStartTime;
+ private Long compactionFinishTime;
+ private String compactionApplicationId;
+ private String compactionStatus;
+ private Long compactionStatusTime;
+ private Long compactionLatestOperationTime;
+
+ public SyncState() {
+ }
+
+ public SyncState(Builder builder) {
+ this.flinkJobId = builder.flinkJobId;
+ this.alias = builder.alias;
+ this.messageId = builder.messageId;
+ this.sourceStartTime = builder.sourceStartTime;
+ this.sourceCheckpointTime = builder.sourceCheckpointTime;
+ this.sourcePublishTime = builder.sourcePublishTime;
+ this.sourceOperationTime = builder.sourceOperationTime;
+ this.compactionStartTime = builder.compactionStartTime;
+ this.compactionFinishTime = builder.compactionFinishTime;
+ this.compactionApplicationId = builder.compactionApplicationId;
+ this.compactionStatus = builder.compactionStatus;
+ this.compactionStatusTime = builder.compactionStatusTime;
+ this.compactionLatestOperationTime = builder.compactionLatestOperationTime;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public Long getFlinkJobId() {
+ return flinkJobId;
+ }
+
+ public void setFlinkJobId(Long flinkJobId) {
+ this.flinkJobId = flinkJobId;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public String getMessageId() {
+ return messageId;
+ }
+
+ public void setMessageId(String messageId) {
+ this.messageId = messageId;
+ }
+
+ public Long getSourceStartTime() {
+ return sourceStartTime;
+ }
+
+ public void setSourceStartTime(Long sourceStartTime) {
+ this.sourceStartTime = sourceStartTime;
+ }
+
+ public Long getSourceCheckpointTime() {
+ return sourceCheckpointTime;
+ }
+
+ public void setSourceCheckpointTime(Long sourceCheckpointTime) {
+ this.sourceCheckpointTime = sourceCheckpointTime;
+ }
+
+ public Long getSourcePublishTime() {
+ return sourcePublishTime;
+ }
+
+ public void setSourcePublishTime(Long sourcePublishTime) {
+ this.sourcePublishTime = sourcePublishTime;
+ }
+
+ public Long getSourceOperationTime() {
+ return sourceOperationTime;
+ }
+
+ public void setSourceOperationTime(Long sourceOperationTime) {
+ this.sourceOperationTime = sourceOperationTime;
+ }
+
+ public Long getCompactionStartTime() {
+ return compactionStartTime;
+ }
+
+ public void setCompactionStartTime(Long compactionStartTime) {
+ this.compactionStartTime = compactionStartTime;
+ }
+
+ public Long getCompactionFinishTime() {
+ return compactionFinishTime;
+ }
+
+ public void setCompactionFinishTime(Long compactionFinishTime) {
+ this.compactionFinishTime = compactionFinishTime;
+ }
+
+ public String getCompactionApplicationId() {
+ return compactionApplicationId;
+ }
+
+ public void setCompactionApplicationId(String compactionApplicationId) {
+ this.compactionApplicationId = compactionApplicationId;
+ }
+
+ public String getCompactionStatus() {
+ return compactionStatus;
+ }
+
+ public void setCompactionStatus(String compactionStatus) {
+ this.compactionStatus = compactionStatus;
+ }
+
+ public Long getCompactionStatusTime() {
+ return compactionStatusTime;
+ }
+
+ public void setCompactionStatusTime(Long compactionStatusTime) {
+ this.compactionStatusTime = compactionStatusTime;
+ }
+
+ public Long getCompactionLatestOperationTime() {
+ return compactionLatestOperationTime;
+ }
+
+ public void setCompactionLatestOperationTime(Long compactionLatestOperationTime) {
+ this.compactionLatestOperationTime = compactionLatestOperationTime;
+ }
+
+ @Override
+ public String toString() {
+ return "SyncState{" +
+ "flinkJobId=" + flinkJobId +
+ ", alias='" + alias + '\'' +
+ ", messageId='" + messageId + '\'' +
+ ", sourceStartTime=" + sourceStartTime +
+ ", sourceCheckpointTime=" + sourceCheckpointTime +
+ ", sourcePublishTime=" + sourcePublishTime +
+ ", sourceOperationTime=" + sourceOperationTime +
+ ", compactionStartTime=" + compactionStartTime +
+ ", compactionFinishTime=" + compactionFinishTime +
+ ", compactionApplicationId='" + compactionApplicationId + '\'' +
+ ", compactionStatus='" + compactionStatus + '\'' +
+ ", compactionStatusTime=" + compactionStatusTime +
+ ", compactionLatestOperationTime=" + compactionLatestOperationTime +
+ '}';
+ }
+
+ public static final class Builder {
+ private Long flinkJobId;
+ private String alias;
+ private String messageId;
+ private Long sourceStartTime;
+ private Long sourceCheckpointTime;
+ private Long sourcePublishTime;
+ private Long sourceOperationTime;
+ private Long compactionStartTime;
+ private Long compactionFinishTime;
+ private String compactionApplicationId;
+ private String compactionStatus;
+ private Long compactionStatusTime;
+ private Long compactionLatestOperationTime;
+
+ private Builder() {
+ }
+
+ public Builder flinkJobId(Long flinkJobId) {
+ this.flinkJobId = flinkJobId;
+ return this;
+ }
+
+ public Builder alias(String alias) {
+ this.alias = alias;
+ return this;
+ }
+
+ public Builder messageId(String messageId) {
+ this.messageId = messageId;
+ return this;
+ }
+
+ public Builder sourceStartTime(Long sourceStartTime) {
+ this.sourceStartTime = sourceStartTime;
+ return this;
+ }
+
+ public Builder sourceCheckpointTime(Long sourceCheckpointTime) {
+ this.sourceCheckpointTime = sourceCheckpointTime;
+ return this;
+ }
+
+ public Builder sourcePublishTime(Long sourcePublishTime) {
+ this.sourcePublishTime = sourcePublishTime;
+ return this;
+ }
+
+ public Builder sourceOperationTime(Long sourceOperationTime) {
+ this.sourceOperationTime = sourceOperationTime;
+ return this;
+ }
+
+ public Builder compactionStartTime(Long compactionStartTime) {
+ this.compactionStartTime = compactionStartTime;
+ return this;
+ }
+
+ public Builder compactionFinishTime(Long compactionFinishTime) {
+ this.compactionFinishTime = compactionFinishTime;
+ return this;
+ }
+
+ public Builder compactionApplicationId(String compactionApplicationId) {
+ this.compactionApplicationId = compactionApplicationId;
+ return this;
+ }
+
+ public Builder compactionStatus(String compactionStatus) {
+ this.compactionStatus = compactionStatus;
+ return this;
+ }
+
+ public Builder compactionStatusTime(Long compactionStatusTime) {
+ this.compactionStatusTime = compactionStatusTime;
+ return this;
+ }
+
+ public Builder compactionLatestOperationTime(Long compactionLatestOperationTime) {
+ this.compactionLatestOperationTime = compactionLatestOperationTime;
+ return this;
+ }
+
+ public SyncState build() {
+ return new SyncState(this);
+ }
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/TableMeta.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/TableMeta.java
new file mode 100644
index 0000000..a17a801
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/TableMeta.java
@@ -0,0 +1,2390 @@
+package com.lanyuanxiaoyao.service.common.entity;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * 配置表项
+ *
+ * @author ZhangJiacheng
+ * @version 0.0.1
+ * @date 2021-11-30
+ */
+public class TableMeta implements Serializable {
+ private String alias;
+ private String source;
+ private String schema;
+ private String table;
+ private String type;
+ private List primaryKeys;
+ private List partitionKeys;
+ private HudiMeta hudi;
+ private List fields;
+ private String filterField;
+ private List filterValues;
+ private FilterType filterType;
+ private String topic;
+ private String pulsarAddress;
+ private YarnMeta syncYarn;
+ private YarnMeta compactionYarn;
+ private String partitionField;
+ private ConfigMeta config;
+ private JobMeta job;
+ private ConnectionMeta connection;
+ private Integer priority;
+ private SourceType sourceType;
+ private List tags;
+ private Integer version;
+
+ public TableMeta() {
+ }
+
+ public TableMeta(Builder builder) {
+ this.alias = builder.alias;
+ this.source = builder.source;
+ this.schema = builder.schema;
+ this.table = builder.table;
+ this.type = builder.type;
+ this.primaryKeys = builder.primaryKeys;
+ this.partitionKeys = builder.partitionKeys;
+ this.hudi = builder.hudi;
+ this.fields = builder.fields;
+ this.filterField = builder.filterField;
+ this.filterValues = builder.filterValues;
+ this.filterType = builder.filterType;
+ this.topic = builder.topic;
+ this.pulsarAddress = builder.pulsarAddress;
+ this.syncYarn = builder.syncYarn;
+ this.compactionYarn = builder.compactionYarn;
+ this.partitionField = builder.partitionField;
+ this.config = builder.config;
+ this.job = builder.job;
+ this.connection = builder.connection;
+ this.priority = builder.priority;
+ this.sourceType = builder.sourceType;
+ this.tags = builder.tags;
+ this.version = builder.version;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source;
+ }
+
+ public String getSchema() {
+ return schema;
+ }
+
+ public void setSchema(String schema) {
+ this.schema = schema;
+ }
+
+ public String getTable() {
+ return table;
+ }
+
+ public void setTable(String table) {
+ this.table = table;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public List getPrimaryKeys() {
+ return primaryKeys;
+ }
+
+ public void setPrimaryKeys(List primaryKeys) {
+ this.primaryKeys = primaryKeys;
+ }
+
+ public List getPartitionKeys() {
+ return partitionKeys;
+ }
+
+ public void setPartitionKeys(List partitionKeys) {
+ this.partitionKeys = partitionKeys;
+ }
+
+ public HudiMeta getHudi() {
+ return hudi;
+ }
+
+ public void setHudi(HudiMeta hudi) {
+ this.hudi = hudi;
+ }
+
+ public List getFields() {
+ return fields;
+ }
+
+ public void setFields(List fields) {
+ this.fields = fields;
+ }
+
+ public String getFilterField() {
+ return filterField;
+ }
+
+ public void setFilterField(String filterField) {
+ this.filterField = filterField;
+ }
+
+ public List getFilterValues() {
+ return filterValues;
+ }
+
+ public void setFilterValues(List filterValues) {
+ this.filterValues = filterValues;
+ }
+
+ public FilterType getFilterType() {
+ return filterType;
+ }
+
+ public void setFilterType(FilterType filterType) {
+ this.filterType = filterType;
+ }
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public String getPulsarAddress() {
+ return pulsarAddress;
+ }
+
+ public void setPulsarAddress(String pulsarAddress) {
+ this.pulsarAddress = pulsarAddress;
+ }
+
+ public YarnMeta getSyncYarn() {
+ return syncYarn;
+ }
+
+ public void setSyncYarn(YarnMeta syncYarn) {
+ this.syncYarn = syncYarn;
+ }
+
+ public YarnMeta getCompactionYarn() {
+ return compactionYarn;
+ }
+
+ public void setCompactionYarn(YarnMeta compactionYarn) {
+ this.compactionYarn = compactionYarn;
+ }
+
+ public String getPartitionField() {
+ return partitionField;
+ }
+
+ public void setPartitionField(String partitionField) {
+ this.partitionField = partitionField;
+ }
+
+ public ConfigMeta getConfig() {
+ return config;
+ }
+
+ public void setConfig(ConfigMeta config) {
+ this.config = config;
+ }
+
+ public JobMeta getJob() {
+ return job;
+ }
+
+ public void setJob(JobMeta job) {
+ this.job = job;
+ }
+
+ public ConnectionMeta getConnection() {
+ return connection;
+ }
+
+ public void setConnection(ConnectionMeta connection) {
+ this.connection = connection;
+ }
+
+ public Integer getPriority() {
+ return priority;
+ }
+
+ public void setPriority(Integer priority) {
+ this.priority = priority;
+ }
+
+ public SourceType getSourceType() {
+ return sourceType;
+ }
+
+ public void setSourceType(SourceType sourceType) {
+ this.sourceType = sourceType;
+ }
+
+ public List getTags() {
+ return tags;
+ }
+
+ public void setTags(List tags) {
+ this.tags = tags;
+ }
+
+ public Integer getVersion() {
+ return version;
+ }
+
+ public void setVersion(Integer version) {
+ this.version = version;
+ }
+
+ @Override
+ public String toString() {
+ return "TableMeta{" +
+ "alias='" + alias + '\'' +
+ ", source='" + source + '\'' +
+ ", schema='" + schema + '\'' +
+ ", table='" + table + '\'' +
+ ", type='" + type + '\'' +
+ ", primaryKeys=" + primaryKeys +
+ ", partitionKeys=" + partitionKeys +
+ ", hudi=" + hudi +
+ ", fields=" + fields +
+ ", filterField='" + filterField + '\'' +
+ ", filterValues=" + filterValues +
+ ", filterType=" + filterType +
+ ", topic='" + topic + '\'' +
+ ", pulsarAddress='" + pulsarAddress + '\'' +
+ ", syncYarn=" + syncYarn +
+ ", compactionYarn=" + compactionYarn +
+ ", partitionField='" + partitionField + '\'' +
+ ", config=" + config +
+ ", job=" + job +
+ ", connection=" + connection +
+ ", priority=" + priority +
+ ", sourceType=" + sourceType +
+ ", tags=" + tags +
+ ", version=" + version +
+ '}';
+ }
+
+ /**
+ * 过滤操作的类型
+ */
+ public enum FilterType {
+ /**
+ * 包含
+ */
+ INCLUDE,
+ /**
+ * 排除
+ */
+ EXCLUDE,
+ /**
+ * 无操作
+ */
+ NONE,
+ }
+
+ public enum SourceType {
+ /**
+ * UDAL
+ */
+ UDAL,
+ /**
+ * TELEPG
+ */
+ TELEPG,
+ }
+
+ public static final class Builder {
+ private String alias;
+ private String source;
+ private String schema;
+ private String table;
+ private String type;
+ private List primaryKeys;
+ private List partitionKeys;
+ private HudiMeta hudi;
+ private List fields;
+ private String filterField;
+ private List filterValues;
+ private FilterType filterType;
+ private String topic;
+ private String pulsarAddress;
+ private YarnMeta syncYarn;
+ private YarnMeta compactionYarn;
+ private String partitionField;
+ private ConfigMeta config;
+ private JobMeta job;
+ private ConnectionMeta connection;
+ private Integer priority;
+ private SourceType sourceType;
+ private List tags;
+ private Integer version;
+
+ private Builder() {
+ }
+
+ public Builder alias(String alias) {
+ this.alias = alias;
+ return this;
+ }
+
+ public Builder source(String source) {
+ this.source = source;
+ return this;
+ }
+
+ public Builder schema(String schema) {
+ this.schema = schema;
+ return this;
+ }
+
+ public Builder table(String table) {
+ this.table = table;
+ return this;
+ }
+
+ public Builder type(String type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder primaryKeys(List primaryKeys) {
+ this.primaryKeys = primaryKeys;
+ return this;
+ }
+
+ public Builder partitionKeys(List partitionKeys) {
+ this.partitionKeys = partitionKeys;
+ return this;
+ }
+
+ public Builder hudi(HudiMeta hudi) {
+ this.hudi = hudi;
+ return this;
+ }
+
+ public Builder fields(List fields) {
+ this.fields = fields;
+ return this;
+ }
+
+ public Builder filterField(String filterField) {
+ this.filterField = filterField;
+ return this;
+ }
+
+ public Builder filterValues(List filterValues) {
+ this.filterValues = filterValues;
+ return this;
+ }
+
+ public Builder filterType(FilterType filterType) {
+ this.filterType = filterType;
+ return this;
+ }
+
+ public Builder topic(String topic) {
+ this.topic = topic;
+ return this;
+ }
+
+ public Builder pulsarAddress(String pulsarAddress) {
+ this.pulsarAddress = pulsarAddress;
+ return this;
+ }
+
+ public Builder syncYarn(YarnMeta syncYarn) {
+ this.syncYarn = syncYarn;
+ return this;
+ }
+
+ public Builder compactionYarn(YarnMeta compactionYarn) {
+ this.compactionYarn = compactionYarn;
+ return this;
+ }
+
+ public Builder partitionField(String partitionField) {
+ this.partitionField = partitionField;
+ return this;
+ }
+
+ public Builder config(ConfigMeta config) {
+ this.config = config;
+ return this;
+ }
+
+ public Builder job(JobMeta job) {
+ this.job = job;
+ return this;
+ }
+
+ public Builder connection(ConnectionMeta connection) {
+ this.connection = connection;
+ return this;
+ }
+
+ public Builder priority(Integer priority) {
+ this.priority = priority;
+ return this;
+ }
+
+
+ public Builder sourceType(SourceType sourceType) {
+ this.sourceType = sourceType;
+ return this;
+ }
+
+ public Builder tags(List tags) {
+ this.tags = tags;
+ return this;
+ }
+
+ public Builder version(Integer version) {
+ this.version = version;
+ return this;
+ }
+
+ public TableMeta build() {
+ return new TableMeta(this);
+ }
+ }
+
+ public static final class FieldMeta implements Serializable {
+ private String name;
+ private Integer sequence;
+ private String type;
+ private Boolean isPrimaryKey;
+ private Boolean partitionKey;
+ private Long length;
+ private Integer scala;
+
+ public FieldMeta() {
+ }
+
+ public FieldMeta(Builder builder) {
+ this.name = builder.name;
+ this.sequence = builder.sequence;
+ this.type = builder.type;
+ this.isPrimaryKey = builder.isPrimaryKey;
+ this.partitionKey = builder.partitionKey;
+ this.length = builder.length;
+ this.scala = builder.scala;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public String toString() {
+ return "FieldMeta{" +
+ "name='" + name + '\'' +
+ ", sequence=" + sequence +
+ ", type='" + type + '\'' +
+ ", isPrimaryKey=" + isPrimaryKey +
+ ", partitionKey=" + partitionKey +
+ ", length=" + length +
+ '}';
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public Integer getSequence() {
+ return sequence;
+ }
+
+ public void setSequence(Integer sequence) {
+ this.sequence = sequence;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public Boolean getPrimaryKey() {
+ return isPrimaryKey;
+ }
+
+ public void setPrimaryKey(Boolean primaryKey) {
+ isPrimaryKey = primaryKey;
+ }
+
+ public Boolean getPartitionKey() {
+ return partitionKey;
+ }
+
+ public void setPartitionKey(Boolean partitionKey) {
+ this.partitionKey = partitionKey;
+ }
+
+ public Long getLength() {
+ return length;
+ }
+
+ public void setLength(Long length) {
+ this.length = length;
+ }
+
+ public Integer getScala() {
+ return scala;
+ }
+
+ public void setScala(Integer scala) {
+ this.scala = scala;
+ }
+
+ public static final class Builder {
+ private String name;
+ private Integer sequence;
+ private String type;
+ private Boolean isPrimaryKey;
+ private Boolean partitionKey;
+ private Long length;
+ private Integer scala;
+
+ private Builder() {
+ }
+
+ public Builder name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder sequence(Integer sequence) {
+ this.sequence = sequence;
+ return this;
+ }
+
+ public Builder type(String type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder isPrimaryKey(Boolean isPrimaryKey) {
+ this.isPrimaryKey = isPrimaryKey;
+ return this;
+ }
+
+ public Builder partitionKey(Boolean partitionKey) {
+ this.partitionKey = partitionKey;
+ return this;
+ }
+
+ public Builder length(Long length) {
+ this.length = length;
+ return this;
+ }
+
+ public Builder scala(Integer scala) {
+ this.scala = scala;
+ return this;
+ }
+
+ public FieldMeta build() {
+ return new FieldMeta(this);
+ }
+ }
+ }
+
+ public static final class HudiMeta implements Serializable {
+ private String targetDataSource;
+ private String targetTable;
+ private String targetTableType;
+ private String targetHdfsPath;
+ private Integer sourceTasks;
+ private Integer writeTasks;
+ private String writeOperation;
+ private Integer writeTaskMaxMemory;
+ private Integer writeBatchSize;
+ private Integer writeRateLimit;
+ private Integer bucketIndexNumber;
+ private String compactionStrategy;
+ private Integer compactionTasks;
+ private Integer compactionDeltaCommits;
+ private Integer compactionDeltaSeconds;
+ private String compactionAsyncEnabled;
+ private Integer compactionMaxMemory;
+ private String configs;
+ private Integer keepFileVersion;
+ private Integer keepCommitVersion;
+
+ public HudiMeta() {
+ }
+
+ public HudiMeta(Builder builder) {
+ this.targetDataSource = builder.targetDataSource;
+ this.targetTable = builder.targetTable;
+ this.targetTableType = builder.targetTableType;
+ this.targetHdfsPath = builder.targetHdfsPath;
+ this.sourceTasks = builder.sourceTasks;
+ this.writeTasks = builder.writeTasks;
+ this.writeOperation = builder.writeOperation;
+ this.writeTaskMaxMemory = builder.writeTaskMaxMemory;
+ this.writeBatchSize = builder.writeBatchSize;
+ this.writeRateLimit = builder.writeRateLimit;
+ this.bucketIndexNumber = builder.bucketIndexNumber;
+ this.compactionStrategy = builder.compactionStrategy;
+ this.compactionTasks = builder.compactionTasks;
+ this.compactionDeltaCommits = builder.compactionDeltaCommits;
+ this.compactionDeltaSeconds = builder.compactionDeltaSeconds;
+ this.compactionAsyncEnabled = builder.compactionAsyncEnabled;
+ this.compactionMaxMemory = builder.compactionMaxMemory;
+ this.configs = builder.configs;
+ this.keepFileVersion = builder.keepFileVersion;
+ this.keepCommitVersion = builder.keepCommitVersion;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getTargetDataSource() {
+ return targetDataSource;
+ }
+
+ public void setTargetDataSource(String targetDataSource) {
+ this.targetDataSource = targetDataSource;
+ }
+
+ public String getTargetTable() {
+ return targetTable;
+ }
+
+ public void setTargetTable(String targetTable) {
+ this.targetTable = targetTable;
+ }
+
+ public String getTargetTableType() {
+ return targetTableType;
+ }
+
+ public void setTargetTableType(String targetTableType) {
+ this.targetTableType = targetTableType;
+ }
+
+ public String getTargetHdfsPath() {
+ return targetHdfsPath;
+ }
+
+ public void setTargetHdfsPath(String targetHdfsPath) {
+ this.targetHdfsPath = targetHdfsPath;
+ }
+
+ public Integer getSourceTasks() {
+ return sourceTasks;
+ }
+
+ public void setSourceTasks(Integer sourceTasks) {
+ this.sourceTasks = sourceTasks;
+ }
+
+ public Integer getWriteTasks() {
+ return writeTasks;
+ }
+
+ public void setWriteTasks(Integer writeTasks) {
+ this.writeTasks = writeTasks;
+ }
+
+ public String getWriteOperation() {
+ return writeOperation;
+ }
+
+ public void setWriteOperation(String writeOperation) {
+ this.writeOperation = writeOperation;
+ }
+
+ public Integer getWriteTaskMaxMemory() {
+ return writeTaskMaxMemory;
+ }
+
+ public void setWriteTaskMaxMemory(Integer writeTaskMaxMemory) {
+ this.writeTaskMaxMemory = writeTaskMaxMemory;
+ }
+
+ public Integer getWriteBatchSize() {
+ return writeBatchSize;
+ }
+
+ public void setWriteBatchSize(Integer writeBatchSize) {
+ this.writeBatchSize = writeBatchSize;
+ }
+
+ public Integer getWriteRateLimit() {
+ return writeRateLimit;
+ }
+
+ public void setWriteRateLimit(Integer writeRateLimit) {
+ this.writeRateLimit = writeRateLimit;
+ }
+
+ public Integer getBucketIndexNumber() {
+ return bucketIndexNumber;
+ }
+
+ public void setBucketIndexNumber(Integer bucketIndexNumber) {
+ this.bucketIndexNumber = bucketIndexNumber;
+ }
+
+ public String getCompactionStrategy() {
+ return compactionStrategy;
+ }
+
+ public void setCompactionStrategy(String compactionStrategy) {
+ this.compactionStrategy = compactionStrategy;
+ }
+
+ public Integer getCompactionTasks() {
+ return compactionTasks;
+ }
+
+ public void setCompactionTasks(Integer compactionTasks) {
+ this.compactionTasks = compactionTasks;
+ }
+
+ public Integer getCompactionDeltaCommits() {
+ return compactionDeltaCommits;
+ }
+
+ public void setCompactionDeltaCommits(Integer compactionDeltaCommits) {
+ this.compactionDeltaCommits = compactionDeltaCommits;
+ }
+
+ public Integer getCompactionDeltaSeconds() {
+ return compactionDeltaSeconds;
+ }
+
+ public void setCompactionDeltaSeconds(Integer compactionDeltaSeconds) {
+ this.compactionDeltaSeconds = compactionDeltaSeconds;
+ }
+
+ public String getCompactionAsyncEnabled() {
+ return compactionAsyncEnabled;
+ }
+
+ public void setCompactionAsyncEnabled(String compactionAsyncEnabled) {
+ this.compactionAsyncEnabled = compactionAsyncEnabled;
+ }
+
+ public Integer getCompactionMaxMemory() {
+ return compactionMaxMemory;
+ }
+
+ public void setCompactionMaxMemory(Integer compactionMaxMemory) {
+ this.compactionMaxMemory = compactionMaxMemory;
+ }
+
+ public String getConfigs() {
+ return configs;
+ }
+
+ public void setConfigs(String configs) {
+ this.configs = configs;
+ }
+
+ public Integer getKeepFileVersion() {
+ return keepFileVersion;
+ }
+
+ public void setKeepFileVersion(Integer keepFileVersion) {
+ this.keepFileVersion = keepFileVersion;
+ }
+
+ public Integer getKeepCommitVersion() {
+ return keepCommitVersion;
+ }
+
+ public void setKeepCommitVersion(Integer keepCommitVersion) {
+ this.keepCommitVersion = keepCommitVersion;
+ }
+
+ @Override
+ public String toString() {
+ return "HudiMeta{" +
+ "targetDataSource='" + targetDataSource + '\'' +
+ ", targetTable='" + targetTable + '\'' +
+ ", targetTableType='" + targetTableType + '\'' +
+ ", targetHdfsPath='" + targetHdfsPath + '\'' +
+ ", sourceTasks=" + sourceTasks +
+ ", writeTasks=" + writeTasks +
+ ", writeOperation='" + writeOperation + '\'' +
+ ", writeTaskMaxMemory=" + writeTaskMaxMemory +
+ ", writeBatchSize=" + writeBatchSize +
+ ", writeRateLimit=" + writeRateLimit +
+ ", bucketIndexNumber=" + bucketIndexNumber +
+ ", compactionStrategy='" + compactionStrategy + '\'' +
+ ", compactionTasks=" + compactionTasks +
+ ", compactionDeltaCommits=" + compactionDeltaCommits +
+ ", compactionDeltaSeconds=" + compactionDeltaSeconds +
+ ", compactionAsyncEnabled='" + compactionAsyncEnabled + '\'' +
+ ", compactionMaxMemory=" + compactionMaxMemory +
+ ", configs='" + configs + '\'' +
+ ", keepFileVersion=" + keepFileVersion +
+ ", keepCommitVersion=" + keepCommitVersion +
+ '}';
+ }
+
+ public static final class Builder {
+ private String targetDataSource;
+ private String targetTable;
+ private String targetTableType;
+ private String targetHdfsPath;
+ private Integer sourceTasks;
+ private Integer writeTasks;
+ private String writeOperation;
+ private Integer writeTaskMaxMemory;
+ private Integer writeBatchSize;
+ private Integer writeRateLimit;
+ private Integer bucketIndexNumber;
+ private String compactionStrategy;
+ private Integer compactionTasks;
+ private Integer compactionDeltaCommits;
+ private Integer compactionDeltaSeconds;
+ private String compactionAsyncEnabled;
+ private Integer compactionMaxMemory;
+ private String configs;
+ private Integer keepFileVersion;
+ private Integer keepCommitVersion;
+
+ private Builder() {
+ }
+
+ public Builder targetDataSource(String targetDataSource) {
+ this.targetDataSource = targetDataSource;
+ return this;
+ }
+
+ public Builder targetTable(String targetTable) {
+ this.targetTable = targetTable;
+ return this;
+ }
+
+ public Builder targetTableType(String targetTableType) {
+ this.targetTableType = targetTableType;
+ return this;
+ }
+
+ public Builder targetHdfsPath(String targetHdfsPath) {
+ this.targetHdfsPath = targetHdfsPath;
+ return this;
+ }
+
+ public Builder sourceTasks(Integer sourceTasks) {
+ this.sourceTasks = sourceTasks;
+ return this;
+ }
+
+ public Builder writeTasks(Integer writeTasks) {
+ this.writeTasks = writeTasks;
+ return this;
+ }
+
+ public Builder writeOperation(String writeOperation) {
+ this.writeOperation = writeOperation;
+ return this;
+ }
+
+ public Builder writeTaskMaxMemory(Integer writeTaskMaxMemory) {
+ this.writeTaskMaxMemory = writeTaskMaxMemory;
+ return this;
+ }
+
+ public Builder writeBatchSize(Integer writeBatchSize) {
+ this.writeBatchSize = writeBatchSize;
+ return this;
+ }
+
+ public Builder writeRateLimit(Integer writeRateLimit) {
+ this.writeRateLimit = writeRateLimit;
+ return this;
+ }
+
+ public Builder bucketIndexNumber(Integer bucketIndexNumber) {
+ this.bucketIndexNumber = bucketIndexNumber;
+ return this;
+ }
+
+ public Builder compactionStrategy(String compactionStrategy) {
+ this.compactionStrategy = compactionStrategy;
+ return this;
+ }
+
+ public Builder compactionTasks(Integer compactionTasks) {
+ this.compactionTasks = compactionTasks;
+ return this;
+ }
+
+ public Builder compactionDeltaCommits(Integer compactionDeltaCommits) {
+ this.compactionDeltaCommits = compactionDeltaCommits;
+ return this;
+ }
+
+ public Builder compactionDeltaSeconds(Integer compactionDeltaSeconds) {
+ this.compactionDeltaSeconds = compactionDeltaSeconds;
+ return this;
+ }
+
+ public Builder compactionAsyncEnabled(String compactionAsyncEnabled) {
+ this.compactionAsyncEnabled = compactionAsyncEnabled;
+ return this;
+ }
+
+ public Builder compactionMaxMemory(Integer compactionMaxMemory) {
+ this.compactionMaxMemory = compactionMaxMemory;
+ return this;
+ }
+
+ public Builder configs(String configs) {
+ this.configs = configs;
+ return this;
+ }
+
+ public Builder keepFileVersion(Integer keepFileVersion) {
+ this.keepFileVersion = keepFileVersion;
+ return this;
+ }
+
+ public Builder keepCommitVersion(Integer keepCommitVersion) {
+ this.keepCommitVersion = keepCommitVersion;
+ return this;
+ }
+
+ public HudiMeta build() {
+ return new HudiMeta(this);
+ }
+ }
+ }
+
+ public static final class YarnMeta implements Serializable {
+ private Integer jobManagerMemory;
+ private Integer taskManagerMemory;
+
+ public YarnMeta() {
+ }
+
+ public YarnMeta(Builder builder) {
+ this.jobManagerMemory = builder.jobManagerMemory;
+ this.taskManagerMemory = builder.taskManagerMemory;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public String toString() {
+ return "YarnMeta{" + "jobManagerMemory=" + jobManagerMemory + ", taskManagerMemory=" + taskManagerMemory + '}';
+ }
+
+ public Integer getJobManagerMemory() {
+ return jobManagerMemory;
+ }
+
+ public YarnMeta setJobManagerMemory(Integer jobManagerMemory) {
+ this.jobManagerMemory = jobManagerMemory;
+ return this;
+ }
+
+ public Integer getTaskManagerMemory() {
+ return taskManagerMemory;
+ }
+
+ public YarnMeta setTaskManagerMemory(Integer taskManagerMemory) {
+ this.taskManagerMemory = taskManagerMemory;
+ return this;
+ }
+
+ public static final class Builder {
+ private Integer jobManagerMemory;
+ private Integer taskManagerMemory;
+
+ private Builder() {
+ }
+
+ public Builder jobManagerMemory(Integer jobManagerMemory) {
+ this.jobManagerMemory = jobManagerMemory;
+ return this;
+ }
+
+ public Builder taskManagerMemory(Integer taskManagerMemory) {
+ this.taskManagerMemory = taskManagerMemory;
+ return this;
+ }
+
+ public YarnMeta build() {
+ return new YarnMeta(this);
+ }
+ }
+ }
+
+ public static final class ConfigMeta implements Serializable {
+ private String messageId;
+ private String metricPublishUrl;
+ private String metricPrometheusUrl;
+ private String metricApiUrl;
+ private Integer metricPublishDelay;
+ private Integer metricPublishPeriod;
+ private Integer metricPublishTimeout;
+ private Integer metricPublishBatch;
+ private String checkpointRootPath;
+ private String zookeeperUrl;
+
+ public ConfigMeta() {
+ }
+
+ public ConfigMeta(Builder builder) {
+ this.messageId = builder.messageId;
+ this.metricPublishUrl = builder.metricPublishUrl;
+ this.metricPrometheusUrl = builder.metricPrometheusUrl;
+ this.metricApiUrl = builder.metricApiUrl;
+ this.metricPublishDelay = builder.metricPublishDelay;
+ this.metricPublishPeriod = builder.metricPublishPeriod;
+ this.metricPublishTimeout = builder.metricPublishTimeout;
+ this.metricPublishBatch = builder.metricPublishBatch;
+ this.checkpointRootPath = builder.checkpointRootPath;
+ this.zookeeperUrl = builder.zookeeperUrl;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getMessageId() {
+ return messageId;
+ }
+
+ public void setMessageId(String messageId) {
+ this.messageId = messageId;
+ }
+
+ public String getMetricPublishUrl() {
+ return metricPublishUrl;
+ }
+
+ public void setMetricPublishUrl(String metricPublishUrl) {
+ this.metricPublishUrl = metricPublishUrl;
+ }
+
+ public String getMetricPrometheusUrl() {
+ return metricPrometheusUrl;
+ }
+
+ public void setMetricPrometheusUrl(String metricPrometheusUrl) {
+ this.metricPrometheusUrl = metricPrometheusUrl;
+ }
+
+ public String getMetricApiUrl() {
+ return metricApiUrl;
+ }
+
+ public void setMetricApiUrl(String metricApiUrl) {
+ this.metricApiUrl = metricApiUrl;
+ }
+
+ public Integer getMetricPublishDelay() {
+ return metricPublishDelay;
+ }
+
+ public void setMetricPublishDelay(Integer metricPublishDelay) {
+ this.metricPublishDelay = metricPublishDelay;
+ }
+
+ public Integer getMetricPublishPeriod() {
+ return metricPublishPeriod;
+ }
+
+ public void setMetricPublishPeriod(Integer metricPublishPeriod) {
+ this.metricPublishPeriod = metricPublishPeriod;
+ }
+
+ public Integer getMetricPublishTimeout() {
+ return metricPublishTimeout;
+ }
+
+ public void setMetricPublishTimeout(Integer metricPublishTimeout) {
+ this.metricPublishTimeout = metricPublishTimeout;
+ }
+
+ public Integer getMetricPublishBatch() {
+ return metricPublishBatch;
+ }
+
+ public void setMetricPublishBatch(Integer metricPublishBatch) {
+ this.metricPublishBatch = metricPublishBatch;
+ }
+
+ public String getCheckpointRootPath() {
+ return checkpointRootPath;
+ }
+
+ public void setCheckpointRootPath(String checkpointRootPath) {
+ this.checkpointRootPath = checkpointRootPath;
+ }
+
+ public String getZookeeperUrl() {
+ return zookeeperUrl;
+ }
+
+ public void setZookeeperUrl(String zookeeperUrl) {
+ this.zookeeperUrl = zookeeperUrl;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigMeta{" +
+ "messageId='" + messageId + '\'' +
+ ", metricPublishUrl='" + metricPublishUrl + '\'' +
+ ", metricPrometheusUrl='" + metricPrometheusUrl + '\'' +
+ ", metricApiUrl='" + metricApiUrl + '\'' +
+ ", metricPublishDelay=" + metricPublishDelay +
+ ", metricPublishPeriod=" + metricPublishPeriod +
+ ", metricPublishTimeout=" + metricPublishTimeout +
+ ", metricPublishBatch=" + metricPublishBatch +
+ ", checkpointRootPath='" + checkpointRootPath + '\'' +
+ ", zookeeperUrl='" + zookeeperUrl + '\'' +
+ '}';
+ }
+
+ public static final class Builder {
+ private String messageId;
+ private String metricPublishUrl;
+ private String metricPrometheusUrl;
+ private String metricApiUrl;
+ private Integer metricPublishDelay;
+ private Integer metricPublishPeriod;
+ private Integer metricPublishTimeout;
+ private Integer metricPublishBatch;
+ private String checkpointRootPath;
+ private String zookeeperUrl;
+
+ private Builder() {
+ }
+
+ public Builder messageId(String messageId) {
+ this.messageId = messageId;
+ return this;
+ }
+
+ public Builder metricPublishUrl(String metricPublishUrl) {
+ this.metricPublishUrl = metricPublishUrl;
+ return this;
+ }
+
+ public Builder metricPrometheusUrl(String metricPrometheusUrl) {
+ this.metricPrometheusUrl = metricPrometheusUrl;
+ return this;
+ }
+
+ public Builder metricApiUrl(String metricApiUrl) {
+ this.metricApiUrl = metricApiUrl;
+ return this;
+ }
+
+ public Builder metricPublishDelay(Integer metricPublishDelay) {
+ this.metricPublishDelay = metricPublishDelay;
+ return this;
+ }
+
+ public Builder metricPublishPeriod(Integer metricPublishPeriod) {
+ this.metricPublishPeriod = metricPublishPeriod;
+ return this;
+ }
+
+ public Builder metricPublishTimeout(Integer metricPublishTimeout) {
+ this.metricPublishTimeout = metricPublishTimeout;
+ return this;
+ }
+
+ public Builder metricPublishBatch(Integer metricPublishBatch) {
+ this.metricPublishBatch = metricPublishBatch;
+ return this;
+ }
+
+ public Builder checkpointRootPath(String checkpointRootPath) {
+ this.checkpointRootPath = checkpointRootPath;
+ return this;
+ }
+
+ public Builder zookeeperUrl(String zookeeperUrl) {
+ this.zookeeperUrl = zookeeperUrl;
+ return this;
+ }
+
+ public ConfigMeta build() {
+ return new ConfigMeta(this);
+ }
+ }
+ }
+
+ public static final class JobMeta implements Serializable {
+ private Long id;
+ private String name;
+
+ public JobMeta() {
+ }
+
+ public JobMeta(Builder builder) {
+ this.id = builder.id;
+ this.name = builder.name;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public Long getId() {
+ return id;
+ }
+
+ public void setId(Long id) {
+ this.id = id;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return "JobMeta{" +
+ "id=" + id +
+ ", name='" + name + '\'' +
+ '}';
+ }
+
+ public static final class Builder {
+ private Long id;
+ private String name;
+
+ private Builder() {
+ }
+
+ public Builder id(Long id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder name(String name) {
+ this.name = name;
+ return this;
+ }
+
+ public JobMeta build() {
+ return new JobMeta(this);
+ }
+ }
+ }
+
+ public static final class RowMeta implements Serializable {
+ private String dsName;
+ private String schemaName;
+ private String tableName;
+ private String tableType;
+ private String fieldName;
+ private Integer fieldSeq;
+ private String fieldType;
+ private String primaryKey;
+ private String partitionKey;
+ private Long length;
+ private String tgtDb;
+ private String tgtTable;
+ private String tgtTableType;
+ private String tgtHdfsPath;
+ private Integer writeTasks;
+ private String writeOperation;
+ private Integer writeTaskMaxMemory;
+ private Integer writeBatchSize;
+ private Integer writeRateLimit;
+ private Integer bucketIndexNumber;
+ private String compactionStrategy;
+ private Integer compactionTasks;
+ private Integer compactionDeltaCommits;
+ private Integer compactionDeltaSeconds;
+ private String compactionAsyncEnabled;
+ private Integer compactionMaxMemory;
+ private String configs;
+ private String filterField;
+ private String filterValues;
+ private String filterType;
+ private String topic;
+ private String pulsarAddress;
+ private Integer syncJobManagerMemory;
+ private Integer syncTaskManagerMemory;
+ private Integer compactionJobManagerMemory;
+ private Integer compactionTaskManagerMemory;
+ private String partitionField;
+ private String messageId;
+ private String metricPublishUrl;
+ private String metricPrometheusUrl;
+ private String metricApiUrl;
+ private Integer metricPublishDelay;
+ private Integer metricPublishPeriod;
+ private Integer metricPublishTimeout;
+ private Integer metricPublishBatch;
+ private Long jobId;
+ private String jobName;
+ private String checkpointRootPath;
+ private Integer sourceTasks;
+ private String alias;
+ private String connection;
+ private Integer priority;
+ private String sourceType;
+ private Integer keepFileVersion;
+ private Integer keepCommitVersion;
+ private String tags;
+ private String zookeeperUrl;
+ private Integer version;
+ private Integer scala;
+
+ public RowMeta() {
+ }
+
+ public RowMeta(Builder builder) {
+ this.dsName = builder.dsName;
+ this.schemaName = builder.schemaName;
+ this.tableName = builder.tableName;
+ this.tableType = builder.tableType;
+ this.fieldName = builder.fieldName;
+ this.fieldSeq = builder.fieldSeq;
+ this.fieldType = builder.fieldType;
+ this.primaryKey = builder.primaryKey;
+ this.partitionKey = builder.partitionKey;
+ this.length = builder.length;
+ this.tgtDb = builder.tgtDb;
+ this.tgtTable = builder.tgtTable;
+ this.tgtTableType = builder.tgtTableType;
+ this.tgtHdfsPath = builder.tgtHdfsPath;
+ this.writeTasks = builder.writeTasks;
+ this.writeOperation = builder.writeOperation;
+ this.writeTaskMaxMemory = builder.writeTaskMaxMemory;
+ this.writeBatchSize = builder.writeBatchSize;
+ this.writeRateLimit = builder.writeRateLimit;
+ this.bucketIndexNumber = builder.bucketIndexNumber;
+ this.compactionStrategy = builder.compactionStrategy;
+ this.compactionTasks = builder.compactionTasks;
+ this.compactionDeltaCommits = builder.compactionDeltaCommits;
+ this.compactionDeltaSeconds = builder.compactionDeltaSeconds;
+ this.compactionAsyncEnabled = builder.compactionAsyncEnabled;
+ this.compactionMaxMemory = builder.compactionMaxMemory;
+ this.configs = builder.configs;
+ this.filterField = builder.filterField;
+ this.filterValues = builder.filterValues;
+ this.filterType = builder.filterType;
+ this.topic = builder.topic;
+ this.pulsarAddress = builder.pulsarAddress;
+ this.syncJobManagerMemory = builder.syncJobManagerMemory;
+ this.syncTaskManagerMemory = builder.syncTaskManagerMemory;
+ this.compactionJobManagerMemory = builder.compactionJobManagerMemory;
+ this.compactionTaskManagerMemory = builder.compactionTaskManagerMemory;
+ this.partitionField = builder.partitionField;
+ this.messageId = builder.messageId;
+ this.metricPublishUrl = builder.metricPublishUrl;
+ this.metricPrometheusUrl = builder.metricPrometheusUrl;
+ this.metricApiUrl = builder.metricApiUrl;
+ this.metricPublishDelay = builder.metricPublishDelay;
+ this.metricPublishPeriod = builder.metricPublishPeriod;
+ this.metricPublishTimeout = builder.metricPublishTimeout;
+ this.metricPublishBatch = builder.metricPublishBatch;
+ this.jobId = builder.jobId;
+ this.jobName = builder.jobName;
+ this.checkpointRootPath = builder.checkpointRootPath;
+ this.sourceTasks = builder.sourceTasks;
+ this.alias = builder.alias;
+ this.connection = builder.connection;
+ this.priority = builder.priority;
+ this.sourceType = builder.sourceType;
+ this.keepFileVersion = builder.keepFileVersion;
+ this.keepCommitVersion = builder.keepCommitVersion;
+ this.tags = builder.tags;
+ this.zookeeperUrl = builder.zookeeperUrl;
+ this.version = builder.version;
+ this.scala = builder.scala;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getDsName() {
+ return dsName;
+ }
+
+ public void setDsName(String dsName) {
+ this.dsName = dsName;
+ }
+
+ public String getSchemaName() {
+ return schemaName;
+ }
+
+ public void setSchemaName(String schemaName) {
+ this.schemaName = schemaName;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public void setTableName(String tableName) {
+ this.tableName = tableName;
+ }
+
+ public String getTableType() {
+ return tableType;
+ }
+
+ public void setTableType(String tableType) {
+ this.tableType = tableType;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+
+ public void setFieldName(String fieldName) {
+ this.fieldName = fieldName;
+ }
+
+ public Integer getFieldSeq() {
+ return fieldSeq;
+ }
+
+ public void setFieldSeq(Integer fieldSeq) {
+ this.fieldSeq = fieldSeq;
+ }
+
+ public String getFieldType() {
+ return fieldType;
+ }
+
+ public void setFieldType(String fieldType) {
+ this.fieldType = fieldType;
+ }
+
+ public String getPrimaryKey() {
+ return primaryKey;
+ }
+
+ public void setPrimaryKey(String primaryKey) {
+ this.primaryKey = primaryKey;
+ }
+
+ public String getPartitionKey() {
+ return partitionKey;
+ }
+
+ public void setPartitionKey(String partitionKey) {
+ this.partitionKey = partitionKey;
+ }
+
+ public Long getLength() {
+ return length;
+ }
+
+ public void setLength(Long length) {
+ this.length = length;
+ }
+
+ public String getTgtDb() {
+ return tgtDb;
+ }
+
+ public void setTgtDb(String tgtDb) {
+ this.tgtDb = tgtDb;
+ }
+
+ public String getTgtTable() {
+ return tgtTable;
+ }
+
+ public void setTgtTable(String tgtTable) {
+ this.tgtTable = tgtTable;
+ }
+
+ public String getTgtTableType() {
+ return tgtTableType;
+ }
+
+ public void setTgtTableType(String tgtTableType) {
+ this.tgtTableType = tgtTableType;
+ }
+
+ public String getTgtHdfsPath() {
+ return tgtHdfsPath;
+ }
+
+ public void setTgtHdfsPath(String tgtHdfsPath) {
+ this.tgtHdfsPath = tgtHdfsPath;
+ }
+
+ public Integer getWriteTasks() {
+ return writeTasks;
+ }
+
+ public void setWriteTasks(Integer writeTasks) {
+ this.writeTasks = writeTasks;
+ }
+
+ public String getWriteOperation() {
+ return writeOperation;
+ }
+
+ public void setWriteOperation(String writeOperation) {
+ this.writeOperation = writeOperation;
+ }
+
+ public Integer getWriteTaskMaxMemory() {
+ return writeTaskMaxMemory;
+ }
+
+ public void setWriteTaskMaxMemory(Integer writeTaskMaxMemory) {
+ this.writeTaskMaxMemory = writeTaskMaxMemory;
+ }
+
+ public Integer getWriteBatchSize() {
+ return writeBatchSize;
+ }
+
+ public void setWriteBatchSize(Integer writeBatchSize) {
+ this.writeBatchSize = writeBatchSize;
+ }
+
+ public Integer getWriteRateLimit() {
+ return writeRateLimit;
+ }
+
+ public void setWriteRateLimit(Integer writeRateLimit) {
+ this.writeRateLimit = writeRateLimit;
+ }
+
+ public Integer getBucketIndexNumber() {
+ return bucketIndexNumber;
+ }
+
+ public void setBucketIndexNumber(Integer bucketIndexNumber) {
+ this.bucketIndexNumber = bucketIndexNumber;
+ }
+
+ public String getCompactionStrategy() {
+ return compactionStrategy;
+ }
+
+ public void setCompactionStrategy(String compactionStrategy) {
+ this.compactionStrategy = compactionStrategy;
+ }
+
+ public Integer getCompactionTasks() {
+ return compactionTasks;
+ }
+
+ public void setCompactionTasks(Integer compactionTasks) {
+ this.compactionTasks = compactionTasks;
+ }
+
+ public Integer getCompactionDeltaCommits() {
+ return compactionDeltaCommits;
+ }
+
+ public void setCompactionDeltaCommits(Integer compactionDeltaCommits) {
+ this.compactionDeltaCommits = compactionDeltaCommits;
+ }
+
+ public Integer getCompactionDeltaSeconds() {
+ return compactionDeltaSeconds;
+ }
+
+ public void setCompactionDeltaSeconds(Integer compactionDeltaSeconds) {
+ this.compactionDeltaSeconds = compactionDeltaSeconds;
+ }
+
+ public String getCompactionAsyncEnabled() {
+ return compactionAsyncEnabled;
+ }
+
+ public void setCompactionAsyncEnabled(String compactionAsyncEnabled) {
+ this.compactionAsyncEnabled = compactionAsyncEnabled;
+ }
+
+ public Integer getCompactionMaxMemory() {
+ return compactionMaxMemory;
+ }
+
+ public void setCompactionMaxMemory(Integer compactionMaxMemory) {
+ this.compactionMaxMemory = compactionMaxMemory;
+ }
+
+ public String getConfigs() {
+ return configs;
+ }
+
+ public void setConfigs(String configs) {
+ this.configs = configs;
+ }
+
+ public String getFilterField() {
+ return filterField;
+ }
+
+ public void setFilterField(String filterField) {
+ this.filterField = filterField;
+ }
+
+ public String getFilterValues() {
+ return filterValues;
+ }
+
+ public void setFilterValues(String filterValues) {
+ this.filterValues = filterValues;
+ }
+
+ public String getFilterType() {
+ return filterType;
+ }
+
+ public void setFilterType(String filterType) {
+ this.filterType = filterType;
+ }
+
+ public String getTopic() {
+ return topic;
+ }
+
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ public String getPulsarAddress() {
+ return pulsarAddress;
+ }
+
+ public void setPulsarAddress(String pulsarAddress) {
+ this.pulsarAddress = pulsarAddress;
+ }
+
+ public Integer getSyncJobManagerMemory() {
+ return syncJobManagerMemory;
+ }
+
+ public void setSyncJobManagerMemory(Integer syncJobManagerMemory) {
+ this.syncJobManagerMemory = syncJobManagerMemory;
+ }
+
+ public Integer getSyncTaskManagerMemory() {
+ return syncTaskManagerMemory;
+ }
+
+ public void setSyncTaskManagerMemory(Integer syncTaskManagerMemory) {
+ this.syncTaskManagerMemory = syncTaskManagerMemory;
+ }
+
+ public Integer getCompactionJobManagerMemory() {
+ return compactionJobManagerMemory;
+ }
+
+ public void setCompactionJobManagerMemory(Integer compactionJobManagerMemory) {
+ this.compactionJobManagerMemory = compactionJobManagerMemory;
+ }
+
+ public Integer getCompactionTaskManagerMemory() {
+ return compactionTaskManagerMemory;
+ }
+
+ public void setCompactionTaskManagerMemory(Integer compactionTaskManagerMemory) {
+ this.compactionTaskManagerMemory = compactionTaskManagerMemory;
+ }
+
+ public String getPartitionField() {
+ return partitionField;
+ }
+
+ public void setPartitionField(String partitionField) {
+ this.partitionField = partitionField;
+ }
+
+ public String getMessageId() {
+ return messageId;
+ }
+
+ public void setMessageId(String messageId) {
+ this.messageId = messageId;
+ }
+
+ public String getMetricPublishUrl() {
+ return metricPublishUrl;
+ }
+
+ public void setMetricPublishUrl(String metricPublishUrl) {
+ this.metricPublishUrl = metricPublishUrl;
+ }
+
+ public String getMetricPrometheusUrl() {
+ return metricPrometheusUrl;
+ }
+
+ public void setMetricPrometheusUrl(String metricPrometheusUrl) {
+ this.metricPrometheusUrl = metricPrometheusUrl;
+ }
+
+ public String getMetricApiUrl() {
+ return metricApiUrl;
+ }
+
+ public void setMetricApiUrl(String metricApiUrl) {
+ this.metricApiUrl = metricApiUrl;
+ }
+
+ public Integer getMetricPublishDelay() {
+ return metricPublishDelay;
+ }
+
+ public void setMetricPublishDelay(Integer metricPublishDelay) {
+ this.metricPublishDelay = metricPublishDelay;
+ }
+
+ public Integer getMetricPublishPeriod() {
+ return metricPublishPeriod;
+ }
+
+ public void setMetricPublishPeriod(Integer metricPublishPeriod) {
+ this.metricPublishPeriod = metricPublishPeriod;
+ }
+
+ public Integer getMetricPublishTimeout() {
+ return metricPublishTimeout;
+ }
+
+ public void setMetricPublishTimeout(Integer metricPublishTimeout) {
+ this.metricPublishTimeout = metricPublishTimeout;
+ }
+
+ public Integer getMetricPublishBatch() {
+ return metricPublishBatch;
+ }
+
+ public void setMetricPublishBatch(Integer metricPublishBatch) {
+ this.metricPublishBatch = metricPublishBatch;
+ }
+
+ public Long getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(Long jobId) {
+ this.jobId = jobId;
+ }
+
+ public String getJobName() {
+ return jobName;
+ }
+
+ public void setJobName(String jobName) {
+ this.jobName = jobName;
+ }
+
+ public String getCheckpointRootPath() {
+ return checkpointRootPath;
+ }
+
+ public void setCheckpointRootPath(String checkpointRootPath) {
+ this.checkpointRootPath = checkpointRootPath;
+ }
+
+ public Integer getSourceTasks() {
+ return sourceTasks;
+ }
+
+ public void setSourceTasks(Integer sourceTasks) {
+ this.sourceTasks = sourceTasks;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public String getConnection() {
+ return connection;
+ }
+
+ public void setConnection(String connection) {
+ this.connection = connection;
+ }
+
+ public Integer getPriority() {
+ return priority;
+ }
+
+ public void setPriority(Integer priority) {
+ this.priority = priority;
+ }
+
+ public String getSourceType() {
+ return sourceType;
+ }
+
+ public void setSourceType(String sourceType) {
+ this.sourceType = sourceType;
+ }
+
+ public Integer getKeepFileVersion() {
+ return keepFileVersion;
+ }
+
+ public void setKeepFileVersion(Integer keepFileVersion) {
+ this.keepFileVersion = keepFileVersion;
+ }
+
+ public Integer getKeepCommitVersion() {
+ return keepCommitVersion;
+ }
+
+ public void setKeepCommitVersion(Integer keepCommitVersion) {
+ this.keepCommitVersion = keepCommitVersion;
+ }
+
+ public String getTags() {
+ return tags;
+ }
+
+ public void setTags(String tags) {
+ this.tags = tags;
+ }
+
+ public String getZookeeperUrl() {
+ return zookeeperUrl;
+ }
+
+ public void setZookeeperUrl(String zookeeperUrl) {
+ this.zookeeperUrl = zookeeperUrl;
+ }
+
+ public Integer getVersion() {
+ return version;
+ }
+
+ public void setVersion(Integer version) {
+ this.version = version;
+ }
+
+ public Integer getScala() {
+ return scala;
+ }
+
+ public void setScala(Integer scala) {
+ this.scala = scala;
+ }
+
+ public static final class Builder {
+ private String dsName;
+ private String schemaName;
+ private String tableName;
+ private String tableType;
+ private String fieldName;
+ private Integer fieldSeq;
+ private String fieldType;
+ private String primaryKey;
+ private String partitionKey;
+ private Long length;
+ private String tgtDb;
+ private String tgtTable;
+ private String tgtTableType;
+ private String tgtHdfsPath;
+ private Integer writeTasks;
+ private String writeOperation;
+ private Integer writeTaskMaxMemory;
+ private Integer writeBatchSize;
+ private Integer writeRateLimit;
+ private Integer bucketIndexNumber;
+ private String compactionStrategy;
+ private Integer compactionTasks;
+ private Integer compactionDeltaCommits;
+ private Integer compactionDeltaSeconds;
+ private String compactionAsyncEnabled;
+ private Integer compactionMaxMemory;
+ private String configs;
+ private String filterField;
+ private String filterValues;
+ private String filterType;
+ private String topic;
+ private String pulsarAddress;
+ private Integer syncJobManagerMemory;
+ private Integer syncTaskManagerMemory;
+ private Integer compactionJobManagerMemory;
+ private Integer compactionTaskManagerMemory;
+ private String partitionField;
+ private String messageId;
+ private String metricPublishUrl;
+ private String metricPrometheusUrl;
+ private String metricApiUrl;
+ private Integer metricPublishDelay;
+ private Integer metricPublishPeriod;
+ private Integer metricPublishTimeout;
+ private Integer metricPublishBatch;
+ private Long jobId;
+ private String jobName;
+ private String checkpointRootPath;
+ private Integer sourceTasks;
+ private String alias;
+ private String connection;
+ private Integer priority;
+ private String sourceType;
+ private Integer keepFileVersion;
+ private Integer keepCommitVersion;
+ private String tags;
+ private String zookeeperUrl;
+ private Integer version;
+ private Integer scala;
+
+ private Builder() {
+ }
+
+ public Builder dsName(String dsName) {
+ this.dsName = dsName;
+ return this;
+ }
+
+ public Builder schemaName(String schemaName) {
+ this.schemaName = schemaName;
+ return this;
+ }
+
+ public Builder tableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public Builder tableType(String tableType) {
+ this.tableType = tableType;
+ return this;
+ }
+
+ public Builder fieldName(String fieldName) {
+ this.fieldName = fieldName;
+ return this;
+ }
+
+ public Builder fieldSeq(Integer fieldSeq) {
+ this.fieldSeq = fieldSeq;
+ return this;
+ }
+
+ public Builder fieldType(String fieldType) {
+ this.fieldType = fieldType;
+ return this;
+ }
+
+ public Builder primaryKey(String primaryKey) {
+ this.primaryKey = primaryKey;
+ return this;
+ }
+
+ public Builder partitionKey(String partitionKey) {
+ this.partitionKey = partitionKey;
+ return this;
+ }
+
+ public Builder length(Long length) {
+ this.length = length;
+ return this;
+ }
+
+ public Builder tgtDb(String tgtDb) {
+ this.tgtDb = tgtDb;
+ return this;
+ }
+
+ public Builder tgtTable(String tgtTable) {
+ this.tgtTable = tgtTable;
+ return this;
+ }
+
+ public Builder tgtTableType(String tgtTableType) {
+ this.tgtTableType = tgtTableType;
+ return this;
+ }
+
+ public Builder tgtHdfsPath(String tgtHdfsPath) {
+ this.tgtHdfsPath = tgtHdfsPath;
+ return this;
+ }
+
+ public Builder writeTasks(Integer writeTasks) {
+ this.writeTasks = writeTasks;
+ return this;
+ }
+
+ public Builder writeOperation(String writeOperation) {
+ this.writeOperation = writeOperation;
+ return this;
+ }
+
+ public Builder writeTaskMaxMemory(Integer writeTaskMaxMemory) {
+ this.writeTaskMaxMemory = writeTaskMaxMemory;
+ return this;
+ }
+
+ public Builder writeBatchSize(Integer writeBatchSize) {
+ this.writeBatchSize = writeBatchSize;
+ return this;
+ }
+
+ public Builder writeRateLimit(Integer writeRateLimit) {
+ this.writeRateLimit = writeRateLimit;
+ return this;
+ }
+
+ public Builder bucketIndexNumber(Integer bucketIndexNumber) {
+ this.bucketIndexNumber = bucketIndexNumber;
+ return this;
+ }
+
+ public Builder compactionStrategy(String compactionStrategy) {
+ this.compactionStrategy = compactionStrategy;
+ return this;
+ }
+
+ public Builder compactionTasks(Integer compactionTasks) {
+ this.compactionTasks = compactionTasks;
+ return this;
+ }
+
+ public Builder compactionDeltaCommits(Integer compactionDeltaCommits) {
+ this.compactionDeltaCommits = compactionDeltaCommits;
+ return this;
+ }
+
+ public Builder compactionDeltaSeconds(Integer compactionDeltaSeconds) {
+ this.compactionDeltaSeconds = compactionDeltaSeconds;
+ return this;
+ }
+
+ public Builder compactionAsyncEnabled(String compactionAsyncEnabled) {
+ this.compactionAsyncEnabled = compactionAsyncEnabled;
+ return this;
+ }
+
+ public Builder compactionMaxMemory(Integer compactionMaxMemory) {
+ this.compactionMaxMemory = compactionMaxMemory;
+ return this;
+ }
+
+ public Builder configs(String configs) {
+ this.configs = configs;
+ return this;
+ }
+
+ public Builder filterField(String filterField) {
+ this.filterField = filterField;
+ return this;
+ }
+
+ public Builder filterValues(String filterValues) {
+ this.filterValues = filterValues;
+ return this;
+ }
+
+ public Builder filterType(String filterType) {
+ this.filterType = filterType;
+ return this;
+ }
+
+ public Builder topic(String topic) {
+ this.topic = topic;
+ return this;
+ }
+
+ public Builder pulsarAddress(String pulsarAddress) {
+ this.pulsarAddress = pulsarAddress;
+ return this;
+ }
+
+ public Builder syncJobManagerMemory(Integer syncJobManagerMemory) {
+ this.syncJobManagerMemory = syncJobManagerMemory;
+ return this;
+ }
+
+ public Builder syncTaskManagerMemory(Integer syncTaskManagerMemory) {
+ this.syncTaskManagerMemory = syncTaskManagerMemory;
+ return this;
+ }
+
+ public Builder compactionJobManagerMemory(Integer compactionJobManagerMemory) {
+ this.compactionJobManagerMemory = compactionJobManagerMemory;
+ return this;
+ }
+
+ public Builder compactionTaskManagerMemory(Integer compactionTaskManagerMemory) {
+ this.compactionTaskManagerMemory = compactionTaskManagerMemory;
+ return this;
+ }
+
+ public Builder partitionField(String partitionField) {
+ this.partitionField = partitionField;
+ return this;
+ }
+
+ public Builder messageId(String messageId) {
+ this.messageId = messageId;
+ return this;
+ }
+
+ public Builder metricPublishUrl(String metricPublishUrl) {
+ this.metricPublishUrl = metricPublishUrl;
+ return this;
+ }
+
+ public Builder metricPrometheusUrl(String metricPrometheusUrl) {
+ this.metricPrometheusUrl = metricPrometheusUrl;
+ return this;
+ }
+
+ public Builder metricApiUrl(String metricApiUrl) {
+ this.metricApiUrl = metricApiUrl;
+ return this;
+ }
+
+ public Builder metricPublishDelay(Integer metricPublishDelay) {
+ this.metricPublishDelay = metricPublishDelay;
+ return this;
+ }
+
+ public Builder metricPublishPeriod(Integer metricPublishPeriod) {
+ this.metricPublishPeriod = metricPublishPeriod;
+ return this;
+ }
+
+ public Builder metricPublishTimeout(Integer metricPublishTimeout) {
+ this.metricPublishTimeout = metricPublishTimeout;
+ return this;
+ }
+
+ public Builder metricPublishBatch(Integer metricPublishBatch) {
+ this.metricPublishBatch = metricPublishBatch;
+ return this;
+ }
+
+ public Builder jobId(Long jobId) {
+ this.jobId = jobId;
+ return this;
+ }
+
+ public Builder jobName(String jobName) {
+ this.jobName = jobName;
+ return this;
+ }
+
+ public Builder checkpointRootPath(String checkpointRootPath) {
+ this.checkpointRootPath = checkpointRootPath;
+ return this;
+ }
+
+ public Builder sourceTasks(Integer sourceTasks) {
+ this.sourceTasks = sourceTasks;
+ return this;
+ }
+
+ public Builder alias(String alias) {
+ this.alias = alias;
+ return this;
+ }
+
+ public Builder connection(String connection) {
+ this.connection = connection;
+ return this;
+ }
+
+ public Builder priority(Integer priority) {
+ this.priority = priority;
+ return this;
+ }
+
+ public Builder sourceType(String sourceType) {
+ this.sourceType = sourceType;
+ return this;
+ }
+
+ public Builder keepFileVersion(Integer keepFileVersion) {
+ this.keepFileVersion = keepFileVersion;
+ return this;
+ }
+
+ public Builder keepCommitVersion(Integer keepCommitVersion) {
+ this.keepCommitVersion = keepCommitVersion;
+ return this;
+ }
+
+ public Builder tags(String tags) {
+ this.tags = tags;
+ return this;
+ }
+
+ public Builder zookeeperUrl(String zookeeperUrl) {
+ this.zookeeperUrl = zookeeperUrl;
+ return this;
+ }
+
+ public Builder version(Integer version) {
+ this.version = version;
+ return this;
+ }
+
+ public Builder scala(Integer scala) {
+ this.scala = scala;
+ return this;
+ }
+
+ public RowMeta build() {
+ return new RowMeta(this);
+ }
+ }
+ }
+
+ public static final class ConnectionMeta implements Serializable {
+ /**
+ * {
+ * "jdbc_url": "8eWr854+ycQhBY8X/Ye/rzxxHBoTscQvLfACZ+8AczHONHl5Vaq92DpRzLDZVNY8seyvheBnS0co0FYC4+B5pFhJrK/ilTB5d9yFY5cOtzT3eAk7n/fEAGaaA/xbP0Ei9NqhgBTsT1UgVfT9j3JkLFtbw5RvCOmOGnOVxriAzI1Q0JALMjBAu9FSxkTg8QYL619rUQEovmL1EJsHejoUPie2VsYbUYI3MAnXBU8d80k5xLjRx+PgLTrk5o1ZERT8uWebhjNFoRhVVZr1hzllFMB2BeFBadLb+yJjQ5brgrQJb7uEhtP1pfG5+Wv0+a5DuFjkLPtOwnWDVPxu9N+jKgUTxuTzWbA5M3JIsH+zNo0J55B86rpiheFv3PhZxrt4RJTZwoh27SjsePtOAYQbmNeSt1zseBYnsH2BAbBlCXU=",
+ * "jdbc_driver": "oracle.jdbc.driver.OracleDriver",
+ * "jdbc_user": "lnmuI3MYb2eYSAXa+ZpodQ==",
+ * "jdbc_password": "N/RLVdGIVRCoBX9yfU//sg=="
+ * }
+ */
+ private String url;
+ private String user;
+ private String password;
+ private String driver;
+
+ public ConnectionMeta() {
+ }
+
+ public ConnectionMeta(Builder builder) {
+ this.url = builder.url;
+ this.user = builder.user;
+ this.password = builder.password;
+ this.driver = builder.driver;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getUrl() {
+ return url;
+ }
+
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ public String getUser() {
+ return user;
+ }
+
+ public void setUser(String user) {
+ this.user = user;
+ }
+
+ public String getPassword() {
+ return password;
+ }
+
+ public void setPassword(String password) {
+ this.password = password;
+ }
+
+ public String getDriver() {
+ return driver;
+ }
+
+ public void setDriver(String driver) {
+ this.driver = driver;
+ }
+
+ @Override
+ public String toString() {
+ return "ConnectionMeta{" +
+ "url='" + url + '\'' +
+ ", user='" + user + '\'' +
+ ", password='" + password + '\'' +
+ ", driver='" + driver + '\'' +
+ '}';
+ }
+
+ public static final class Builder {
+ private String url;
+ private String user;
+ private String password;
+ private String driver;
+
+ private Builder() {
+ }
+
+ public Builder url(String url) {
+ this.url = url;
+ return this;
+ }
+
+ public Builder user(String user) {
+ this.user = user;
+ return this;
+ }
+
+ public Builder password(String password) {
+ this.password = password;
+ return this;
+ }
+
+ public Builder driver(String driver) {
+ this.driver = driver;
+ return this;
+ }
+
+ public ConnectionMeta build() {
+ return new ConnectionMeta(this);
+ }
+ }
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleJob.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleJob.java
new file mode 100644
index 0000000..aba5aa1
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleJob.java
@@ -0,0 +1,144 @@
+package com.lanyuanxiaoyao.service.common.entity.compaction;
+
+/**
+ * 压缩调度任务
+ *
+ * @author ZhangJiacheng
+ * @date 2022-09-26
+ */
+public class ScheduleJob {
+ private String id;
+ private Long flinkJobId;
+ private String alias;
+ private String batch;
+ private String status;
+ private String comment;
+
+ public ScheduleJob() {
+ }
+
+ public ScheduleJob(String id, Long flinkJobId, String alias, String batch, String status, String comment) {
+ this.id = id;
+ this.flinkJobId = flinkJobId;
+ this.alias = alias;
+ this.batch = batch;
+ this.status = status;
+ this.comment = comment;
+ }
+
+ public ScheduleJob(Builder builder) {
+ this.id = builder.id;
+ this.flinkJobId = builder.flinkJobId;
+ this.alias = builder.alias;
+ this.batch = builder.batch;
+ this.status = builder.status;
+ this.comment = builder.comment;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ public Long getFlinkJobId() {
+ return flinkJobId;
+ }
+
+ public void setFlinkJobId(Long flinkJobId) {
+ this.flinkJobId = flinkJobId;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public String getBatch() {
+ return batch;
+ }
+
+ public void setBatch(String batch) {
+ this.batch = batch;
+ }
+
+ public String getStatus() {
+ return status;
+ }
+
+ public void setStatus(String status) {
+ this.status = status;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ public void setComment(String comment) {
+ this.comment = comment;
+ }
+
+ @Override
+ public String toString() {
+ return "ScheduleJob{" +
+ "id=" + id +
+ ", flinkJobId=" + flinkJobId +
+ ", alias='" + alias + '\'' +
+ '}';
+ }
+
+ public static final class Builder {
+ private String id;
+ private Long flinkJobId;
+ private String alias;
+ private String batch;
+ private String status;
+ private String comment;
+
+ private Builder() {
+ }
+
+ public Builder id(String id) {
+ this.id = id;
+ return this;
+ }
+
+ public Builder flinkJobId(Long flinkJobId) {
+ this.flinkJobId = flinkJobId;
+ return this;
+ }
+
+ public Builder alias(String alias) {
+ this.alias = alias;
+ return this;
+ }
+
+ public Builder batch(String batch) {
+ this.batch = batch;
+ return this;
+ }
+
+ public Builder status(String status) {
+ this.status = status;
+ return this;
+ }
+
+ public Builder comment(String comment) {
+ this.comment = comment;
+ return this;
+ }
+
+ public ScheduleJob build() {
+ return new ScheduleJob(this);
+ }
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleQueue.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleQueue.java
new file mode 100644
index 0000000..11e9876
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleQueue.java
@@ -0,0 +1,15 @@
+package com.lanyuanxiaoyao.service.common.entity.compaction;
+
+import java.util.Deque;
+
+/**
+ * 队列
+ *
+ * @author ZhangJiacheng
+ * @date 2022-09-26
+ */
+public interface ScheduleQueue extends Deque {
+ Iterable poll(int limit);
+
+ Iterable pollWithoutSame(int limit);
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/CheckpointRootPathNotFoundException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/CheckpointRootPathNotFoundException.java
new file mode 100644
index 0000000..0ce7ab1
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/CheckpointRootPathNotFoundException.java
@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-05-23
+ */
+public class CheckpointRootPathNotFoundException extends RuntimeException{
+ public CheckpointRootPathNotFoundException() {
+ super("Checkpoint root path not found");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ConfigException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ConfigException.java
new file mode 100644
index 0000000..3cb21c5
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ConfigException.java
@@ -0,0 +1,29 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+import java.util.function.Supplier;
+
+/**
+ * 配置异常
+ *
+ * @author ZhangJiacheng
+ * @date 2022-05-16
+ */
+public class ConfigException extends Exception {
+ public ConfigException(String message) {
+ super(message);
+ }
+
+ public static void check(String message, Supplier checkFunction) throws ConfigException {
+ if (checkFunction.get()) {
+ throw new ConfigException(message);
+ }
+ }
+
+ public static void checkQuiet(String message, Supplier checkFunction) {
+ try {
+ check(message, checkFunction);
+ } catch (ConfigException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/FlinkJobNotFoundException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/FlinkJobNotFoundException.java
new file mode 100644
index 0000000..3885d35
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/FlinkJobNotFoundException.java
@@ -0,0 +1,15 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-05-23
+ */
+public class FlinkJobNotFoundException extends RuntimeException {
+ public FlinkJobNotFoundException() {
+ super("Flink job not found");
+ }
+
+ public FlinkJobNotFoundException(Long flinkJobId) {
+ super("Flink job " + flinkJobId + " not found");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MessageIdEmptyException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MessageIdEmptyException.java
new file mode 100644
index 0000000..0ecc2ed
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MessageIdEmptyException.java
@@ -0,0 +1,14 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+
+import com.lanyuanxiaoyao.service.common.Constants;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-05-23
+ */
+public class MessageIdEmptyException extends RuntimeException {
+ public MessageIdEmptyException() {
+ super(Constants.LOG_POINT_MESSAGE_ID_EMPTY + " Message id is empty");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MissingArgumentException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MissingArgumentException.java
new file mode 100644
index 0000000..d58f5ba
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MissingArgumentException.java
@@ -0,0 +1,13 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * 缺参数异常
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-20
+ */
+public class MissingArgumentException extends Exception {
+ public MissingArgumentException(String argumentName) {
+ super("Argument: '" + argumentName + "' is not found");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PartitionPathNotFoundException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PartitionPathNotFoundException.java
new file mode 100644
index 0000000..293cf0f
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PartitionPathNotFoundException.java
@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-05-23
+ */
+public class PartitionPathNotFoundException extends RuntimeException{
+ public PartitionPathNotFoundException() {
+ super("Partition path not found");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PulsarInfoNotFoundException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PulsarInfoNotFoundException.java
new file mode 100644
index 0000000..3256c4e
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PulsarInfoNotFoundException.java
@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2023-04-28
+ */
+public class PulsarInfoNotFoundException extends RuntimeException {
+ public PulsarInfoNotFoundException(String message) {
+ super(message);
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/SyncStateNotFoundException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/SyncStateNotFoundException.java
new file mode 100644
index 0000000..ada90c9
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/SyncStateNotFoundException.java
@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-05-23
+ */
+public class SyncStateNotFoundException extends RuntimeException{
+ public SyncStateNotFoundException() {
+ super("Sync state not found");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/TableMetaNotFoundException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/TableMetaNotFoundException.java
new file mode 100644
index 0000000..4605518
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/TableMetaNotFoundException.java
@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-05-23
+ */
+public class TableMetaNotFoundException extends RuntimeException{
+ public TableMetaNotFoundException() {
+ super("Table meta not found");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ZookeeperUrlNotFoundException.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ZookeeperUrlNotFoundException.java
new file mode 100644
index 0000000..252341c
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ZookeeperUrlNotFoundException.java
@@ -0,0 +1,11 @@
+package com.lanyuanxiaoyao.service.common.exception;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-05-23
+ */
+public class ZookeeperUrlNotFoundException extends RuntimeException{
+ public ZookeeperUrlNotFoundException() {
+ super("Zookeeper url not found");
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/FlinkJobHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/FlinkJobHelper.java
new file mode 100644
index 0000000..c46a829
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/FlinkJobHelper.java
@@ -0,0 +1,86 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Flink Job 工具类
+ *
+ * @author ZhangJiacheng
+ * @version 0.0.1
+ * @date 2021-12-08
+ */
+public class FlinkJobHelper {
+ public static String allFlinkJobSql(String database) {
+ // language=MySQL
+ return "select tafjc.id,\n" +
+ " tafjc.name,\n" +
+ " tafjc.run_mode,\n" +
+ " tayjc.job_manager_memory,\n" +
+ " tayjc.task_manager_memory\n" +
+ "from `" + database + "`.tb_app_flink_job_config tafjc\n" +
+ " left join\n" +
+ " `" + database + "`.tb_app_yarn_job_config tayjc on tafjc.one_in_one_yarn_job_id = tayjc.id and tayjc.status = 'y'\n" +
+ "where tafjc.status = 'y'";
+ }
+
+ public static String flinkJobSql(String database) {
+ // language=MySQL
+ return "select tafjc.id,\n" +
+ " tafjc.name,\n" +
+ " tafjc.run_mode,\n" +
+ " tayjc.job_manager_memory,\n" +
+ " tayjc.task_manager_memory\n" +
+ "from `" + database + "`.tb_app_flink_job_config tafjc\n" +
+ " left join\n" +
+ " `" + database + "`.tb_app_yarn_job_config tayjc on tafjc.one_in_one_yarn_job_id = tayjc.id and tayjc.status = 'y'\n" +
+ "where tafjc.id = ?\n" +
+ " and tafjc.status = 'y'";
+ }
+
+ public static List from(ResultSet rs) throws SQLException {
+ List results = new ArrayList<>();
+ while (rs.next()) {
+ String runModeText = rs.getString(3);
+ FlinkJob.RunMode mode;
+ try {
+ mode = FlinkJob.RunMode.valueOf(runModeText);
+ } catch (IllegalArgumentException e) {
+ mode = FlinkJob.RunMode.ALL_IN_ONE;
+ }
+ TableMeta.YarnMeta yarnMeta = TableMeta.YarnMeta.builder()
+ .jobManagerMemory(rs.getInt(4))
+ .taskManagerMemory(rs.getInt(5))
+ .build();
+ results.add(
+ FlinkJob.builder()
+ .id(rs.getLong(1))
+ .name(rs.getString(2))
+ .runMode(mode)
+ .oneInOneSyncYarn(yarnMeta)
+ .build()
+ );
+ }
+ return results;
+ }
+
+ public static Optional fromOne(ResultSet rs) throws SQLException {
+ List results = from(rs);
+ if (results.size() < 1) {
+ return Optional.empty();
+ } else if (results.size() > 1) {
+ throw new SQLException("Found more than 1 records");
+ } else {
+ return Optional.of(results.get(0));
+ }
+ }
+
+ public static boolean isOneInOneMode(FlinkJob job) {
+ return FlinkJob.RunMode.ONE_IN_ONE.equals(job.getRunMode());
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/LogHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/LogHelper.java
new file mode 100644
index 0000000..80b0d14
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/LogHelper.java
@@ -0,0 +1,117 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapBuilder;
+import cn.hutool.core.map.MapUtil;
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import java.util.List;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.MDC;
+
+/**
+ * 日志相关
+ *
+ * @author ZhangJiacheng
+ * @date 2023-12-25
+ */
+public class LogHelper {
+ private static String generateLog(LogPoint point, String template, Object[] args) {
+ if (ObjectUtil.isEmpty(template)) {
+ return point.toString();
+ }
+ if (ObjectUtil.isEmpty(args)) {
+ return point + " " + template;
+ }
+ Object[] items = new Object[args.length + 1];
+ items[0] = point;
+ System.arraycopy(args, 0, items, 1, args.length);
+ return StrUtil.format("{} " + template, items);
+ }
+
+ public static void info(Logger logger, LogPoint point) {
+ logger.info(generateLog(point, null, null));
+ }
+
+ public static void info(Logger logger, LogPoint point, String template) {
+ logger.info(generateLog(point, template, null));
+ }
+
+ public static void info(Logger logger, LogPoint point, String template, Object... args) {
+ logger.info(generateLog(point, template, args));
+ }
+
+ public static void debug(Logger logger, LogPoint point, String template, Object... args) {
+ logger.debug(generateLog(point, template, args));
+ }
+
+ public static void warn(Logger logger, LogPoint point, String template, Object... args) {
+ logger.warn(generateLog(point, template, args));
+ }
+
+ public static void error(Logger logger, LogPoint point, String template, Object... args) {
+ logger.error(generateLog(point, template, args));
+ }
+
+ public static void setMdc(Map mdcList) {
+ mdcList.forEach(MDC::put);
+ }
+
+ public static void setMdc(String... items) {
+ if (items.length % 2 != 0) {
+ throw new IllegalArgumentException("Items must key-value");
+ }
+ MapBuilder builder = MapUtil.builder();
+ for (int i = 0, j = 1; j < items.length; i++, j++) {
+ builder.put(items[i], items[j]);
+ }
+ setMdc(builder.build());
+ }
+
+ public static void removeMdc(List mdcList) {
+ mdcList.forEach(MDC::remove);
+ }
+
+ public static void removeMdc(String... names) {
+ removeMdc(ListUtil.of(names));
+ }
+
+ public static void setMdcFlinkJobAndAlias(Long flinkJobId, String alias) {
+ setMdc(MapUtil.builder()
+ .put(Constants.LOG_FLINK_JOB_ID_LABEL, flinkJobId.toString())
+ .put(Constants.LOG_ALIAS_LABEL, alias)
+ .build());
+ }
+
+ public static void removeMdcFlinkJobAndAlias(Long flinkJobId, String alias) {
+ removeMdc(ListUtil.of(Constants.LOG_FLINK_JOB_ID_LABEL, Constants.LOG_ALIAS_LABEL));
+ }
+
+ public enum LogPoint {
+ PULSAR_SOURCE_CHECKPOINT_INITIAL(100),
+ PULSAR_SOURCE_CHECKPOINT_INITIAL_MESSAGE_ID(101),
+
+ MESSAGE_ID_EMPTY(1),
+ CHECKPOINT_INITIAL(2),
+ CHECKPOINT_INITIAL_MESSAGE_ID(3),
+ PULSAR_SOURCE_BOOTSTRAP_MESSAGE_ID(4),
+ PULSAR_SOURCE_BOOTSTRAP_GET_MESSAGE_ERROR(5),
+ FIELD_TYPE_NOT_FOUND(6),
+ VERSION_UPDATE(7),
+ CHECKPOINT_START(8),
+ CHECKPOINT_COMPLETE(9);
+
+ private final Integer code;
+
+ LogPoint(Integer code) {
+ this.code = code;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("LOP-%06d", code);
+ }
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/MapHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/MapHelper.java
new file mode 100644
index 0000000..cee725a
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/MapHelper.java
@@ -0,0 +1,37 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+import java.util.Map;
+
+/**
+ * Map工具类
+ *
+ * @author ZhangJiacheng
+ * @date 2023-03-20
+ */
+public class MapHelper {
+ // 如果大小写同时存在，那么这个忽略大小写的方案就会首先返回大写键的值
+ public static Object getWithoutCase(Map map, String key) {
+ String upperKey = key.toUpperCase(), lowerKey = key.toLowerCase();
+ return map.containsKey(upperKey) ? map.get(upperKey) : map.containsKey(lowerKey) ? map.get(lowerKey) : map.get(key);
+ }
+
+ public static String getStringWithoutCase(Map map, String key) {
+ return (String) getWithoutCase(map, key);
+ }
+
+ public static Integer getIntWithoutCase(Map map, String key) {
+ return (Integer) getWithoutCase(map, key);
+ }
+
+ public static Long getLongWithoutCase(Map map, String key) {
+ return (Long) getWithoutCase(map, key);
+ }
+
+ public static Double getDoubleWithoutCase(Map map, String key) {
+ return (Double) getWithoutCase(map, key);
+ }
+
+ public static Float getFloatWithoutCase(Map map, String key) {
+ return (Float) getWithoutCase(map, key);
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/NameHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/NameHelper.java
new file mode 100644
index 0000000..31db70a
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/NameHelper.java
@@ -0,0 +1,103 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+
+/**
+ * 命名相关工具
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-24
+ */
+public class NameHelper {
+ public static String pulsarSubscriptionName(Long flinkJobId, String alias) {
+ return Constants.PULSAR_SUBSCRIPTION_NAME_PREFIX + "_" + flinkJobId + "_" + alias + "_20230425";
+ }
+
+ // Sync job name
+
+ public static final String SYNC_JOB_NAME_REGEX = "^Sync_(\\d+?)_(.+)$";
+
+ public static boolean isSyncJob(String name) {
+ return StrUtil.isNotBlank(name) && name.matches(SYNC_JOB_NAME_REGEX);
+ }
+
+ public static String syncJobName(Long flinkJobId, String flinkJobName) {
+ return "Sync_" + flinkJobId + "_" + flinkJobName;
+ }
+
+ public static final String COMPACTION_JOB_NAME_REGEX = "^Compaction_(\\d+?)_(.+?)$";
+
+ public static boolean isCompactionJob(String name) {
+ return StrUtil.isNotBlank(name) && name.matches(COMPACTION_JOB_NAME_REGEX);
+ }
+
+ public static String compactionJobName(Long flinkJobId, String alias) {
+ return "Compaction_" + flinkJobId + "_" + alias;
+ }
+
+ // flink job name
+
+ public static String syncFlinkName(Long flinkJobId, String flinkJobName) {
+ return flinkJobName + " (ID: " + flinkJobId + ")";
+ }
+
+ public static String syncFlinkName(Long flinkJobId, String flinkJobName, String alias) {
+ return flinkJobName + " " + alias + " (ID: " + flinkJobId + ")";
+ }
+
+ public static String compactionFlinkName(Long flinkJobId, String schema, String alias) {
+ return schema + " " + alias + " (ID: " + flinkJobId + ")";
+ }
+
+ // sync state name
+
+ public static String syncStateName(Long flinkJobId, String alias) {
+ return flinkJobId + "-" + alias;
+ }
+
+ // zk lock name
+ public static final String ZK_ROOT_PATH = "/hudi";
+ public static final String ZK_LOCK_PATH = ZK_ROOT_PATH + "/lock";
+ public static final String ZK_LAUNCHER_LOCK_PATH = ZK_LOCK_PATH + "/launcher";
+ public static final String ZK_RUNNING_LOCK_PATH = ZK_LOCK_PATH + "/running";
+
+ public static final String ZK_SYNC_SUFFIX_PATH = "/sync";
+ public static final String ZK_SYNC_LAUNCHER_LOCK_PATH = ZK_LAUNCHER_LOCK_PATH + ZK_SYNC_SUFFIX_PATH;
+ public static final String ZK_SYNC_RUNNING_LOCK_PATH = ZK_RUNNING_LOCK_PATH + ZK_SYNC_SUFFIX_PATH;
+
+ public static String syncLockName(Long flinkJobId, String alias) {
+ if (StrUtil.isNotBlank(alias)) {
+ return "sync_lock_" + flinkJobId + "_" + alias;
+ }
+ return "sync_lock_" + flinkJobId;
+ }
+
+ public static String syncLauncherLockPath(Long flinkJobId) {
+ return ZK_SYNC_LAUNCHER_LOCK_PATH + "/" + syncLockName(flinkJobId, null);
+ }
+
+ public static String syncRunningLockPath(Long flinkJobId) {
+ return syncRunningLockPath(flinkJobId, null);
+ }
+
+ public static String syncRunningLockPath(Long flinkJobId, String alias) {
+ return ZK_SYNC_RUNNING_LOCK_PATH + "/" + syncLockName(flinkJobId, alias);
+ }
+
+ public static final String ZK_COMPACTION_SUFFIX_PATH = "/compaction";
+ public static final String ZK_COMPACTION_LAUNCHER_LOCK_PATH = ZK_LAUNCHER_LOCK_PATH + ZK_COMPACTION_SUFFIX_PATH;
+ public static final String ZK_COMPACTION_RUNNING_LOCK_PATH = ZK_RUNNING_LOCK_PATH + ZK_COMPACTION_SUFFIX_PATH;
+
+ public static String compactionLockName(Long flinkJobId, String alias) {
+ return "compaction_lock_" + flinkJobId + "_" + alias;
+ }
+
+ public static String compactionLauncherLockPath(Long flinkJobId, String alias) {
+ return ZK_COMPACTION_LAUNCHER_LOCK_PATH + "/" + compactionLockName(flinkJobId, alias);
+ }
+
+ public static String compactionRunningLockPath(Long flinkJobId, String alias) {
+ return ZK_COMPACTION_RUNNING_LOCK_PATH + "/" + compactionLockName(flinkJobId, alias);
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/RecordHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/RecordHelper.java
new file mode 100644
index 0000000..53b9be3
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/RecordHelper.java
@@ -0,0 +1,127 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.Record;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Record 工具类
+ *
+ * @author ZhangJiacheng
+ * @version 0.0.1
+ * @date 2021-12-03
+ */
+public class RecordHelper {
+ public static Boolean isNotVersionUpdateRecord(String record) {
+ return !isVersionUpdateRecord(record);
+ }
+
+ public static Boolean isVersionUpdateRecord(String record) {
+ return record.contains(Constants.VERSION_UPDATE_KEY);
+ }
+
+ public static Boolean isNotVersionUpdateRecord(Record record) {
+ return !isVersionUpdateRecord(record);
+ }
+
+ public static Boolean isVersionUpdateRecord(Record record) {
+ // Record{source=Source{sourceId='versionUpdate', sourceType='null', sourcePos='null', currentTs='2022-11-15 22:17:44'}, statement=Statement{schema='crm_ivpn_cust', table='customer', opStatement='null', opType='version', op='null', opTs='2022-11-15 00:17:43', version='20220925', before=null, after=null}}
+ return Constants.VERSION_UPDATE_KEY.equals(record.getSource().getSourceId())
+ && Constants.VERSION_KEY.equals(record.getStatement().getOpType());
+ }
+
+ /**
+ * 获取当前的 Statement, 即 source 和 target 选不为空的操作
+ *
+ * @param record 消息
+ * @return 更新的字段内容
+ */
+ public static Map getCurrentStatement(Record record) {
+ Map before = record.getStatement().getBefore();
+ Map after = record.getStatement().getAfter();
+ return (after == null ? before : after.isEmpty() ? before : after);
+ }
+
+ public static Optional> getBeforeStatement(Record record) {
+ return Optional.ofNullable(record.getStatement().getBefore());
+ }
+
+ public static Optional> getAfterStatement(Record record) {
+ return Optional.ofNullable(record.getStatement().getAfter());
+ }
+
+ private static Boolean isMapEmpty(Map map) {
+ return map == null || map.isEmpty();
+ }
+
+ public static Boolean isChangeField(TableMeta meta, Record record, Function> fieldGetter) {
+ Map before = record.getStatement().getBefore();
+ Map after = record.getStatement().getAfter();
+ Optional field = fieldGetter.apply(meta);
+ if (isMapEmpty(before) || isMapEmpty(after) || !field.isPresent()) {
+ return false;
+ }
+ Object beforeField = before.getOrDefault(field.get(), null);
+ Object afterField = after.getOrDefault(field.get(), null);
+ if (beforeField == null || afterField == null) {
+ return false;
+ }
+ return !Objects.equals(String.valueOf(beforeField), String.valueOf(afterField));
+ }
+
+ public static Map addExtraMetadata(Map current, TableMeta tableMeta, Record record) {
+ String operationType = record.getStatement().getOpType();
+ return addExtraMetadata(current, tableMeta, record, Constants.DELETE.equals(operationType));
+ }
+
+ public static Map addExtraMetadata(Map current, TableMeta tableMeta, Record record, Boolean isDelete) {
+ Map newMap = new HashMap<>(current);
+ newMap.put(Constants.UNION_KEY_NAME, RecordHelper.createUnionKey(tableMeta, current));
+ newMap.put(Constants.UPDATE_TIMESTAMP_KEY_NAME, SnowFlakeHelper.next());
+ newMap.put(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME, record.getStatement().getOpTs());
+ newMap.put(Constants.HUDI_DELETE_KEY_NAME, isDelete);
+ return newMap;
+ }
+
+ public static String createUnionKey(TableMeta tableMeta, Record record) {
+ return createUnionKey(tableMeta, getCurrentStatement(record));
+ }
+
+ private static final String PRIMARY_KEY_NOT_FOUND = "Primary Key Not Found";
+
+ /**
+ * 生成基于主键和分片键的联合主键
+ *
+ * @param tableMeta 表信息
+ * @param fields 字段
+ * @return 联合主键值
+ */
+ public static String createUnionKey(TableMeta tableMeta, Map fields) {
+ if (tableMeta.getPrimaryKeys().isEmpty()) {
+ throw new RuntimeException(PRIMARY_KEY_NOT_FOUND);
+ }
+ if (Objects.isNull(fields)) {
+ throw new RuntimeException("Fields cannot be null");
+ }
+ List primaryKeys = tableMeta.getPrimaryKeys()
+ .stream()
+ .map(key -> Optional.ofNullable(fields.get(Constants.FIELD_COVERT.apply(tableMeta, key.getName())))
+ .orElseThrow(() -> new RuntimeException(PRIMARY_KEY_NOT_FOUND + " " + fields))
+ .toString())
+ .collect(Collectors.toList());
+ String primaryKey = String.join("-", primaryKeys);
+ if (tableMeta.getPartitionKeys().isEmpty()) {
+ return primaryKey;
+ } else {
+ List partitionKeys = tableMeta.getPartitionKeys()
+ .stream()
+ .map(key -> fields.get(Constants.FIELD_COVERT.apply(tableMeta, key.getName())).toString())
+ .collect(Collectors.toList());
+ String partitionKey = String.join("-", partitionKeys);
+ return primaryKey + "_" + partitionKey;
+ }
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SnowFlakeHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SnowFlakeHelper.java
new file mode 100644
index 0000000..7be6423
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SnowFlakeHelper.java
@@ -0,0 +1,72 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+/**
+ * 雪花算(pi)法(jiu)
+ *
+ * @author ZhangJiacheng
+ * @date 2020-06-05
+ */
+public class SnowFlakeHelper {
+ /**
+ * 起始的时间戳
+ */
+ private final static long START_TIMESTAMP = 1;
+
+ /**
+ * 序列号占用的位数
+ */
+ private final static long SEQUENCE_BIT = 11;
+
+ /**
+ * 序列号最大值
+ */
+ private final static long MAX_SEQUENCE_BIT = ~(-1 << SEQUENCE_BIT);
+
+ /**
+ * 时间戳值向左位移
+ */
+ private final static long TIMESTAMP_OFFSET = SEQUENCE_BIT;
+
+ /**
+ * 序列号
+ */
+ private static long sequence = 0;
+ /**
+ * 上一次时间戳
+ */
+ private static long lastTimestamp = -1;
+
+ public static synchronized long next() {
+ long currentTimestamp = nowTimestamp();
+ if (currentTimestamp < lastTimestamp) {
+ throw new RuntimeException("Clock have moved backwards.");
+ }
+
+ if (currentTimestamp == lastTimestamp) {
+ // 相同毫秒内, 序列号自增
+ sequence = (sequence + 1) & MAX_SEQUENCE_BIT;
+ // 同一毫秒的序列数已经达到最大
+ if (sequence == 0) {
+ currentTimestamp = nextTimestamp();
+ }
+ } else {
+ // 不同毫秒内, 序列号置为0
+ sequence = 0;
+ }
+
+ lastTimestamp = currentTimestamp;
+ return (currentTimestamp - START_TIMESTAMP) << TIMESTAMP_OFFSET | sequence;
+ }
+
+ private static long nextTimestamp() {
+ long milli = nowTimestamp();
+ while (milli <= lastTimestamp) {
+ milli = nowTimestamp();
+ }
+ return milli;
+ }
+
+ private static long nowTimestamp() {
+ return System.currentTimeMillis();
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SyncStateHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SyncStateHelper.java
new file mode 100644
index 0000000..3a85c5c
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SyncStateHelper.java
@@ -0,0 +1,82 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+import com.lanyuanxiaoyao.service.common.entity.SyncState;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2023-04-24
+ */
+public class SyncStateHelper {
+ public static String allSyncStateSql(String database) {
+ // language=MySQL
+ return "select info.flink_job_id,\n" +
+ " info.alias,\n" +
+ " state.message_id,\n" +
+ " state.source_start_time,\n" +
+ " state.source_checkpoint_time,\n" +
+ " state.source_publish_time,\n" +
+ " state.source_op_time,\n" +
+ " state.compaction_start_time,\n" +
+ " state.compaction_finish_time,\n" +
+ " state.compaction_application_id,\n" +
+ " state.compaction_status,\n" +
+ " state.compaction_status_time,\n" +
+ " state.compaction_latest_op_ts\n" +
+ "from " + database + ".tb_app_hudi_sync_state state,\n" +
+ " " + database + ".tb_app_collect_table_info info\n" +
+ "where state.id = concat(info.flink_job_id, '-', info.alias)\n" +
+ " and info.status = 'y'";
+ }
+
+ public static String syncStateSql(String database) {
+ // language=MySQL
+ return "select info.flink_job_id,\n" +
+ " info.alias,\n" +
+ " state.message_id,\n" +
+ " state.source_start_time,\n" +
+ " state.source_checkpoint_time,\n" +
+ " state.source_publish_time,\n" +
+ " state.source_op_time,\n" +
+ " state.compaction_start_time,\n" +
+ " state.compaction_finish_time,\n" +
+ " state.compaction_application_id,\n" +
+ " state.compaction_status,\n" +
+ " state.compaction_status_time,\n" +
+ " state.compaction_latest_op_ts\n" +
+ "from " + database + ".tb_app_hudi_sync_state state,\n" +
+ " " + database + ".tb_app_collect_table_info info\n" +
+ "where state.id = concat(info.flink_job_id, '-', info.alias)\n" +
+ " and info.flink_job_id = ?\n" +
+ " and info.alias = ?\n" +
+ " and info.status = 'y'";
+ }
+
+ public static List from(ResultSet rs) throws SQLException {
+ List results = new ArrayList<>();
+ Function dateConvertor = timestamp -> timestamp == null ? 0 : timestamp.getTime();
+ while (rs.next()) {
+ results.add(SyncState.builder()
+ .flinkJobId(rs.getLong(1))
+ .alias(rs.getString(2))
+ .messageId(rs.getString(3))
+ .sourceStartTime(dateConvertor.apply(rs.getTimestamp(4)))
+ .sourceCheckpointTime(dateConvertor.apply(rs.getTimestamp(5)))
+ .sourcePublishTime(dateConvertor.apply(rs.getTimestamp(6)))
+ .sourceOperationTime(dateConvertor.apply(rs.getTimestamp(7)))
+ .compactionStartTime(dateConvertor.apply(rs.getTimestamp(8)))
+ .compactionFinishTime(dateConvertor.apply(rs.getTimestamp(9)))
+ .compactionApplicationId(rs.getString(10))
+ .compactionStatus(rs.getString(11))
+ .compactionStatusTime(dateConvertor.apply(rs.getTimestamp(12)))
+ .compactionLatestOperationTime(dateConvertor.apply(rs.getTimestamp(13)))
+ .build());
+ }
+ return results;
+ }
+}
diff --git a/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/TableMetaHelper.java b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/TableMetaHelper.java
new file mode 100644
index 0000000..b1bb8f0
--- /dev/null
+++ b/service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/TableMetaHelper.java
@@ -0,0 +1,576 @@
+package com.lanyuanxiaoyao.service.common.utils;
+
+import cn.hutool.json.JSONObject;
+import cn.hutool.json.JSONUtil;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.exception.ConfigException;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Table Meta 工具类
+ *
+ * @author ZhangJiacheng
+ * @version 0.0.1
+ * @date 2021-12-01
+ */
+public class TableMetaHelper {
+ //private static final AES AES = new AES(Mode.CBC, Padding.NoPadding, "6fa22c779ec14b98".getBytes(), "6fa22c779ec14b98".getBytes());
+
+ public static String tableMetaSql(String database) {
+ return tableMetaSql(database, true, false);
+ }
+
+ /*
+ SqlBuilder.select(
+ DataSource.DS_NAME,
+ DataSource.SCHEMA_NAME,
+ DataSourceTable.TABLE_NAME,
+ DataSourceTable.TABLE_TYPE,
+ DataSourceTableField.FIELD_NAME,
+ DataSourceTableField.FIELD_SEQ,
+ DataSourceTableField.FIELD_TYPE,
+ DataSourceTableField.PRIMARY_KEY,
+ DataSourceTableField.PARTITION_KEY,
+ DataSourceTableField.LENGTH,
+ TbAppCollectTableInfo.TGT_DB,
+ TbAppCollectTableInfo.TGT_TABLE,
+ TbAppCollectTableInfo.TGT_TABLE_TYPE,
+ TbAppCollectTableInfo.TGT_HDFS_PATH,
+ TbAppHudiJobConfig.WRITE_TASKS,
+ TbAppHudiJobConfig.WRITE_OPERATION,
+ TbAppHudiJobConfig.WRITE_TASK_MAX_MEMORY,
+ TbAppHudiJobConfig.WRITE_BATCH_SIZE,
+ TbAppHudiJobConfig.WRITE_RATE_LIMIT,
+ TbAppCollectTableInfo.BUCKET_NUMBER,
+ TbAppHudiJobConfig.COMPACTION_STRATEGY,
+ TbAppHudiJobConfig.COMPACTION_TASKS,
+ TbAppHudiJobConfig.COMPACTION_DELTA_COMMITS,
+ TbAppHudiJobConfig.COMPACTION_DELTA_SECONDS,
+ TbAppHudiJobConfig.COMPACTION_ASYNC_ENABLED,
+ TbAppHudiJobConfig.COMPACTION_MAX_MEMORY,
+ TbAppHudiJobConfig.CONFIGS,
+ TbAppCollectTableInfo.FILTER_FIELD,
+ TbAppCollectTableInfo.FILTER_VALUES,
+ TbAppCollectTableInfo.FILTER_TYPE,
+ TbAppCollectTableInfo.SRC_TOPIC,
+ TbAppCollectTableInfo.SRC_PULSAR_ADDR,
+ Alias.of(TbAppYarnJobConfigSync.JOB_MANAGER_MEMORY, "sync_job_manager_memory"),
+ Alias.of(TbAppYarnJobConfigSync.TASK_MANAGER_MEMORY, "sync_task_manager_memory"),
+ Alias.of(TbAppYarnJobConfigCompaction.JOB_MANAGER_MEMORY, "compaction_job_manager_memory"),
+ Alias.of(TbAppYarnJobConfigCompaction.TASK_MANAGER_MEMORY, "compaction_task_manger_momory"),
+ TbAppCollectTableInfo.PARTITION_FIELD,
+ TbAppHudiSyncState.MESSAGE_ID,
+ TbAppGlobalConfig.METRIC_PUBLISH_URL,
+ TbAppGlobalConfig.METRIC_PROMETHEUS_URL,
+ TbAppGlobalConfig.METRIC_API_URL,
+ TbAppGlobalConfig.METRIC_PUBLISH_DELAY,
+ TbAppGlobalConfig.METRIC_PUBLISH_PERIOD,
+ TbAppGlobalConfig.METRIC_PUBLISH_TIMEOUT,
+ TbAppGlobalConfig.METRIC_PUBLISH_BATCH,
+ Alias.of(TbAppFlinkJobConfig.ID, "job_id"),
+ Alias.of(TbAppFlinkJobConfig.NAME, "job_name"),
+ TbAppGlobalConfig.CHECKPOINT_ROOT_PATH,
+ TbAppHudiJobConfig.SOURCE_TASKS,
+ TbAppCollectTableInfo.ALIAS,
+ DataSource.CONNECTION,
+ TbAppCollectTableInfo.PRIORITY,
+ DataSource.DS_TYPE,
+ TbAppHudiJobConfig.KEEP_FILE_VERSION,
+ TbAppHudiJobConfig.KEEP_COMMIT_VERSION,
+ TbAppCollectTableInfo.TAGS,
+ TbAppGlobalConfig.ZK_URL
+ )
+ .from(
+ DataSource._alias_,
+ DataSourceTable._alias_,
+ DataSourceTableField._alias_,
+ TbAppFlinkJobConfig._alias_,
+ TbAppHudiJobConfig._alias_,
+ TbAppYarnJobConfigSync._alias_,
+ TbAppYarnJobConfigCompaction._alias_,
+ TbAppGlobalConfig._alias_,
+ TbAppCollectTableInfo._alias_
+ )
+ .leftJoin(TbAppHudiSyncState._alias_)
+ .onEq(TbAppHudiSyncState.ID, Column.as(StrUtil.format("concat({}, '-', {})", TbAppCollectTableInfo.FLINK_JOB_ID, TbAppCollectTableInfo.ALIAS)))
+ .whereEq(DataSource.DS_ROLE, "src")
+ .andEq(DataSource.DS_STATE, "y")
+ .andEq(DataSource.RECORD_STATE, "y")
+ .andEq(DataSourceTable.DS_ID, Column.as(DataSource.DS_ID))
+ .andEq(DataSourceTable.RECORD_STATE, "y")
+ .andEq(DataSourceTableField.TABLE_ID, Column.as(DataSourceTable.TABLE_ID))
+ .andEq(DataSourceTableField.RECORD_STATE, "y")
+ .andIn(DataSource.DS_TYPE, "udal", "telepg")
+ .andEq(DataSource.DS_NAME, Column.as(TbAppCollectTableInfo.SRC_DB))
+ .andEq(DataSource.SCHEMA_NAME, Column.as(TbAppCollectTableInfo.SRC_SCHEMA))
+ .andEq(DataSourceTable.TABLE_NAME, Column.as(TbAppCollectTableInfo.SRC_TABLE))
+ .andEq(TbAppCollectTableInfo.FLINK_JOB_ID, Column.as(TbAppFlinkJobConfig.ID))
+ .andEq(TbAppCollectTableInfo.HUDI_JOB_ID, Column.as(TbAppHudiJobConfig.ID))
+ .andEq(TbAppCollectTableInfo.SYNC_YARN_JOB_ID, Column.as(TbAppYarnJobConfigSync.ID))
+ .andEq(TbAppCollectTableInfo.COMPACTION_YARN_JOB_ID, Column.as(TbAppYarnJobConfigCompaction.ID))
+ .andEq(TbAppCollectTableInfo.CONFIG_ID, Column.as(TbAppGlobalConfig.ID))
+ .andEq(TbAppFlinkJobConfig.ID, 1542097984132706304L)
+ .andEq(TbAppCollectTableInfo.ALIAS, "crm_cfguse_channel")
+ .andEq(TbAppCollectTableInfo.STATUS, "y")
+ .andEq(TbAppFlinkJobConfig.STATUS, "y")
+ .andEq(TbAppHudiJobConfig.STATUS, "y")
+ .andEq(TbAppYarnJobConfigSync.STATUS, "y")
+ .andEq(TbAppYarnJobConfigCompaction.STATUS, "y")
+ .orderBy(DataSourceTableField.FIELD_SEQ)
+ .build()
+ */
+ public static String tableMetaSql(String database, Boolean filterByFlinkJobId, Boolean filterByAlias) {
+ // language=MySQL
+ return "select dst.ds_name,\n" +
+ " dst.schema_name,\n" +
+ " dst.table_name,\n" +
+ " dst.table_type,\n" +
+ " dstf.field_name,\n" +
+ " dstf.field_seq,\n" +
+ " dstf.field_type,\n" +
+ " dstf.primary_key,\n" +
+ " dstf.partition_key,\n" +
+ " dstf.length,\n" +
+ " tacti.tgt_db,\n" +
+ " tacti.tgt_table,\n" +
+ " tacti.tgt_table_type,\n" +
+ " tacti.tgt_hdfs_path,\n" +
+ " tajhc.write_tasks,\n" +
+ " tajhc.write_operation,\n" +
+ " tajhc.write_task_max_memory,\n" +
+ " tajhc.write_batch_size,\n" +
+ " tajhc.write_rate_limit,\n" +
+ " tacti.bucket_number,\n" +
+ " tajhc.compaction_strategy,\n" +
+ " tajhc.compaction_tasks,\n" +
+ " tajhc.compaction_delta_commits,\n" +
+ " tajhc.compaction_delta_seconds,\n" +
+ " tajhc.compaction_async_enabled,\n" +
+ " tajhc.compaction_max_memory,\n" +
+ " tajhc.configs,\n" +
+ " tacti.filter_field,\n" +
+ " tacti.filter_values,\n" +
+ " tacti.filter_type,\n" +
+ " tacti.src_topic,\n" +
+ " tacti.src_pulsar_addr,\n" +
+ " tayjc_sync.job_manager_memory as sync_job_manager_memory,\n" +
+ " tayjc_sync.task_manager_memory as sync_task_manager_memory,\n" +
+ " tayjc_compaction.job_manager_memory as compaction_job_manager_memory,\n" +
+ " tayjc_compaction.task_manager_memory as compaction_task_manger_momory,\n" +
+ " tacti.partition_field,\n" +
+ " tahss.message_id,\n" +
+ " tagc.metric_publish_url,\n" +
+ " tagc.metric_prometheus_url,\n" +
+ " tagc.metric_api_url,\n" +
+ " tagc.metric_publish_delay,\n" +
+ " tagc.metric_publish_period,\n" +
+ " tagc.metric_publish_timeout,\n" +
+ " tagc.metric_publish_batch,\n" +
+ " tafjc.id as job_id,\n" +
+ " tafjc.name as job_name,\n" +
+ " tagc.checkpoint_root_path,\n" +
+ " tajhc.source_tasks,\n" +
+ " tacti.alias,\n" +
+ " dst.connection,\n" +
+ " tacti.priority,\n" +
+ " dst.ds_type,\n" +
+ " tajhc.keep_file_version,\n" +
+ " tajhc.keep_commit_version,\n" +
+ " tacti.tags,\n" +
+ " tagc.zk_url,\n" +
+ " tacti.version,\n" +
+ " dstf.scale\n" +
+ "from `" + database + "`.tb_app_collect_table_info tacti\n" +
+ " left join `" + database + "`.tb_app_hudi_sync_state tahss\n" +
+ " on tahss.id = concat(tacti.flink_job_id, '-', tacti.alias),\n" +
+ " `" + database + "`.tb_app_flink_job_config tafjc,\n" +
+ " `" + database + "`.tb_app_hudi_job_config tajhc,\n" +
+ " `" + database + "`.tb_app_yarn_job_config tayjc_sync,\n" +
+ " `" + database + "`.tb_app_yarn_job_config tayjc_compaction,\n" +
+ " `" + database + "`.tb_app_global_config tagc,\n" +
+ " `" + database + "`.tb_app_hudi_compaction_schedule tahcs,\n" +
+ " `iap-datahub`.data_source_table_field dstf,\n" +
+ " (select ds.*, dst.table_id, dst.table_name, dst.table_type\n" +
+ " from `iap-datahub`.data_source_table dst,\n" +
+ " (select ds.ds_id, ds.ds_name, ds.ds_type, ds.schema_name, ds.connection\n" +
+ " from `iap-datahub`.data_source ds\n" +
+ " where ds.ds_role = 'src'\n" +
+ " and ds.ds_state = 'y'\n" +
+ " and ds.record_state = 'y') ds\n" +
+ " where dst.ds_id = ds.ds_id\n" +
+ " and dst.record_state = 'y') dst\n" +
+ "where dstf.table_id = dst.table_id\n" +
+ " and dstf.record_state = 'y'\n" +
+ " and dst.ds_type in ('udal', 'telepg')\n" +
+ " and dst.ds_name = tacti.src_db\n" +
+ " and dst.schema_name = tacti.src_schema\n" +
+ " and dst.table_name = tacti.src_table\n" +
+ " and tacti.flink_job_id = tafjc.id\n" +
+ " and tacti.hudi_job_id = tajhc.id\n" +
+ " and tacti.sync_yarn_job_id = tayjc_sync.id\n" +
+ " and tacti.compaction_yarn_job_id = tayjc_compaction.id\n" +
+ " and tacti.config_id = tagc.id\n" +
+ " and tacti.schedule_id = tahcs.id\n" +
+ (filterByFlinkJobId ? " and tafjc.id = ?\n" : "") +
+ (filterByAlias ? " and tacti.alias = ?\n" : "") +
+ " and tacti.status = 'y'\n" +
+ " and tafjc.status = 'y'\n" +
+ " and tajhc.status = 'y'\n" +
+ " and tayjc_sync.status = 'y'\n" +
+ " and tayjc_compaction.status = 'y'\n" +
+ "order by dstf.field_seq;";
+ }
+
+ public static List from(ResultSet rs) throws SQLException {
+ List results = new ArrayList<>();
+ List metaList = new ArrayList<>();
+ while (rs.next( )) {
+ metaList.add(
+ TableMeta.RowMeta.builder()
+ .dsName(rs.getString(1))
+ .schemaName(rs.getString(2))
+ .tableName(rs.getString(3))
+ .tableType(rs.getString(4))
+ .fieldName(rs.getString(5))
+ .fieldSeq(rs.getInt(6))
+ .fieldType(rs.getString(7))
+ .primaryKey(rs.getString(8))
+ .partitionKey(rs.getString(9))
+ .length(rs.getLong(10))
+ .tgtDb(rs.getString(11))
+ .tgtTable(rs.getString(12))
+ .tgtTableType(rs.getString(13))
+ .tgtHdfsPath(rs.getString(14))
+ .writeTasks(rs.getInt(15))
+ .writeOperation(rs.getString(16))
+ .writeTaskMaxMemory(rs.getInt(17))
+ .writeBatchSize(rs.getInt(18))
+ .writeRateLimit(rs.getInt(19))
+ .bucketIndexNumber(rs.getInt(20))
+ .compactionStrategy(rs.getString(21))
+ .compactionTasks(rs.getInt(22))
+ .compactionDeltaCommits(rs.getInt(23))
+ .compactionDeltaSeconds(rs.getInt(24))
+ .compactionAsyncEnabled(rs.getString(25))
+ .compactionMaxMemory(rs.getInt(26))
+ .configs(rs.getString(27))
+ .filterField(rs.getString(28))
+ .filterValues(rs.getString(29))
+ .filterType(rs.getString(30))
+ .topic(rs.getString(31))
+ .pulsarAddress(rs.getString(32))
+ .syncJobManagerMemory(rs.getInt(33))
+ .syncTaskManagerMemory(rs.getInt(34))
+ .compactionJobManagerMemory(rs.getInt(35))
+ .compactionTaskManagerMemory(rs.getInt(36))
+ .partitionField(rs.getString(37))
+ .messageId(rs.getString(38))
+ .metricPublishUrl(rs.getString(39))
+ .metricPrometheusUrl(rs.getString(40))
+ .metricApiUrl(rs.getString(41))
+ .metricPublishDelay(rs.getInt(42))
+ .metricPublishPeriod(rs.getInt(43))
+ .metricPublishTimeout(rs.getInt(44))
+ .metricPublishBatch(rs.getInt(45))
+ .jobId(rs.getLong(46))
+ .jobName(rs.getString(47))
+ .checkpointRootPath(rs.getString(48))
+ .sourceTasks(rs.getInt(49))
+ .alias(rs.getString(50))
+ .connection(rs.getString(51))
+ .priority(rs.getInt(52))
+ .sourceType(rs.getString(53))
+ .keepFileVersion(rs.getInt(54))
+ .keepCommitVersion(rs.getInt(55))
+ .tags(rs.getString(56))
+ .zookeeperUrl(rs.getString(57))
+ .version(rs.getInt(58))
+ .scala(rs.getInt(59))
+ .build()
+ );
+ }
+ metaList.stream()
+ .collect(Collectors.groupingBy(TableMeta.RowMeta::getAlias))
+ .values()
+ .stream()
+ .flatMap(schemaRowMetas -> schemaRowMetas
+ .stream()
+ .collect(Collectors.groupingBy(TableMeta.RowMeta::getTableName))
+ .values()
+ .stream()
+ .map(tableRowMetas -> {
+ try {
+ return fromRowMetas(tableRowMetas);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }))
+ .forEach(results::add);
+ return results;
+ }
+
+ private static void checkMoreThanOne(String fieldName, Collection collection) throws ConfigException {
+ ConfigException.check(fieldName + " cannot be more than 1", () -> collection.size() > 1);
+ }
+
+ private static void checkEmpty(String fieldName, Collection collection) throws ConfigException {
+ ConfigException.check(fieldName + " cannot be empty", collection::isEmpty);
+ }
+
+ private static void checkEmptyOrMoreThanOne(String fieldName, Collection collection) throws ConfigException {
+ checkEmpty(fieldName, collection);
+ checkMoreThanOne(fieldName, collection);
+ }
+
+ public static TableMeta fromRowMetas(List metaList) throws Exception {
+ List aliasList = metaList.stream()
+ .map(TableMeta.RowMeta::getAlias)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("alias", aliasList);
+ String alias = aliasList.get(0);
+ List sourceTypeList = metaList.stream()
+ .map(TableMeta.RowMeta::getSourceType)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("source_type", sourceTypeList);
+ String sourceTypeText = sourceTypeList.get(0).toUpperCase();
+ TableMeta.SourceType sourceType;
+ try {
+ sourceType = TableMeta.SourceType.valueOf(sourceTypeText);
+ } catch (IllegalArgumentException e) {
+ throw new Exception("Cannot parse source type " + sourceTypeText);
+ }
+ List dsNames = metaList.stream()
+ .map(TableMeta.RowMeta::getDsName)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("ds_name", dsNames);
+ String dataSource = dsNames.get(0);
+ List schemaNames = metaList.stream()
+ .map(TableMeta.RowMeta::getSchemaName)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("schema_name", schemaNames);
+ String schema = schemaNames.get(0);
+ List tableNames = metaList.stream()
+ .map(TableMeta.RowMeta::getTableName)
+ .distinct()
+ .collect(Collectors.toList());
+ // 每次只能获取 1 张表的元信息
+ checkMoreThanOne("table_name", tableNames);
+ checkEmpty("table_name", tableNames);
+ String table = tableNames.get(0);
+ List tableTypes = metaList.stream()
+ .map(TableMeta.RowMeta::getTableType)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("table_type", tableTypes);
+ String type = tableTypes.get(0);
+ List filterFields = metaList.stream()
+ .map(TableMeta.RowMeta::getFilterField)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("filter_field", filterFields);
+ String filterField = filterFields.get(0);
+ List filterValueList = metaList.stream()
+ .map(TableMeta.RowMeta::getFilterValues)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("filter_values", filterValueList);
+ String filterValuesText = filterValueList.get(0);
+ List filterValues = (filterValuesText == null || filterValuesText.isEmpty())
+ ? Collections.emptyList()
+ : Arrays.asList(filterValuesText.split(","));
+ List filterTypes = metaList.stream()
+ .map(TableMeta.RowMeta::getFilterType)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("filter_field", filterFields);
+ TableMeta.FilterType filterType;
+ try {
+ filterType = TableMeta.FilterType.valueOf(filterTypes.get(0));
+ } catch (IllegalArgumentException e) {
+ filterType = TableMeta.FilterType.NONE;
+ }
+ List topics = metaList.stream()
+ .map(TableMeta.RowMeta::getTopic)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("topic", topics);
+ String topic = topics.get(0);
+ List pulsarAddresses = metaList.stream()
+ .map(TableMeta.RowMeta::getPulsarAddress)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("pulsar address", pulsarAddresses);
+ String pulsarAddress = pulsarAddresses.get(0);
+ List priorities = metaList.stream()
+ .map(TableMeta.RowMeta::getPriority)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("priority", priorities);
+ Integer priority = priorities.get(0);
+ List tagTexts = metaList.stream()
+ .map(TableMeta.RowMeta::getTags)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("tags", tagTexts);
+ String tagText = tagTexts.get(0) == null ? "" : tagTexts.get(0);
+ List tags = Arrays.asList(tagText.split(","));
+ List versions = metaList.stream()
+ .map(TableMeta.RowMeta::getVersion)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("version", versions);
+ Integer version = versions.get(0);
+
+ // 获取 Hudi 配置, 因为查出来同一张表的配置都相同, 所以直接取第一条即可
+ TableMeta.RowMeta example = metaList.get(0);
+ TableMeta.HudiMeta hudiMeta = TableMeta.HudiMeta.builder()
+ .targetDataSource(example.getTgtDb())
+ .targetTable(example.getTgtTable())
+ .targetTableType(example.getTgtTableType())
+ .targetHdfsPath(example.getTgtHdfsPath())
+ .sourceTasks(example.getSourceTasks())
+ .writeTasks(example.getWriteTasks())
+ .writeOperation(example.getWriteOperation())
+ .writeTaskMaxMemory(example.getWriteTaskMaxMemory())
+ .writeBatchSize(example.getWriteBatchSize())
+ .writeRateLimit(example.getWriteRateLimit())
+ .bucketIndexNumber(example.getBucketIndexNumber())
+ .compactionStrategy(example.getCompactionStrategy())
+ .compactionTasks(example.getCompactionTasks())
+ .compactionDeltaCommits(example.getCompactionDeltaCommits())
+ .compactionDeltaSeconds(example.getCompactionDeltaSeconds())
+ .compactionAsyncEnabled(example.getCompactionAsyncEnabled())
+ .compactionMaxMemory(example.getCompactionMaxMemory())
+ .configs(example.getConfigs())
+ .keepFileVersion(example.getKeepFileVersion())
+ .keepCommitVersion(example.getKeepCommitVersion())
+ .build();
+ TableMeta.YarnMeta syncYarnMeta = TableMeta.YarnMeta.builder()
+ .jobManagerMemory(example.getSyncJobManagerMemory())
+ .taskManagerMemory(example.getSyncTaskManagerMemory())
+ .build();
+ TableMeta.YarnMeta compactionYarnMeta = TableMeta.YarnMeta.builder()
+ .jobManagerMemory(example.getCompactionJobManagerMemory())
+ .taskManagerMemory(example.getCompactionTaskManagerMemory())
+ .build();
+ TableMeta.ConfigMeta configMeta = TableMeta.ConfigMeta.builder()
+ .messageId(example.getMessageId())
+ .metricPublishUrl(example.getMetricPublishUrl())
+ .metricPrometheusUrl(example.getMetricPrometheusUrl())
+ .metricApiUrl(example.getMetricApiUrl())
+ .metricPublishDelay(example.getMetricPublishDelay())
+ .metricPublishPeriod(example.getMetricPublishPeriod())
+ .metricPublishTimeout(example.getMetricPublishTimeout())
+ .metricPublishBatch(example.getMetricPublishBatch())
+ .checkpointRootPath(example.getCheckpointRootPath())
+ .zookeeperUrl(example.getZookeeperUrl())
+ .build();
+ TableMeta.JobMeta jobMeta = TableMeta.JobMeta.builder()
+ .id(example.getJobId())
+ .name(example.getJobName())
+ .build();
+
+ TableMeta.ConnectionMeta connectionMeta = null;
+ String connectionText = example.getConnection();
+ if (connectionText != null && !connectionText.isEmpty()) {
+ JSONObject connectionObj = JSONUtil.parseObj(connectionText);
+ connectionMeta = TableMeta.ConnectionMeta.builder()
+ .url(connectionObj.getStr("jdbc_url"))
+ .user(connectionObj.getStr("jdbc_user"))
+ .password(connectionObj.getStr("jdbc_password"))
+ .driver(connectionObj.getStr("jdbc_driver"))
+ .build();
+ }
+
+ List partitionFields = metaList.stream()
+ .map(TableMeta.RowMeta::getPartitionField)
+ .distinct()
+ .collect(Collectors.toList());
+ checkEmptyOrMoreThanOne("partition_field", filterFields);
+ String partitionField = partitionFields.get(0);
+
+ List primaryKeys = new ArrayList<>(), partitionKeys = new ArrayList<>();
+ List fieldMetaList = new ArrayList<>(metaList.size());
+ for (TableMeta.RowMeta rowMeta : metaList) {
+ boolean isPrimaryKey = "y".equals(rowMeta.getPrimaryKey());
+ boolean isPartitionKey = "y".equals(rowMeta.getPartitionKey());
+ TableMeta.FieldMeta fieldMeta = TableMeta.FieldMeta.builder()
+ .name(rowMeta.getFieldName().toUpperCase(Locale.ROOT))
+ .sequence(rowMeta.getFieldSeq())
+ .type(rowMeta.getFieldType())
+ .isPrimaryKey(isPrimaryKey)
+ .partitionKey(isPartitionKey)
+ .length(rowMeta.getLength())
+ .scala(rowMeta.getScala())
+ .build();
+ if (isPrimaryKey) {
+ primaryKeys.add(fieldMeta);
+ }
+ if (isPartitionKey) {
+ partitionKeys.add(fieldMeta);
+ }
+ fieldMetaList.add(fieldMeta);
+ }
+ return TableMeta.builder()
+ .alias(alias)
+ .source(dataSource)
+ .schema(schema)
+ .table(table)
+ .type(type)
+ .primaryKeys(primaryKeys)
+ .partitionKeys(partitionKeys)
+ .hudi(hudiMeta)
+ .fields(fieldMetaList)
+ .filterField(filterField)
+ .filterValues(filterValues)
+ .filterType(filterType)
+ .topic(topic)
+ .pulsarAddress(pulsarAddress)
+ .syncYarn(syncYarnMeta)
+ .compactionYarn(compactionYarnMeta)
+ .partitionField(partitionField)
+ .config(configMeta)
+ .job(jobMeta)
+ .connection(connectionMeta)
+ .priority(priority)
+ .sourceType(sourceType)
+ .tags(tags)
+ .version(version)
+ .build();
+ }
+
+ public static Optional getPartitionField(TableMeta meta) {
+ if (meta.getPartitionField() == null || "".equals(meta.getPartitionField())) {
+ return Optional.empty();
+ } else {
+ return meta.getFields()
+ .stream()
+ .map(TableMeta.FieldMeta::getName)
+ .filter(name -> meta.getPartitionField().equalsIgnoreCase(name))
+ .findFirst();
+ }
+ }
+
+ public static Optional getFilterField(TableMeta meta) {
+ if (meta.getFilterField() == null || "".equals(meta.getFilterField())) {
+ return Optional.empty();
+ } else {
+ return meta.getFields()
+ .stream()
+ .map(TableMeta.FieldMeta::getName)
+ .filter(name -> meta.getFilterField().equalsIgnoreCase(name))
+ .findFirst();
+ }
+ }
+
+ public static boolean existsTag(TableMeta meta, String tag) {
+ return meta.getTags() != null && meta.getTags().contains(tag);
+ }
+}
diff --git a/utils/executor/pom.xml b/utils/executor/pom.xml
new file mode 100644
index 0000000..1a9d379
--- /dev/null
+++ b/utils/executor/pom.xml
@@ -0,0 +1,161 @@
+
+
+
+ com.lanyuanxiaoyao
+ hudi-service
+ 1.0.0-SNAPSHOT
+
+ 4.0.0
+
+ executor
+
+
+
+ org.apache.flink
+ flink-core
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-java
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-runtime_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-runtime-web_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-optimizer_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-clients_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-streaming-java_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-streaming-scala_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-metrics-core
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-metrics-prometheus_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-container_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-yarn_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.hadoop
+ *
+
+
+
+
+ org.apache.flink
+ flink-hadoop-fs
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-table-uber_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-table-uber-blink_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-state-processor-api_${scala.major.version}
+ ${flink.version}
+
+
+ org.apache.flink
+ flink-shaded-hadoop-3-uber
+ 3.1.1.7.2.9.0-173-9.0
+
+
+ cn.hutool
+ hutool-all
+
+
+ org.apache.parquet
+ parquet-format
+ 2.4.0
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.3.0
+
+ false
+ true
+
+
+ com.eshore.odcp.hudi.connector.utils.executor.Runner
+
+
+ reference.conf
+
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+ log4j-surefire*.properties
+
+
+
+
+
+
+ package
+
+ shade
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/Runner.java b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/Runner.java
new file mode 100644
index 0000000..a9e6854
--- /dev/null
+++ b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/Runner.java
@@ -0,0 +1,43 @@
+package com.lanyuanxiaoyao.service.executor;
+
+import org.apache.flink.client.deployment.ClusterClientFactory;
+import org.apache.flink.client.deployment.ClusterDescriptor;
+import org.apache.flink.client.deployment.ClusterSpecification;
+import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader;
+import org.apache.flink.client.deployment.application.ApplicationConfiguration;
+import org.apache.flink.client.program.ClusterClient;
+import org.apache.flink.client.program.ClusterClientProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.security.SecurityConfiguration;
+import org.apache.flink.runtime.security.SecurityUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+/**
+ * 启动类 #
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-01
+ */
+public class Runner {
+ public static ApplicationId run(Configuration inputConfiguration, String className, String[] args) throws Exception {
+ Configuration configuration = new Configuration(inputConfiguration);
+ SecurityUtils.install(new SecurityConfiguration(configuration));
+ return SecurityUtils.getInstalledContext()
+ .runSecured(() -> {
+ DefaultClusterClientServiceLoader yarnServiceLoader = new DefaultClusterClientServiceLoader();
+ ApplicationConfiguration applicationConfiguration = new ApplicationConfiguration(args, className);
+ //ApplicationDeployer deployer = new ApplicationClusterDeployer(new DefaultClusterClientServiceLoader());
+ //deployer.run(configuration, applicationConfiguration);
+ final ClusterClientFactory clientFactory = yarnServiceLoader.getClusterClientFactory(configuration);
+ try (final ClusterDescriptor clusterDescriptor = clientFactory.createClusterDescriptor(configuration)) {
+ final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(configuration);
+ ClusterClientProvider provider = clusterDescriptor.deployApplicationCluster(clusterSpecification, applicationConfiguration);
+ ClusterClient clusterClient = provider.getClusterClient();
+ if (clusterClient == null) {
+ return null;
+ }
+ return clusterClient.getClusterId();
+ }
+ });
+ }
+}
diff --git a/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsOptions.java b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsOptions.java
new file mode 100644
index 0000000..23fc4ab
--- /dev/null
+++ b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsOptions.java
@@ -0,0 +1,48 @@
+package com.lanyuanxiaoyao.service.executor.metrics;
+
+import org.apache.flink.annotation.docs.Documentation;
+import org.apache.flink.configuration.ConfigOption;
+
+import static org.apache.flink.configuration.ConfigOptions.key;
+
+/**
+ * Config options for the {@link VictoriaMetricsReporter}.
+ */
+@Documentation.SuffixOption
+public class VictoriaMetricsOptions {
+ public static final ConfigOption ENDPOINT =
+ key("endpoint")
+ .stringType()
+ .noDefaultValue()
+ .withDescription("Victoria metrics endpoint. eg: http://localhost:8428/api/v1/import/prometheus");
+
+ public static final ConfigOption TIMEOUT =
+ key("timeout")
+ .intType()
+ .defaultValue(60000)
+ .withDescription("Http push timeout. Default 1 minute");
+
+ public static final ConfigOption TAGS =
+ key("tags")
+ .stringType()
+ .defaultValue("")
+ .withDescription("Extra tags for every metric");
+
+ public static final ConfigOption ENABLE_AUTH =
+ key("enable.auth")
+ .booleanType()
+ .defaultValue(false)
+ .withDescription("Enable metric server http basic auth");
+
+ public static final ConfigOption AUTH_USERNAME =
+ key("auth.username")
+ .stringType()
+ .defaultValue("")
+ .withDescription("Basic auth username");
+
+ public static final ConfigOption AUTH_PASSWORD =
+ key("auth.password")
+ .stringType()
+ .defaultValue("")
+ .withDescription("Basic auth password");
+}
diff --git a/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporter.java b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporter.java
new file mode 100644
index 0000000..ee9b2a0
--- /dev/null
+++ b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporter.java
@@ -0,0 +1,64 @@
+package com.lanyuanxiaoyao.service.executor.metrics;
+
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpRequest;
+import cn.hutool.http.HttpResponse;
+import cn.hutool.http.HttpUtil;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.exporter.common.TextFormat;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.stream.Collectors;
+import org.apache.flink.metrics.prometheus.AbstractPrometheusReporter;
+import org.apache.flink.metrics.reporter.Scheduled;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-06-22
+ */
+public class VictoriaMetricsReporter extends AbstractPrometheusReporter implements Scheduled {
+ private static final Logger logger = LoggerFactory.getLogger(VictoriaMetricsReporter.class);
+
+ private final String endpoint;
+ private final Integer timout;
+ private final Map tags;
+ private final Boolean enableBasicAuth;
+ private final String basicAuthUsername;
+ private final String basicAuthPassword;
+
+ public VictoriaMetricsReporter(String endpoint, Integer timout, Map tags, Boolean enableBasicAuth, String basicAuthUsername, String basicAuthPassword) {
+ this.endpoint = endpoint;
+ this.timout = timout;
+ this.tags = tags;
+ this.enableBasicAuth = enableBasicAuth;
+ this.basicAuthUsername = basicAuthUsername;
+ this.basicAuthPassword = basicAuthPassword;
+ }
+
+ @Override
+ public void report() {
+ try (StringWriter writer = new StringWriter()) {
+ TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples());
+
+ String query = tags.entrySet()
+ .stream()
+ .map(entry -> StrUtil.format("extra_label={}={}", entry.getKey(), entry.getValue()))
+ .collect(Collectors.joining("&"));
+ HttpRequest request = HttpUtil.createPost(StrUtil.format("{}?{}", endpoint, query))
+ .body(writer.toString())
+ .timeout(timout);
+ if (enableBasicAuth) {
+ request.basicAuth(basicAuthUsername, basicAuthPassword);
+ }
+ HttpResponse response = request.execute();
+ if (!response.isOk()) {
+ logger.warn("Fail to push metrics: {}, {}, endpoint: {}, tags: {}", response.getStatus(), response.body(), endpoint, tags);
+ }
+ } catch (IOException e) {
+ logger.error("Fail to write metrics, endpoint: {}, tags: {}, exception: {}", endpoint, tags, e);
+ }
+ }
+}
diff --git a/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporterFactory.java b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporterFactory.java
new file mode 100644
index 0000000..4a24be6
--- /dev/null
+++ b/utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporterFactory.java
@@ -0,0 +1,44 @@
+package com.lanyuanxiaoyao.service.executor.metrics;
+
+import cn.hutool.core.util.StrUtil;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.flink.metrics.MetricConfig;
+import org.apache.flink.metrics.reporter.InterceptInstantiationViaReflection;
+import org.apache.flink.metrics.reporter.MetricReporterFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.lanyuanxiaoyao.service.executor.metrics.VictoriaMetricsOptions.*;
+
+/**
+ * @author ZhangJiacheng
+ * @date 2022-06-22
+ */
+@InterceptInstantiationViaReflection(
+ reporterClassName = "com.eshore.odcp.hudi.connector.utils.executor.metrics.VictoriaMetricsReporter")
+public class VictoriaMetricsReporterFactory implements MetricReporterFactory {
+ private static final Logger logger = LoggerFactory.getLogger(VictoriaMetricsReporterFactory.class);
+
+ @Override
+ public VictoriaMetricsReporter createMetricReporter(Properties properties) {
+ MetricConfig metricConfig = (MetricConfig) properties;
+ String endpoint = metricConfig.getString(ENDPOINT.key(), ENDPOINT.defaultValue());
+ int timeout = metricConfig.getInteger(TIMEOUT.key(), TIMEOUT.defaultValue());
+ String tagsText = metricConfig.getString(TAGS.key(), TAGS.defaultValue());
+ Boolean enableAuth = metricConfig.getBoolean(ENABLE_AUTH.key(), ENABLE_AUTH.defaultValue());
+ String authUsername = metricConfig.getString(AUTH_USERNAME.key(), AUTH_USERNAME.defaultValue());
+ String authPassword = metricConfig.getString(AUTH_PASSWORD.key(), AUTH_PASSWORD.defaultValue());
+
+ Map tags = new HashMap<>(10);
+ if (StrUtil.isNotBlank(tagsText)) {
+ for (String item : tagsText.split(";")) {
+ String[] parsed = item.split("=");
+ tags.put(parsed[0], parsed[1]);
+ }
+ }
+ logger.info("Create victoria metric reporter for endpoint {} timeout: {}, tags: {}, enable_auth: {}, auth_username: {}, auth_password: {}", endpoint, timeout, tags, enableAuth, authUsername, authPassword);
+ return new VictoriaMetricsReporter(endpoint, timeout, tags, enableAuth, authUsername, authPassword);
+ }
+}
diff --git a/utils/executor/src/main/resources/log4j.properties b/utils/executor/src/main/resources/log4j.properties
new file mode 100644
index 0000000..859064b
--- /dev/null
+++ b/utils/executor/src/main/resources/log4j.properties
@@ -0,0 +1,54 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds.
+monitorInterval=30
+
+# This affects logging for both user code and Flink
+rootLogger.level = INFO
+rootLogger.appenderRef.file.ref = MainAppender
+
+# Uncomment this if you want to _only_ change Flink's logging
+#logger.flink.name = org.apache.flink
+#logger.flink.level = INFO
+
+# The following lines keep the log level of common libraries/connectors on
+# log level INFO. The root logger does not override this. You have to manually
+# change the log levels here.
+logger.akka.name = akka
+logger.akka.level = INFO
+logger.kafka.name= org.apache.kafka
+logger.kafka.level = INFO
+logger.hadoop.name = org.apache.hadoop
+logger.hadoop.level = INFO
+logger.zookeeper.name = org.apache.zookeeper
+logger.zookeeper.level = INFO
+logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3
+logger.shaded_zookeeper.level = INFO
+logger.hudi.name=org.apache.hudi
+logger.hudi.level=INFO
+
+# Log all infos in the given file
+appender.main.name = MainAppender
+appender.main.type = Console
+appender.main.layout.type = PatternLayout
+appender.main.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+# Suppress the irrelevant (wrong) warnings from the Netty channel handler
+logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline
+logger.netty.level = OFF
diff --git a/utils/executor/src/main/resources/log4j2.properties b/utils/executor/src/main/resources/log4j2.properties
new file mode 100644
index 0000000..7daba93
--- /dev/null
+++ b/utils/executor/src/main/resources/log4j2.properties
@@ -0,0 +1 @@
+log4j.rootLogger=INFO
\ No newline at end of file
diff --git a/utils/executor/src/main/resources/services/org.apache.flink.client.deployment.ClusterClientFactory b/utils/executor/src/main/resources/services/org.apache.flink.client.deployment.ClusterClientFactory
new file mode 100644
index 0000000..ea1c4de
--- /dev/null
+++ b/utils/executor/src/main/resources/services/org.apache.flink.client.deployment.ClusterClientFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.yarn.YarnClusterClientFactory
\ No newline at end of file
diff --git a/utils/executor/src/main/resources/services/org.apache.flink.core.execution.PipelineExecutorFactory b/utils/executor/src/main/resources/services/org.apache.flink.core.execution.PipelineExecutorFactory
new file mode 100644
index 0000000..d56f8c5
--- /dev/null
+++ b/utils/executor/src/main/resources/services/org.apache.flink.core.execution.PipelineExecutorFactory
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.yarn.executors.YarnJobClusterExecutorFactory
+org.apache.flink.yarn.executors.YarnSessionClusterExecutorFactory
\ No newline at end of file
diff --git a/utils/executor/src/main/resources/services/org.apache.flink.metrics.reporter.MetricReporterFactory b/utils/executor/src/main/resources/services/org.apache.flink.metrics.reporter.MetricReporterFactory
new file mode 100644
index 0000000..e16bdb6
--- /dev/null
+++ b/utils/executor/src/main/resources/services/org.apache.flink.metrics.reporter.MetricReporterFactory
@@ -0,0 +1 @@
+com.eshore.odcp.hudi.connector.utils.executor.metrics.VictoriaMetricsReporterFactory
\ No newline at end of file
diff --git a/utils/executor/src/main/resources/services/org.apache.flink.table.factories.TableFactory b/utils/executor/src/main/resources/services/org.apache.flink.table.factories.TableFactory
new file mode 100644
index 0000000..d8c15c5
--- /dev/null
+++ b/utils/executor/src/main/resources/services/org.apache.flink.table.factories.TableFactory
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.flink.table.planner.delegation.BlinkPlannerFactory
+org.apache.flink.table.planner.delegation.BlinkExecutorFactory
+org.apache.flink.table.planner.delegation.DefaultParserFactory
diff --git a/utils/sync/pom.xml b/utils/sync/pom.xml
new file mode 100644
index 0000000..57d4483
--- /dev/null
+++ b/utils/sync/pom.xml
@@ -0,0 +1,158 @@
+
+
+
+ com.lanyuanxiaoyao
+ hudi-service
+ 1.0.0-SNAPSHOT
+
+ 4.0.0
+
+ sync
+
+
+ 1.10.1
+
+
+
+
+ com.lanyuanxiaoyao
+ service-common
+ 1.0.0-SNAPSHOT
+
+
+ org.apache.hudi
+ hudi-flink${flink.major.version}-bundle
+ ${hudi.version}
+ provided
+
+
+ org.apache.flink
+ flink-java
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-streaming-java_${scala.major.version}
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-clients_${scala.major.version}
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-table-common
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-table-runtime-blink_${scala.major.version}
+ ${flink.version}
+ provided
+
+
+ org.apache.flink
+ flink-table-planner-blink_${scala.major.version}
+ ${flink.version}
+ provided
+
+
+ org.apache.pulsar
+ pulsar-client
+ provided
+
+
+ cn.hutool
+ hutool-all
+ provided
+
+
+ org.apache.flink
+ flink-shaded-guava
+ 30.1.1-jre-15.0
+ provided
+
+
+ org.apache.hadoop
+ hadoop-client
+ 3.1.2
+ provided
+
+
+ dev.failsafe
+ failsafe
+ 3.2.4
+ provided
+
+
+ pl.tkowalcz.tjahzi
+ logback-appender
+ 0.9.23
+ provided
+
+
+ org.slf4j
+ log4j-over-slf4j
+ 1.7.15
+
+
+ org.slf4j
+ jcl-over-slf4j
+ 1.7.15
+
+
+ org.junit.jupiter
+ junit-jupiter
+ 5.7.2
+ test
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-shade-plugin
+ 3.3.0
+
+ false
+ true
+
+
+ reference.conf
+
+
+
+
+
+ *:*
+
+ META-INF/*.SF
+ META-INF/*.DSA
+ META-INF/*.RSA
+ log4j-surefire*.properties
+
+
+
+
+
+
+ package
+
+ shade
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/Compactor.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/Compactor.java
new file mode 100644
index 0000000..fcabc1c
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/Compactor.java
@@ -0,0 +1,263 @@
+package com.lanyuanxiaoyao.service.sync;
+
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.RunMeta;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.utils.NameHelper;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.functions.CompactionEventHandler;
+import com.lanyuanxiaoyao.service.sync.utils.*;
+import java.io.IOException;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.operators.ProcessOperator;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.client.HoodieFlinkWriteClient;
+import org.apache.hudi.common.model.HoodieLogFile;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.TableSchemaResolver;
+import org.apache.hudi.common.table.log.HoodieLogFormat;
+import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock;
+import org.apache.hudi.common.table.log.block.HoodieDeleteBlock;
+import org.apache.hudi.common.table.log.block.HoodieLogBlock;
+import org.apache.hudi.common.table.log.block.HoodieParquetDataBlock;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.util.ClosableIterator;
+import org.apache.hudi.common.util.CompactionUtils;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.configuration.FlinkOptions;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.org.apache.avro.Schema;
+import org.apache.hudi.org.apache.avro.generic.IndexedRecord;
+import org.apache.hudi.sink.compact.*;
+import org.apache.hudi.sink.compact.strategy.CompactionPlanStrategies;
+import org.apache.hudi.sink.compact.strategy.CompactionPlanStrategy;
+import org.apache.hudi.table.HoodieFlinkTable;
+import org.apache.hudi.util.CompactionUtil;
+import org.apache.hudi.util.StreamerUtil;
+import org.apache.parquet.avro.AvroSchemaConverter;
+import org.apache.parquet.schema.MessageType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 单表离线压缩
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-21
+ */
+public class Compactor {
+ private static final Logger logger = LoggerFactory.getLogger(Compactor.class);
+ private static final ObjectMapper mapper = JacksonUtils.getMapper();
+
+ public static void main(String[] args) throws Exception {
+ FlinkJob flinkJob = ArgumentsUtils.getFlinkJob(args);
+ TableMeta tableMeta = ArgumentsUtils.getTableMeta(args);
+ String selectedInstants = ArgumentsUtils.getInstants(args);
+ String cluster = ArgumentsUtils.getCluster(args);
+
+ logger.info("Bootstrap flink job: {}", mapper.writeValueAsString(flinkJob));
+ logger.info("Bootstrap table meta: {}", mapper.writeValueAsString(tableMeta));
+ logger.info("Bootstrap instants: {}", selectedInstants);
+ logger.info("Bootstrap cluster: {}", cluster);
+
+ String applicationId = System.getenv("_APP_ID");
+
+ RunMeta runMeta = new RunMeta(cluster, flinkJob.getId(), tableMeta.getAlias());
+ logger.info("Run meta: {}", runMeta);
+ ZkUtils.createCompactionLock(flinkJob, tableMeta, tableMeta.getConfig().getZookeeperUrl(), mapper.writeValueAsString(runMeta));
+ logger.info("Lock for {} {} success", flinkJob.getId(), tableMeta.getAlias());
+
+ StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ FlinkCompactionConfig config = new FlinkCompactionConfig();
+ if (ObjectUtil.isEmpty(selectedInstants)) {
+ config.compactionPlanSelectStrategy = CompactionPlanStrategy.ALL;
+ } else {
+ config.compactionPlanSelectStrategy = CompactionPlanStrategy.INSTANTS;
+ config.compactionPlanInstant = selectedInstants;
+ }
+
+ GlobalConfiguration globalConfiguration = new GlobalConfiguration(cluster, applicationId, tableMeta);
+ Configuration configuration = SyncUtils.getCompactionFlinkConfiguration(
+ globalConfiguration,
+ new Configuration(),
+ flinkJob,
+ tableMeta,
+ SyncUtils.avroSchemaWithExtraFields(tableMeta),
+ 1
+ );
+
+ CompactionEventHandler eventHandler = new CompactionEventHandler(globalConfiguration, flinkJob, tableMeta);
+
+ HoodieFlinkWriteClient writeClient = StreamerUtil.createWriteClient(configuration);
+ HoodieFlinkTable table = writeClient.getHoodieTable();
+
+ table.getMetaClient().reloadActiveTimeline();
+
+ StatusUtils.compactionStart(globalConfiguration, flinkJob, tableMeta);
+
+ // 检查时间线当前状态
+ logger.info("{} timeline detail ({})", tableMeta.getAlias(), LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
+ table.getActiveTimeline()
+ .getInstants()
+ .forEach(instant -> logger.info("{} {} {}", instant.getTimestamp(), instant.getAction(), instant.getState()));
+
+ HoodieTimeline pendingCompactionTimeline = table.getActiveTimeline().filterPendingCompactionTimeline();
+ List requested = CompactionPlanStrategies.getStrategy(config).select(pendingCompactionTimeline);
+ if (requested.isEmpty()) {
+ logger.info("No compaction plan scheduled");
+ eventHandler.closed("No compaction plan scheduled", null);
+ return;
+ }
+
+ List compactionInstantTimes = requested.stream().map(HoodieInstant::getTimestamp).collect(Collectors.toList());
+ compactionInstantTimes.forEach(timestamp -> {
+ HoodieInstant inflightInstant = HoodieTimeline.getCompactionInflightInstant(timestamp);
+ if (pendingCompactionTimeline.containsInstant(inflightInstant)) {
+ logger.info("Rollback inflight compaction instant: [" + timestamp + "]");
+ table.rollbackInflightCompaction(inflightInstant);
+ table.getMetaClient().reloadActiveTimeline();
+ }
+ });
+
+ List> compactionPlans = compactionInstantTimes.stream()
+ .map(timestamp -> {
+ try {
+ return Pair.of(timestamp, CompactionUtils.getCompactionPlan(table.getMetaClient(), timestamp));
+ } catch (IOException e) {
+ throw new HoodieException("Get compaction plan at instant " + timestamp + " error", e);
+ }
+ })
+ .filter(pair -> {
+ HoodieCompactionPlan plan = pair.getRight();
+ return plan != null && plan.getOperations() != null && plan.getOperations().size() > 0;
+ })
+ .collect(Collectors.toList());
+
+ if (compactionPlans.isEmpty()) {
+ logger.info("No compaction plan for instant " + String.join(",", compactionInstantTimes));
+ eventHandler.closed("No compaction plan for instant " + String.join(",", compactionInstantTimes), null);
+ return;
+ }
+
+ List instants = compactionInstantTimes.stream().map(HoodieTimeline::getCompactionRequestedInstant).collect(Collectors.toList());
+
+ logger.info("Start to compaction for instant " + compactionInstantTimes);
+
+ for (HoodieInstant instant : instants) {
+ if (!pendingCompactionTimeline.containsInstant(instant)) {
+ CompactionUtil.cleanInstant(table.getMetaClient(), instant);
+ }
+ table.getActiveTimeline().transitionCompactionRequestedToInflight(instant);
+ }
+ table.getMetaClient().reloadActiveTimeline();
+
+ try {
+ compactionPlans.forEach(pair -> preCommit(globalConfiguration, flinkJob, tableMeta, table, pair.getLeft(), pair.getRight()));
+ } catch (Throwable e) {
+ logger.warn("Cannot submit pre-commit log");
+ }
+
+ environment.addSource(new CompactionPlanSourceFunction(compactionPlans))
+ .name("compaction_source")
+ .uid("uid_compaction_source")
+ .rebalance()
+ .transform("compact_task",
+ TypeInformation.of(CompactionCommitEvent.class),
+ new ProcessOperator<>(new CompactFunction(configuration)))
+ .setParallelism(configuration.getInteger(FlinkOptions.COMPACTION_TASKS))
+ .addSink(new CompactionCommitSink(configuration, eventHandler))
+ .name("compaction_commit")
+ .uid("uid_compaction_commit")
+ .setParallelism(1);
+
+ environment.execute(NameHelper.compactionFlinkName(flinkJob.getId(), tableMeta.getSchema(), tableMeta.getAlias()));
+ }
+
+ private static void preCommit(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, HoodieFlinkTable table, String instant, HoodieCompactionPlan compactionPlan) {
+ HoodieTableMetaClient client = table.getMetaClient();
+ FileSystem fileSystem = client.getRawFs();
+ List deltaFilePaths = compactionPlan.getOperations()
+ .stream()
+ .flatMap(operation -> operation.getDeltaFilePaths().stream().map(path -> StrUtil.format("{}/{}", operation.getPartitionPath(), path)))
+ .map(path -> new Path(StrUtil.format("{}/{}", client.getBasePathV2(), path)))
+ .collect(Collectors.toList());
+ AtomicLong totalLogFilesCompacted = new AtomicLong(0);
+ AtomicLong totalLogFilesSize = new AtomicLong(0);
+ AtomicLong totalRecordsDeleted = new AtomicLong(0);
+ AtomicLong totalCompactedRecordsUpdated = new AtomicLong(0);
+ deltaFilePaths.parallelStream().forEach(path -> {
+ try {
+ FileStatus fileStatus = fileSystem.getFileStatus(path);
+ totalLogFilesCompacted.incrementAndGet();
+ totalLogFilesSize.addAndGet(fileStatus.getLen());
+ MessageType messageType = TableSchemaResolver.readSchemaFromLogFile(fileSystem, path);
+ if (ObjectUtil.isNull(messageType)) {
+ return;
+ }
+ Schema writerSchema = new AvroSchemaConverter().convert(messageType);
+ try (HoodieLogFormat.Reader reader = HoodieLogFormat.newReader(fileSystem, new HoodieLogFile(path), writerSchema)) {
+ while (reader.hasNext()) {
+ HoodieLogBlock block = reader.next();
+ switch (block.getBlockType()) {
+ case AVRO_DATA_BLOCK:
+ HoodieAvroDataBlock avroDataBlock = (HoodieAvroDataBlock) block;
+ try (ClosableIterator avroDataBlockRecordIterator = avroDataBlock.getRecordIterator()) {
+ while (avroDataBlockRecordIterator.hasNext()) {
+ // logger.info("totalCompactedRecordsUpdated {}", totalCompactedRecordsUpdated.incrementAndGet());
+ totalCompactedRecordsUpdated.incrementAndGet();
+ avroDataBlockRecordIterator.next();
+ }
+ }
+ break;
+ case PARQUET_DATA_BLOCK:
+ HoodieParquetDataBlock parquetDataBlock = (HoodieParquetDataBlock) block;
+ try (ClosableIterator parquetDataBlockRecordIterator = parquetDataBlock.getRecordIterator()) {
+ while (parquetDataBlockRecordIterator.hasNext()) {
+ // logger.info("totalCompactedRecordsUpdated {}", totalCompactedRecordsUpdated.incrementAndGet());
+ totalCompactedRecordsUpdated.incrementAndGet();
+ parquetDataBlockRecordIterator.next();
+ }
+ }
+ break;
+ case DELETE_BLOCK:
+ HoodieDeleteBlock deleteBlock = (HoodieDeleteBlock) block;
+ // logger.info("totalRecordsDeleted {}", totalRecordsDeleted.addAndGet(deleteBlock.getRecordsToDelete().length));
+ totalRecordsDeleted.addAndGet(deleteBlock.getRecordsToDelete().length);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ } catch (Exception e) {
+ logger.warn("Parse log file failure for " + path, e);
+ }
+ });
+
+ Map metadata = new ConcurrentHashMap<>(5);
+ metadata.put("totalLogFilesCompacted", totalLogFilesCompacted.get());
+ metadata.put("totalLogFilesSize", totalLogFilesSize.get());
+ metadata.put("totalRecordsDeleted", totalRecordsDeleted.get());
+ metadata.put("totalCompactedRecordsUpdated", totalCompactedRecordsUpdated.get());
+ metadata.put("totalLogRecordsCompacted", totalRecordsDeleted.get() + totalCompactedRecordsUpdated.get());
+
+ StatusUtils.compactionPreCommit(configuration, flinkJob, tableMeta, instant, metadata);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/Synchronizer.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/Synchronizer.java
new file mode 100644
index 0000000..c5acfac
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/Synchronizer.java
@@ -0,0 +1,170 @@
+package com.lanyuanxiaoyao.service.sync;
+
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.Record;
+import com.lanyuanxiaoyao.service.common.entity.RunMeta;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.exception.CheckpointRootPathNotFoundException;
+import com.lanyuanxiaoyao.service.common.exception.ZookeeperUrlNotFoundException;
+import com.lanyuanxiaoyao.service.common.utils.NameHelper;
+import com.lanyuanxiaoyao.service.common.utils.TableMetaHelper;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.functions.PulsarMessage2RecordFunction;
+import com.lanyuanxiaoyao.service.sync.functions.PulsarMessageSourceReader;
+import com.lanyuanxiaoyao.service.sync.functions.ValidateRecordFilter;
+import com.lanyuanxiaoyao.service.sync.utils.*;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import org.apache.flink.api.common.restartstrategy.RestartStrategies;
+import org.apache.flink.api.common.serialization.SimpleStringEncoder;
+import org.apache.flink.core.fs.Path;
+import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.flink.streaming.api.CheckpointingMode;
+import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.streaming.api.functions.sink.filesystem.StreamingFileSink;
+import org.apache.flink.streaming.api.functions.sink.filesystem.rollingpolicies.DefaultRollingPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.lanyuanxiaoyao.service.common.Constants.*;
+
+/**
+ * 同步应用
+ *
+ * @author lanyuanxiaoyao
+ * @version 0.0.1
+ * @date 2021-11-26
+ */
+public class Synchronizer {
+ private static final Logger logger = LoggerFactory.getLogger(Synchronizer.class);
+ private static final ObjectMapper mapper = JacksonUtils.getMapper();
+
+ private static String findConfigFromList(List metas, Function getter, Supplier notFoundException) throws Exception {
+ return metas.stream()
+ .map(getter)
+ .distinct()
+ .findFirst()
+ .orElseThrow(notFoundException);
+ }
+
+ public static void main(String[] args) throws Exception {
+ FlinkJob flinkJob = ArgumentsUtils.getFlinkJob(args);
+ List tableMetaList = ArgumentsUtils.getTableMetaList(args);
+ String cluster = ArgumentsUtils.getCluster(args);
+
+ logger.info("Bootstrap flink job: {}", mapper.writeValueAsString(flinkJob));
+ logger.info("Bootstrap table meta list: {}", mapper.writeValueAsString(tableMetaList));
+ logger.info("Bootstrap cluster: {}", cluster);
+
+ String applicationId = System.getenv("_APP_ID");
+
+ String zkUrl = findConfigFromList(tableMetaList, meta -> meta.getConfig().getZookeeperUrl(), ZookeeperUrlNotFoundException::new);
+ for (TableMeta tableMeta : tableMetaList) {
+ RunMeta runMeta = new RunMeta(cluster, flinkJob.getId(), tableMeta.getAlias());
+ logger.info("Run meta: {}", runMeta);
+ ZkUtils.createSynchronizerLock(flinkJob, tableMeta, zkUrl, mapper.writeValueAsString(runMeta));
+ }
+ RunMeta runMeta = new RunMeta(cluster, flinkJob.getId());
+ logger.info("Run meta: {}", runMeta);
+ ZkUtils.createSynchronizerLock(flinkJob, zkUrl, mapper.writeValueAsString(runMeta));
+ logger.info("Lock for {} success", flinkJob.getId());
+
+ StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
+
+ environment.enableCheckpointing(15 * MINUTE);
+ String checkpointRootPath = findConfigFromList(tableMetaList, meta -> meta.getConfig().getCheckpointRootPath(), CheckpointRootPathNotFoundException::new);
+ environment.getCheckpointConfig().setCheckpointStorage(new Path(checkpointRootPath + "/" + flinkJob.getId()));
+ environment.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
+ environment.getCheckpointConfig().setCheckpointTimeout(2 * HOUR);
+ environment.getCheckpointConfig().setMinPauseBetweenCheckpoints(15 * MINUTE);
+ environment.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
+ environment.getCheckpointConfig().setForceUnalignedCheckpoints(true);
+ environment.getCheckpointConfig().enableUnalignedCheckpoints();
+ environment.getCheckpointConfig().setTolerableCheckpointFailureNumber(5);
+
+ if (tableMetaList.stream().anyMatch(meta -> TableMetaHelper.existsTag(meta, TAGS_DISABLE_CHAINING))) {
+ logger.warn("Disable operator chaining");
+ environment.disableOperatorChaining();
+ }
+
+ environment.setRestartStrategy(RestartStrategies.fixedDelayRestart(5, MINUTE));
+
+ environment.setStateBackend(new HashMapStateBackend());
+
+ switch (flinkJob.getRunMode()) {
+ case ALL_IN_ONE:
+ for (TableMeta tableMeta : tableMetaList) {
+ GlobalConfiguration globalConfiguration = new GlobalConfiguration(cluster, applicationId, tableMeta);
+ createFlinkJob(environment, globalConfiguration, flinkJob, tableMeta);
+ publishSyncStart(globalConfiguration, flinkJob, tableMeta);
+ }
+ environment.execute(NameHelper.syncFlinkName(flinkJob.getId(), flinkJob.getName()));
+ break;
+ case ONE_IN_ONE:
+ for (TableMeta tableMeta : tableMetaList) {
+ GlobalConfiguration globalConfiguration = new GlobalConfiguration(cluster, applicationId, tableMeta);
+ createFlinkJob(environment, globalConfiguration, flinkJob, tableMeta);
+ publishSyncStart(globalConfiguration, flinkJob, tableMeta);
+ environment.execute(NameHelper.syncFlinkName(flinkJob.getId(), flinkJob.getName(), tableMeta.getAlias()));
+ }
+ break;
+ case ALL_IN_ONE_BY_TABLE:
+ scheduleOneInOneRegistryByField(environment, cluster, applicationId, flinkJob, tableMetaList, TableMeta::getTable);
+ break;
+ case ALL_IN_ONE_BY_SCHEMA:
+ scheduleOneInOneRegistryByField(environment, cluster, applicationId, flinkJob, tableMetaList, TableMeta::getSchema);
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported run mode: " + flinkJob.getRunMode());
+ }
+ }
+
+ private static void scheduleOneInOneRegistryByField(StreamExecutionEnvironment environment, String cluster, String applicationId, FlinkJob flinkJob, List tableMetaList, Function field) throws Exception {
+ Map> map = tableMetaList.stream()
+ .collect(Collectors.groupingBy(field));
+ for (Map.Entry> entry : map.entrySet()) {
+ for (TableMeta tableMeta : entry.getValue()) {
+ GlobalConfiguration globalConfiguration = new GlobalConfiguration(cluster, applicationId, tableMeta);
+ createFlinkJob(environment, globalConfiguration, flinkJob, tableMeta);
+ publishSyncStart(globalConfiguration, flinkJob, tableMeta);
+ }
+ environment.execute(NameHelper.syncFlinkName(flinkJob.getId(), flinkJob.getName(), entry.getKey()));
+ }
+ }
+
+ private static void createFlinkJob(StreamExecutionEnvironment environment, GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta) throws IOException {
+ logger.info("Table meta: {}", mapper.writeValueAsString(tableMeta));
+ logger.info("Config meta: {}", mapper.writeValueAsString(configuration));
+ SingleOutputStreamOperator source = environment
+ .addSource(new PulsarMessageSourceReader(configuration, flinkJob, tableMeta))
+ .setParallelism(tableMeta.getHudi().getSourceTasks());
+ if (TableMetaHelper.existsTag(tableMeta, Constants.TAGS_PULSAR_BACKUP)) {
+ Path path = new Path(StrUtil.format("hdfs://b2/apps/datalake/hive_test/source/{}/{}", String.join("_", flinkJob.getName().split("\\s")), tableMeta.getAlias()));
+ StreamingFileSink fileSink = StreamingFileSink.forRowFormat(path, new SimpleStringEncoder<>("UTF-8"))
+ .withRollingPolicy(DefaultRollingPolicy.builder()
+ .withInactivityInterval(HOUR)
+ .withMaxPartSize(GB)
+ .build())
+ .build();
+ source.addSink(fileSink).name("Backup pulsar data");
+ }
+ SingleOutputStreamOperator middle = source
+ .map(new PulsarMessage2RecordFunction(configuration, flinkJob, tableMeta))
+ .name("Json ( " + tableMeta.getSchema() + "-" + tableMeta.getAlias() + " )")
+ .filter(new ValidateRecordFilter(configuration, flinkJob, tableMeta))
+ .name("Reject json parse failure");
+ SyncUtils.sinkToHoodieByTable(configuration, flinkJob, tableMeta, environment, middle);
+ }
+
+ private static void publishSyncStart(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta) {
+ StatusUtils.syncStart(configuration, flinkJob, tableMeta);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/DefaultPartitionNameKeyGenerator.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/DefaultPartitionNameKeyGenerator.java
new file mode 100644
index 0000000..19f46bd
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/DefaultPartitionNameKeyGenerator.java
@@ -0,0 +1,36 @@
+package com.lanyuanxiaoyao.service.sync.configuration;
+
+import org.apache.hudi.avro.HoodieAvroUtils;
+import org.apache.hudi.common.config.TypedProperties;
+import org.apache.hudi.common.util.PartitionPathEncodeUtils;
+import org.apache.hudi.configuration.FlinkOptions;
+import org.apache.hudi.keygen.SimpleAvroKeyGenerator;
+import org.apache.hudi.org.apache.avro.generic.GenericRecord;
+
+/**
+ * @author ZhangJiacheng
+ */
+public class DefaultPartitionNameKeyGenerator extends SimpleAvroKeyGenerator {
+ private final String defaultPartitionName;
+
+ public DefaultPartitionNameKeyGenerator(TypedProperties props) {
+ super(props);
+ defaultPartitionName = props.getString(FlinkOptions.PARTITION_DEFAULT_NAME.key(), FlinkOptions.PARTITION_DEFAULT_NAME.defaultValue());
+ }
+
+ @Override
+ public String getPartitionPath(GenericRecord record) {
+ String partitionPathField = getPartitionPathFields().get(0);
+ String partitionPath = HoodieAvroUtils.getNestedFieldValAsString(record, partitionPathField, true, consistentLogicalTimestampEnabled);
+ if (partitionPath == null || partitionPath.isEmpty()) {
+ partitionPath = defaultPartitionName;
+ }
+ if (encodePartitionPath) {
+ partitionPath = PartitionPathEncodeUtils.escapePathName(partitionPath);
+ }
+ if (hiveStylePartitioning) {
+ partitionPath = partitionPathField + "=" + partitionPath;
+ }
+ return partitionPath;
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/GlobalConfiguration.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/GlobalConfiguration.java
new file mode 100644
index 0000000..3219277
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/GlobalConfiguration.java
@@ -0,0 +1,92 @@
+package com.lanyuanxiaoyao.service.sync.configuration;
+
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import java.io.Serializable;
+
+/**
+ * 同步静态配置
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public class GlobalConfiguration implements Serializable {
+ private final String cluster;
+ private final String applicationId;
+ private final Boolean metricEnable = false;
+ private final String metricPublishUrl;
+ private final String metricPublishPrometheusUrl;
+ private final String metricsPublishCustomUrl;
+ private final Integer metricPublishDelay;
+ private final Integer metricPublishPeriod;
+ private final Integer metricPublishTimeout;
+ private final Integer metricPublishBatch;
+
+ public GlobalConfiguration(String cluster, String applicationId, TableMeta meta) {
+ this.cluster = cluster;
+ this.applicationId = applicationId;
+ this.metricPublishUrl = meta.getConfig().getMetricPublishUrl();
+ this.metricPublishPrometheusUrl = meta.getConfig().getMetricPrometheusUrl();
+ this.metricsPublishCustomUrl = meta.getConfig().getMetricApiUrl();
+ this.metricPublishDelay = meta.getConfig().getMetricPublishDelay();
+ this.metricPublishPeriod = meta.getConfig().getMetricPublishPeriod();
+ this.metricPublishTimeout = meta.getConfig().getMetricPublishTimeout();
+ this.metricPublishBatch = meta.getConfig().getMetricPublishBatch();
+ }
+
+ public String getCluster() {
+ return cluster;
+ }
+
+ public String getApplicationId() {
+ return applicationId;
+ }
+
+ public Boolean getMetricEnable() {
+ return metricEnable;
+ }
+
+ public String getMetricPublishUrl() {
+ return metricPublishUrl;
+ }
+
+ public String getMetricPublishPrometheusUrl() {
+ return metricPublishPrometheusUrl;
+ }
+
+ public String getMetricsPublishCustomUrl() {
+ return metricsPublishCustomUrl;
+ }
+
+ public Integer getMetricPublishDelay() {
+ return metricPublishDelay;
+ }
+
+ public Integer getMetricPublishPeriod() {
+ return metricPublishPeriod;
+ }
+
+ public Integer getMetricPublishTimeout() {
+ return metricPublishTimeout;
+ }
+
+ public Integer getMetricPublishBatch() {
+ return metricPublishBatch;
+ }
+
+
+ @Override
+ public String toString() {
+ return "GlobalConfiguration{" +
+ "cluster='" + cluster + '\'' +
+ ", applicationId='" + applicationId + '\'' +
+ ", metricEnable=" + metricEnable +
+ ", metricPublishUrl='" + metricPublishUrl + '\'' +
+ ", metricPublishPrometheusUrl='" + metricPublishPrometheusUrl + '\'' +
+ ", metricsPublishCustomUrl='" + metricsPublishCustomUrl + '\'' +
+ ", metricPublishDelay=" + metricPublishDelay +
+ ", metricPublishPeriod=" + metricPublishPeriod +
+ ", metricPublishTimeout=" + metricPublishTimeout +
+ ", metricPublishBatch=" + metricPublishBatch +
+ '}';
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/RetryPolicyProvider.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/RetryPolicyProvider.java
new file mode 100644
index 0000000..68c2a9d
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/RetryPolicyProvider.java
@@ -0,0 +1,12 @@
+package com.lanyuanxiaoyao.service.sync.configuration;
+
+import dev.failsafe.RetryPolicy;
+import java.time.Duration;
+
+public interface RetryPolicyProvider {
+ RetryPolicy HTTP_RETRY = RetryPolicy.builder()
+ .handle(Throwable.class)
+ .withDelay(Duration.ofSeconds(1))
+ .withMaxAttempts(10)
+ .build();
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/TraceOverwriteWithLatestAvroPayload.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/TraceOverwriteWithLatestAvroPayload.java
new file mode 100644
index 0000000..8744632
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/TraceOverwriteWithLatestAvroPayload.java
@@ -0,0 +1,52 @@
+package com.lanyuanxiaoyao.service.sync.configuration;
+
+import com.lanyuanxiaoyao.service.common.Constants;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.org.apache.avro.generic.GenericRecord;
+import org.apache.hudi.org.apache.avro.util.Utf8;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-04-18
+ */
+public class TraceOverwriteWithLatestAvroPayload extends OverwriteWithLatestAvroPayload {
+ private static final Logger logger = LoggerFactory.getLogger(TraceOverwriteWithLatestAvroPayload.class);
+
+ private final String latestOpts;
+
+ public TraceOverwriteWithLatestAvroPayload(GenericRecord record, Comparable orderingVal) {
+ super(record, orderingVal);
+ this.latestOpts = updateLatestOpts(Option.ofNullable(record));
+ }
+
+ public TraceOverwriteWithLatestAvroPayload(Option record) {
+ super(record);
+ this.latestOpts = updateLatestOpts(record);
+ }
+
+ private String updateLatestOpts(Option record) {
+ try {
+ return record
+ .map(r -> ((Utf8) r.get(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME)).toString())
+ .orElse(null);
+ } catch (Throwable throwable) {
+ logger.error("Get latest opts failure", throwable);
+ }
+ return null;
+ }
+
+ @Override
+ public Option> getMetadata() {
+ if (this.latestOpts == null) {
+ return Option.empty();
+ }
+ Map metadata = super.getMetadata().orElse(new HashMap<>());
+ metadata.put(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME, this.latestOpts);
+ return Option.of(metadata);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/TraceWriteStatus.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/TraceWriteStatus.java
new file mode 100644
index 0000000..2b18575
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/configuration/TraceWriteStatus.java
@@ -0,0 +1,58 @@
+package com.lanyuanxiaoyao.service.sync.configuration;
+
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Map;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.util.Option;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author lanyuanxiaoyao
+ * @date 2023-04-17
+ */
+public class TraceWriteStatus extends WriteStatus {
+ private static final Logger logger = LoggerFactory.getLogger(TraceWriteStatus.class);
+ private final static DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+ private long latestOpts = 0L;
+
+ public TraceWriteStatus() {
+ super();
+ }
+
+ public TraceWriteStatus(Boolean trackSuccessRecords, Double failureFraction) {
+ super(trackSuccessRecords, failureFraction);
+ }
+
+ public long getLatestOpts() {
+ return latestOpts;
+ }
+
+ @Override
+ public void markSuccess(HoodieRecord record, Option> optionalRecordMetadata) {
+ super.markSuccess(record, optionalRecordMetadata);
+ try {
+ optionalRecordMetadata.ifPresent(map -> {
+ if (map.containsKey(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME)) {
+ String inOpts = map.get(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME);
+ if (StrUtil.isNotBlank(inOpts)) {
+ long current = LocalDateTime.parse(inOpts, FORMATTER).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
+ latestOpts = Long.max(latestOpts, current);
+ }
+ }
+ });
+ } catch (Throwable throwable) {
+ logger.error("Parse latest opts failure", throwable);
+ }
+ }
+
+ @Override
+ public void markFailure(HoodieRecord record, Throwable t, Option> optionalRecordMetadata) {
+ super.markFailure(record, t, optionalRecordMetadata);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/CompactionEventHandler.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/CompactionEventHandler.java
new file mode 100644
index 0000000..ec7a286
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/CompactionEventHandler.java
@@ -0,0 +1,65 @@
+package com.lanyuanxiaoyao.service.sync.functions;
+
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.utils.TableMetaHelper;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.configuration.TraceWriteStatus;
+import com.lanyuanxiaoyao.service.sync.utils.StatusUtils;
+import java.io.Serializable;
+import java.util.List;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.apache.hudi.sink.compact.CompactEventHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 压缩事件处理
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-15
+ */
+public class CompactionEventHandler implements CompactEventHandler, Serializable {
+ private static final Logger logger = LoggerFactory.getLogger(CompactionEventHandler.class);
+
+ private final GlobalConfiguration configuration;
+ private final FlinkJob flinkJob;
+ private final TableMeta tableMeta;
+
+ public CompactionEventHandler(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta) {
+ this.configuration = configuration;
+ this.flinkJob = flinkJob;
+ this.tableMeta = tableMeta;
+ }
+
+ @Override
+ public void failure(String instant) {
+ }
+
+ @Override
+ public void success(String instant, List statuses, HoodieCommitMetadata metadata) {
+ StatusUtils.compactionCommit(configuration, flinkJob, tableMeta, instant, metadata);
+ logger.info("WriteStatus: {}", statuses);
+ if (TableMetaHelper.existsTag(tableMeta, Constants.TAGS_TRACE_LATEST_OP_TS)) {
+ Long max = statuses.stream()
+ .map(status -> {
+ if (status instanceof TraceWriteStatus) {
+ TraceWriteStatus s = (TraceWriteStatus) status;
+ return s.getLatestOpts();
+ }
+ return 0L;
+ })
+ .max(Long::compare)
+ .orElse(0L);
+ logger.info("Latest op ts: {}", max);
+ StatusUtils.compactionLatestOpTs(configuration, flinkJob, tableMeta, max);
+ }
+ }
+
+ @Override
+ public void closed(String message, Exception exception) {
+ StatusUtils.compactionFinish(configuration, flinkJob, tableMeta, message, exception);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/OperationTypeFilter.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/OperationTypeFilter.java
new file mode 100644
index 0000000..5592447
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/OperationTypeFilter.java
@@ -0,0 +1,90 @@
+package com.lanyuanxiaoyao.service.sync.functions;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.Record;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.metrics.CountMetric;
+import com.lanyuanxiaoyao.service.sync.utils.MetricsUtils;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.flink.api.common.functions.RichFilterFunction;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+
+/**
+ * 操作类型过滤算子
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-12
+ */
+public class OperationTypeFilter extends RichFilterFunction implements CheckpointedFunction {
+ private final CountMetric insertRateMetric;
+ private final CountMetric updateRateMetric;
+ private final CountMetric deleteRateMetric;
+ private final CountMetric ddlRateMetric;
+ private final CountMetric unknownRateMetric;
+ private final List metrics;
+ private final GlobalConfiguration globalConfiguration;
+
+ public OperationTypeFilter(GlobalConfiguration globalConfiguration, FlinkJob flinkJob, TableMeta tableMeta) {
+ this.globalConfiguration = globalConfiguration;
+
+ Function> fillTags = operator -> MapUtil.builder()
+ .put(Constants.METRICS_LABEL_TYPE, operator)
+ .build();
+
+ insertRateMetric = new CountMetric(globalConfiguration, Constants.METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE, flinkJob, tableMeta, fillTags.apply(Constants.INSERT));
+ updateRateMetric = new CountMetric(globalConfiguration, Constants.METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE, flinkJob, tableMeta, fillTags.apply(Constants.UPDATE));
+ deleteRateMetric = new CountMetric(globalConfiguration, Constants.METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE, flinkJob, tableMeta, fillTags.apply(Constants.DELETE));
+ ddlRateMetric = new CountMetric(globalConfiguration, Constants.METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE, flinkJob, tableMeta, fillTags.apply(Constants.DDL));
+ unknownRateMetric = new CountMetric(globalConfiguration, Constants.METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE, flinkJob, tableMeta, fillTags.apply(Constants.UNKNOWN));
+
+ metrics = ListUtil.toList(insertRateMetric, updateRateMetric, deleteRateMetric, ddlRateMetric, unknownRateMetric);
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ // 初始化指标
+ MetricsUtils.createMakePointTimer(globalConfiguration, metrics);
+ }
+
+ @Override
+ public boolean filter(Record record) {
+ String opType = record.getStatement().getOpType();
+ switch (opType) {
+ case Constants.INSERT:
+ insertRateMetric.increment();
+ break;
+ case Constants.UPDATE:
+ updateRateMetric.increment();
+ break;
+ case Constants.DELETE:
+ deleteRateMetric.increment();
+ break;
+ case Constants.DDL:
+ ddlRateMetric.increment();
+ break;
+ default:
+ unknownRateMetric.increment();
+ }
+ return !Constants.DDL.equals(record.getStatement().getOpType());
+ }
+
+ @Override
+ public void initializeState(FunctionInitializationContext context) {
+ }
+
+ @Override
+ public void snapshotState(FunctionSnapshotContext context) {
+ MetricsUtils.publishAllMetrics(metrics);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/PulsarMessage2RecordFunction.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/PulsarMessage2RecordFunction.java
new file mode 100644
index 0000000..6ad5425
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/PulsarMessage2RecordFunction.java
@@ -0,0 +1,80 @@
+package com.lanyuanxiaoyao.service.sync.functions;
+
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.Record;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.utils.RecordHelper;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.utils.JacksonUtils;
+import com.lanyuanxiaoyao.service.sync.utils.StatusUtils;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Pulsar message to object
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-11
+ */
+public class PulsarMessage2RecordFunction extends RichMapFunction implements CheckpointedFunction {
+ private static final Logger logger = LoggerFactory.getLogger(PulsarMessage2RecordFunction.class);
+ private static final AtomicReference lastOperationTime = new AtomicReference<>("");
+ private final static DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+ private static final Pattern OPTS_PATTERN = Pattern.compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$");
+ private final GlobalConfiguration globalConfiguration;
+ private final FlinkJob flinkJob;
+ private final TableMeta tableMeta;
+ private final ObjectMapper mapper = JacksonUtils.getMapper();
+
+ public PulsarMessage2RecordFunction(GlobalConfiguration globalConfiguration, FlinkJob flinkJob, TableMeta tableMeta) {
+ this.globalConfiguration = globalConfiguration;
+ this.flinkJob = flinkJob;
+ this.tableMeta = tableMeta;
+ }
+
+ @Override
+ public Record map(String message) throws JsonProcessingException {
+ Record record = null;
+ try {
+ record = mapper.readValue(message, Record.class);
+ if (RecordHelper.isNotVersionUpdateRecord(record)) {
+ lastOperationTime.set(record.getStatement().getOpTs());
+ }
+ } catch (Exception exception) {
+ logger.error("Message json parse failure", exception);
+ }
+ return record;
+ }
+
+ @Override
+ public void snapshotState(FunctionSnapshotContext context) {
+ String opTs = lastOperationTime.get();
+ Long timestamp = null;
+ try {
+ if (StrUtil.isNotBlank(opTs) && OPTS_PATTERN.matcher(opTs).matches()) {
+ timestamp = LocalDateTime.parse(opTs, FORMATTER).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
+ } else {
+ throw new Exception("Opts is not match regex " + OPTS_PATTERN.pattern());
+ }
+ } catch (Exception e) {
+ logger.error("Parse operation time error", e);
+ }
+ StatusUtils.syncOperation(globalConfiguration, flinkJob, tableMeta, timestamp);
+ }
+
+ @Override
+ public void initializeState(FunctionInitializationContext context) {
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/PulsarMessageSourceReader.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/PulsarMessageSourceReader.java
new file mode 100644
index 0000000..98b32d4
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/PulsarMessageSourceReader.java
@@ -0,0 +1,235 @@
+package com.lanyuanxiaoyao.service.sync.functions;
+
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.utils.LogHelper;
+import com.lanyuanxiaoyao.service.common.utils.NameHelper;
+import com.lanyuanxiaoyao.service.common.utils.RecordHelper;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.metrics.MessageSizeSizeMetric;
+import com.lanyuanxiaoyao.service.sync.metrics.RateMetric;
+import com.lanyuanxiaoyao.service.sync.utils.LoadBalance;
+import com.lanyuanxiaoyao.service.sync.utils.MetricsUtils;
+import com.lanyuanxiaoyao.service.sync.utils.StatusUtils;
+import dev.failsafe.Failsafe;
+import dev.failsafe.RetryPolicy;
+import java.time.Duration;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.flink.api.common.state.CheckpointListener;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+import org.apache.flink.streaming.api.functions.source.RichParallelSourceFunction;
+import org.apache.pulsar.client.api.*;
+import org.apache.pulsar.client.impl.schema.StringSchema;
+import org.apache.pulsar.client.internal.DefaultImplementation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.lanyuanxiaoyao.service.common.utils.LogHelper.LogPoint.*;
+
+/**
+ * Pulsar Reader Source
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-11
+ */
+public class PulsarMessageSourceReader extends RichParallelSourceFunction implements CheckpointedFunction, CheckpointListener {
+ private static final Logger logger = LoggerFactory.getLogger(PulsarMessageSourceReader.class);
+ private static final RetryPolicy MESSAGE_ID_RETRY = RetryPolicy.builder()
+ .handle(Exception.class)
+ .withDelay(Duration.ofSeconds(1))
+ .withMaxAttempts(10)
+ .build();
+ private final String topic;
+ private final GlobalConfiguration globalConfiguration;
+ private final FlinkJob flinkJob;
+ private final TableMeta tableMeta;
+ private final AtomicReference lastMessageId = new AtomicReference<>();
+ private final AtomicLong lastPublishTime = new AtomicLong(0);
+ private final RateMetric messageReceiveMetric;
+ private final MessageSizeSizeMetric messageSizeReceiveMetric;
+ private final Map messageIdMap = new ConcurrentHashMap<>();
+ private boolean running = true;
+ private PulsarClient client = null;
+ private Reader reader = null;
+
+ public PulsarMessageSourceReader(GlobalConfiguration globalConfiguration, FlinkJob flinkJob, TableMeta tableMeta) {
+ logger.info("Use PulsarMessageSourceReader");
+ this.globalConfiguration = globalConfiguration;
+ this.flinkJob = flinkJob;
+ this.topic = tableMeta.getTopic();
+ this.tableMeta = tableMeta;
+
+ String messageId = tableMeta.getConfig().getMessageId();
+ logger.info("{} {}", Constants.LOG_POINT_PULSAR_SOURCE_BOOTSTRAP_MESSAGE_ID, messageId);
+ if (StrUtil.isNotBlank(messageId)) {
+ lastMessageId.set(parseMessageId(messageId));
+ } else {
+ logger.warn("Message id is empty");
+ lastMessageId.set(MessageId.earliest);
+ }
+
+ messageReceiveMetric = new RateMetric(
+ globalConfiguration,
+ Constants.METRICS_SYNC_SOURCE_MESSAGE_RECEIVE,
+ flinkJob,
+ tableMeta
+ );
+ messageSizeReceiveMetric = new MessageSizeSizeMetric(
+ globalConfiguration,
+ Constants.METRICS_SYNC_SOURCE_MESSAGE_SIZE_RECEIVE_BYTES,
+ flinkJob, tableMeta
+ );
+ }
+
+ private static MessageId parseMessageId(String messageIdText) {
+ return DefaultImplementation.newMessageId(Long.parseLong(messageIdText.split(":")[0]), Long.parseLong(messageIdText.split(":")[1]), -1);
+ }
+
+ @Override
+ public void initializeState(FunctionInitializationContext context) throws Exception {
+ LogHelper.info(logger, CHECKPOINT_INITIAL);
+ String queryUrl = StrUtil.format(
+ "{}/api/message_id?flink_job_id={}&alias={}",
+ LoadBalance.getCustomPublishUrl(globalConfiguration),
+ flinkJob.getId(),
+ tableMeta.getAlias()
+ );
+ logger.info("Query url: {}", queryUrl);
+ String messageId = Failsafe.with(MESSAGE_ID_RETRY)
+ .onFailure(event -> {
+ if (ObjectUtil.isNotNull(event.getException())) {
+ logger.error(StrUtil.format("{} Get message id error", Constants.LOG_POINT_PULSAR_SOURCE_GET_MESSAGE_ID_ERROR), event.getException());
+ }
+ })
+ .get(() ->
+ HttpUtil.createGet(queryUrl)
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .execute()
+ .body()
+ );
+ LogHelper.info(logger, CHECKPOINT_INITIAL_MESSAGE_ID, "Get message id: {}", messageId);
+ if (StrUtil.isNotBlank(messageId)) {
+ lastMessageId.set(parseMessageId(messageId));
+ } else {
+ logger.warn(StrUtil.format("{} Message id is empty, now message id is {}", Constants.LOG_POINT_MESSAGE_ID_EMPTY, lastMessageId.get()));
+ }
+ }
+
+ @Override
+ public void run(SourceContext context) throws Exception {
+ String currentValue = null;
+ while (running) {
+ Message message;
+ try {
+ message = reader.readNext();
+ if (ObjectUtil.isNotNull(message)) {
+ String value = message.getValue();
+ currentValue = value;
+ if (ObjectUtil.isEmpty(value)) {
+ logger.warn("{} {}", message.getValue(), message.getMessageId());
+ }
+ synchronized (context.getCheckpointLock()) {
+ context.collect(value);
+ }
+
+ if (RecordHelper.isNotVersionUpdateRecord(value)) {
+ lastPublishTime.set(message.getPublishTime());
+ }
+ lastMessageId.set(message.getMessageId());
+
+ messageReceiveMetric.increment();
+ try {
+ messageSizeReceiveMetric.increment(message.getValue().getBytes().length);
+ } catch (Throwable t) {
+ logger.warn("Parse message size failure", t);
+ }
+ }
+ } catch (Throwable t) {
+ logger.error("Read message failure, current value: " + currentValue, t);
+ }
+ }
+ }
+
+ @Override
+ public void open(Configuration configuration) throws Exception {
+ super.open(configuration);
+
+ // 初始化指标
+ MetricsUtils.createMakePointTimer(globalConfiguration, messageReceiveMetric);
+ MetricsUtils.createMakePointTimer(globalConfiguration, messageSizeReceiveMetric);
+
+ try {
+ client = PulsarClient.builder()
+ .serviceUrl(tableMeta.getPulsarAddress())
+ .build();
+ reader = client.newReader(new StringSchema())
+ .topic(topic)
+ .receiverQueueSize(10000)
+ .subscriptionName(NameHelper.pulsarSubscriptionName(flinkJob.getId(), tableMeta.getAlias()))
+ .startMessageId(lastMessageId.get())
+ .startMessageIdInclusive()
+ .create();
+ } catch (Exception exception) {
+ logger.error(StrUtil.format("Connect pulsar error ({} {})", tableMeta.getPulsarAddress(), topic), exception);
+ throw exception;
+ }
+ logger.info("Message id set to {}", lastMessageId.get());
+ }
+
+ @Override
+ public void cancel() {
+ running = false;
+ }
+
+ @Override
+ public void close() throws Exception {
+ super.close();
+ if (reader != null) {
+ try {
+ reader.close();
+ } catch (PulsarClientException e) {
+ logger.error("Pulsar reader close error", e);
+ }
+ }
+ if (client != null) {
+ try {
+ client.close();
+ } catch (PulsarClientException e) {
+ logger.error("Pulsar client close error", e);
+ }
+ }
+ }
+
+ @Override
+ public void snapshotState(FunctionSnapshotContext context) {
+ MessageId messageId = lastMessageId.get();
+ messageIdMap.put(context.getCheckpointId(), messageId);
+ LogHelper.info(logger, CHECKPOINT_START, "Checkpoint start message id: {}, checkpoint id: {}", messageId, context.getCheckpointId());
+
+ messageReceiveMetric.publish();
+ messageSizeReceiveMetric.publish();
+ }
+
+ @Override
+ public void notifyCheckpointComplete(long checkpointId) {
+ MessageId messageId = messageIdMap.getOrDefault(checkpointId, MessageId.earliest);
+ LogHelper.info(logger, CHECKPOINT_COMPLETE, "Checkpoint complete message id: {}, checkpoint id: {}", messageId, checkpointId);
+ StatusUtils.syncCheckpoint(globalConfiguration, flinkJob, tableMeta, messageId.toString(), lastPublishTime.get());
+ messageIdMap.remove(checkpointId);
+ }
+
+ @Override
+ public void notifyCheckpointAborted(long checkpointId) throws Exception {
+ CheckpointListener.super.notifyCheckpointAborted(checkpointId);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/Record2RowDataFunction.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/Record2RowDataFunction.java
new file mode 100644
index 0000000..90253d1
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/Record2RowDataFunction.java
@@ -0,0 +1,171 @@
+package com.lanyuanxiaoyao.service.sync.functions;
+
+import cn.hutool.core.collection.ListUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.Record;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.utils.LogHelper;
+import com.lanyuanxiaoyao.service.common.utils.MapHelper;
+import com.lanyuanxiaoyao.service.common.utils.RecordHelper;
+import com.lanyuanxiaoyao.service.common.utils.TableMetaHelper;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.functions.type.TypeConverter;
+import com.lanyuanxiaoyao.service.sync.metrics.CountMetric;
+import com.lanyuanxiaoyao.service.sync.utils.JacksonUtils;
+import com.lanyuanxiaoyao.service.sync.utils.MetricsUtils;
+import com.lanyuanxiaoyao.service.sync.utils.StatusUtils;
+import com.lanyuanxiaoyao.service.sync.utils.SyncUtils;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import org.apache.flink.api.common.functions.RichMapFunction;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.runtime.state.FunctionInitializationContext;
+import org.apache.flink.runtime.state.FunctionSnapshotContext;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.flink.streaming.api.checkpoint.CheckpointedFunction;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.hudi.org.apache.avro.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.lanyuanxiaoyao.service.common.utils.LogHelper.LogPoint.VERSION_UPDATE;
+
+/**
+ * Record 转 Rowdata
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public class Record2RowDataFunction extends RichMapFunction> implements CheckpointedFunction {
+ private static final Logger logger = LoggerFactory.getLogger(Record2RowDataFunction.class);
+
+ private final GlobalConfiguration globalConfiguration;
+ private final TableMeta tableMeta;
+ private final CountMetric changeFilterMetric;
+ private final CountMetric changePartitionMetric;
+ private final ObjectMapper mapper = JacksonUtils.getMapper();
+ private final FlinkJob flinkJob;
+ private Schema schema;
+
+ public Record2RowDataFunction(GlobalConfiguration globalConfiguration, FlinkJob flinkJob, TableMeta tableMeta) {
+ this.globalConfiguration = globalConfiguration;
+ this.flinkJob = flinkJob;
+ this.tableMeta = tableMeta;
+
+ changeFilterMetric = new CountMetric(
+ globalConfiguration,
+ Constants.METRICS_SYNC_SOURCE_CHANGE_FILTER,
+ flinkJob, tableMeta
+ );
+ changePartitionMetric = new CountMetric(
+ globalConfiguration,
+ Constants.METRICS_SYNC_SOURCE_CHANGE_PARTITION,
+ flinkJob, tableMeta
+ );
+ }
+
+ @Override
+ public void initializeState(FunctionInitializationContext context) throws Exception {
+ schema = SyncUtils.avroSchemaWithExtraFields(tableMeta);
+ }
+
+ @Override
+ public void open(Configuration parameters) throws Exception {
+ super.open(parameters);
+
+ // 初始化指标
+ MetricsUtils.createMakePointTimer(globalConfiguration, changeFilterMetric);
+ MetricsUtils.createMakePointTimer(globalConfiguration, changePartitionMetric);
+ }
+
+ @Override
+ public void snapshotState(FunctionSnapshotContext context) throws Exception {
+ MetricsUtils.publishAllMetrics(changeFilterMetric);
+ MetricsUtils.publishAllMetrics(changePartitionMetric);
+ }
+
+ private GenericRowData covert2RowData(Schema schema, Map current) {
+ List fields = schema.getFields();
+ GenericRowData data = new GenericRowData(fields.size());
+ for (int index = 0; index < fields.size(); index++) {
+ Schema.Field field = fields.get(index);
+ // 如果是telepg的话，字段名就要统一改成小写，上游不规范，下游擦屁股
+ Object value = current.getOrDefault(Constants.FIELD_COVERT.apply(tableMeta, field.name()), null);
+ if (field.schema().getType().equals(Schema.Type.STRING)
+ || (field.schema().isUnion() && field.schema().getTypes().contains(Schema.create(Schema.Type.STRING)))
+ || value instanceof String) {
+ data.setField(index, StringData.fromString((String) value));
+ } else {
+ data.setField(index, value);
+ }
+ }
+ return data;
+ }
+
+ private Boolean isFilterOut(TableMeta tableMeta, Map current) {
+ if (!tableMeta.getFilterType().equals(TableMeta.FilterType.NONE)) {
+ if (current.containsKey(tableMeta.getFilterField())) {
+ String fieldValue = MapHelper.getStringWithoutCase(current, tableMeta.getFilterField());
+ if (tableMeta.getFilterType().equals(TableMeta.FilterType.EXCLUDE)) {
+ return tableMeta.getFilterValues().contains(fieldValue);
+ } else if (tableMeta.getFilterType().equals(TableMeta.FilterType.INCLUDE)) {
+ return !tableMeta.getFilterValues().contains(fieldValue);
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public List map(Record record) throws Exception {
+ List> result = ListUtil.list(false);
+
+ if (RecordHelper.isVersionUpdateRecord(record)) {
+ Record.Statement statement = record.getStatement();
+ LogHelper.info(logger, VERSION_UPDATE, "{} {} version: {}", mapper.writeValueAsString(statement.getSchema()), statement.getVersion(), statement.getVersion());
+ LogHelper.info(logger, VERSION_UPDATE, "Raw: {}", mapper.writeValueAsString(record));
+ StatusUtils.versionUpdate(globalConfiguration, flinkJob, tableMeta, record.getStatement().getVersion(), statement.getOpTs());
+ return ListUtil.empty();
+ }
+
+ Map current = RecordHelper.getCurrentStatement(record);
+ if (Objects.isNull(current)) {
+ logger.error("Record: {}", mapper.writeValueAsString(record));
+ throw new RuntimeException("Current cannot be null");
+ }
+
+ // 如果 update 改变了过滤字段的值也需要先删除
+ boolean isChangeFilter = RecordHelper.isChangeField(tableMeta, record, TableMetaHelper::getFilterField);
+ if (isChangeFilter) {
+ logger.info("Change filter: {}", mapper.writeValueAsString(record));
+ changeFilterMetric.increment();
+ }
+
+ // 如果是 update 且 city_id 不相等就先删除旧记录
+ boolean isChangePartition = RecordHelper.isChangeField(tableMeta, record, TableMetaHelper::getPartitionField);
+ if (isChangePartition) {
+ logger.info("Change partition field: {}", mapper.writeValueAsString(record));
+ changePartitionMetric.increment();
+ }
+
+ if (isChangeFilter || isChangePartition) {
+ Map before = record.getStatement().getBefore();
+ result.add(0, RecordHelper.addExtraMetadata(before, tableMeta, record, true));
+ }
+
+ // 增加 Hudi 特有字段信息
+ result.add(RecordHelper.addExtraMetadata(current, tableMeta, record));
+
+ return result.stream()
+ // 按过滤字段过滤
+ .filter(r -> !isFilterOut(tableMeta, r))
+ .map(r -> TypeConverter.getInstance(tableMeta)
+ .convertToGenericRowData(tableMeta, schema, r))
+ .collect(Collectors.toList());
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/ValidateRecordFilter.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/ValidateRecordFilter.java
new file mode 100644
index 0000000..0e9e434
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/ValidateRecordFilter.java
@@ -0,0 +1,40 @@
+package com.lanyuanxiaoyao.service.sync.functions;
+
+import cn.hutool.core.util.ObjectUtil;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.Record;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import org.apache.flink.api.common.functions.RichFilterFunction;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 判断 Record 是否正确
+ *
+ * @author ZhangJiacheng
+ * @date 2022-11-15
+ */
+public class ValidateRecordFilter extends RichFilterFunction {
+ private static final Logger logger = LoggerFactory.getLogger(ValidateRecordFilter.class);
+
+ public ValidateRecordFilter(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta) {
+ }
+
+ @Override
+ public boolean filter(Record record) {
+ if (ObjectUtil.isNull(record)) {
+ logger.warn("Record is null");
+ return false;
+ }
+ if (ObjectUtil.isNull(record.getSource())) {
+ logger.warn("Record Source is null");
+ return false;
+ }
+ if (ObjectUtil.isNull(record.getStatement())) {
+ logger.warn("Record Statement is null");
+ return false;
+ }
+ return true;
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverter.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverter.java
new file mode 100644
index 0000000..291eca0
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverter.java
@@ -0,0 +1,43 @@
+package com.lanyuanxiaoyao.service.sync.functions.type;
+
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.LoadingCache;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import java.io.Serializable;
+import java.util.Map;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.hudi.org.apache.avro.Schema;
+
+/**
+ * 处理类型相关内容
+ *
+ * @author ZhangJiacheng
+ * @date 2023-07-20
+ */
+public interface TypeConverter extends Serializable {
+ LoadingCache CACHE = Caffeine.newBuilder()
+ .build(version -> {
+ switch (version) {
+ case 1:
+ return new TypeConverterV2();
+ case 0:
+ default:
+ return new TypeConverterV1();
+ }
+ });
+
+ static TypeConverter getInstance(TableMeta meta) {
+ return getInstance(meta.getVersion());
+ }
+
+ static TypeConverter getInstance(Integer version) {
+ if (version == null) {
+ version = 0;
+ }
+ return CACHE.get(version);
+ }
+
+ Schema convertToSchema(TableMeta meta);
+
+ GenericRowData convertToGenericRowData(TableMeta meta, Schema schema, Map data);
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV1.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV1.java
new file mode 100644
index 0000000..3e36b7c
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV1.java
@@ -0,0 +1,53 @@
+package com.lanyuanxiaoyao.service.sync.functions.type;
+
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import java.util.List;
+import java.util.Map;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.hudi.org.apache.avro.Schema;
+import org.apache.hudi.org.apache.avro.SchemaBuilder;
+
+/**
+ * 第一版本的类型转换
+ * 所有字段类型都是 String
+ *
+ * @author ZhangJiacheng
+ * @date 2023-07-20
+ */
+public class TypeConverterV1 implements TypeConverter {
+ @Override
+ public Schema convertToSchema(TableMeta meta) {
+ SchemaBuilder.FieldAssembler fieldBuilder = SchemaBuilder.builder()
+ .record(meta.getTable())
+ .fields();
+ fieldBuilder.nullableBoolean(Constants.HUDI_DELETE_KEY_NAME, false);
+ fieldBuilder.nullableString(Constants.UNION_KEY_NAME, "");
+
+ meta.getFields().forEach(fieldMeta -> fieldBuilder.optionalString(fieldMeta.getName()));
+
+ fieldBuilder.nullableLong(Constants.UPDATE_TIMESTAMP_KEY_NAME, -1);
+ fieldBuilder.nullableString(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME, "");
+ return fieldBuilder.endRecord();
+ }
+
+ @Override
+ public GenericRowData convertToGenericRowData(TableMeta meta, Schema schema, Map data) {
+ List fields = schema.getFields();
+ GenericRowData row = new GenericRowData(fields.size());
+ for (int index = 0; index < fields.size(); index++) {
+ Schema.Field field = fields.get(index);
+ // 如果是telepg的话，字段名就要统一改成小写，上游不规范，下游擦屁股
+ Object value = data.getOrDefault(Constants.FIELD_COVERT.apply(meta, field.name()), null);
+ if (field.schema().getType().equals(Schema.Type.STRING)
+ || (field.schema().isUnion() && field.schema().getTypes().contains(Schema.create(Schema.Type.STRING)))
+ || value instanceof String) {
+ row.setField(index, StringData.fromString((String) value));
+ } else {
+ row.setField(index, value);
+ }
+ }
+ return row;
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV2.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV2.java
new file mode 100644
index 0000000..3e1b4c5
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/functions/type/TypeConverterV2.java
@@ -0,0 +1,141 @@
+package com.lanyuanxiaoyao.service.sync.functions.type;
+
+import cn.hutool.core.util.ObjectUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import java.math.BigDecimal;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.regex.Pattern;
+import org.apache.flink.table.data.DecimalData;
+import org.apache.flink.table.data.GenericRowData;
+import org.apache.flink.table.data.StringData;
+import org.apache.hudi.org.apache.avro.JsonProperties;
+import org.apache.hudi.org.apache.avro.LogicalTypes;
+import org.apache.hudi.org.apache.avro.Schema;
+
+import static org.apache.hudi.org.apache.avro.Schema.*;
+
+/**
+ * 类型转换第二版本
+ *
+ * @author ZhangJiacheng
+ * @date 2023-07-20
+ */
+public class TypeConverterV2 implements TypeConverter {
+ public static final Schema NULL_SCHEMA = create(Type.NULL);
+ public static final Schema BOOLEAN_SCHEMA = create(Type.BOOLEAN);
+ public static final Schema INT_SCHEMA = create(Type.INT);
+ public static final Schema LONG_SCHEMA = create(Type.LONG);
+ public static final Schema FLOAT_SCHEMA = create(Type.FLOAT);
+ public static final Schema DOUBLE_SCHEMA = create(Type.DOUBLE);
+ public static final Schema STRING_SCHEMA = create(Type.STRING);
+ public static final Function FIXED_SCHEMA = length -> createFixed("decimal_" + length, null, null, length);
+ public static final BiFunction DECIMAL_SCHEMA = (length, scala) -> LogicalTypes.decimal(length, scala).addToSchema(FIXED_SCHEMA.apply(length));
+ public static final BiFunction NULLABLE_DECIMAL_SCHEMA = (length, scala) -> createUnion(NULL_SCHEMA, DECIMAL_SCHEMA.apply(length, scala));
+ public static final Schema NULLABLE_BOOLEAN_SCHEMA = createUnion(NULL_SCHEMA, BOOLEAN_SCHEMA);
+ public static final Schema NULLABLE_INT_SCHEMA = createUnion(NULL_SCHEMA, INT_SCHEMA);
+ public static final Schema NULLABLE_LONG_SCHEMA = createUnion(NULL_SCHEMA, LONG_SCHEMA);
+ public static final Schema NULLABLE_FLOAT_SCHEMA = createUnion(NULL_SCHEMA, FLOAT_SCHEMA);
+ public static final Schema NULLABLE_DOUBLE_SCHEMA = createUnion(NULL_SCHEMA, DOUBLE_SCHEMA);
+ public static final Schema NULLABLE_STRING_SCHEMA = createUnion(NULL_SCHEMA, STRING_SCHEMA);
+ public static final Function NULLABLE_FIXED_SCHEMA = length -> createUnion(NULL_SCHEMA, FIXED_SCHEMA.apply(length));
+ private static final Pattern BOOLEAN_REGEX = Pattern.compile("^boolean|bool$");
+ private static final Pattern INT_REGEX = Pattern.compile("^(tinyint|smallint|int|smallserial|integer)(\\(\\d+\\))?$");
+ private static final Pattern LONG_REGEX = Pattern.compile("^(bigint unsigned)|((bigint|serial|long)(\\(\\d+\\))?)$");
+ private static final Pattern DATE_REGEX = Pattern.compile("^date|timestamp|timestamp without time zone|datetime|time$");
+ private static final Pattern FLOAT_REGEX = Pattern.compile("^float(\\(\\d+\\))?$");
+ private static final Pattern DOUBLE_REGEX = Pattern.compile("^double(\\(\\d+\\))?$");
+ private static final Pattern FIXED_REGEX = Pattern.compile("^(number|money|bigserial)(\\(\\d+\\))?$");
+ private static final Pattern DECIMAL_REGEX = Pattern.compile("^(double precision)|(decimal(\\(\\s*\\d+\\s*(,\\s*\\d+\\s*)?\\))?)$");
+ private static final Pattern NUMERIC_REGEX = Pattern.compile("^numeric(\\(\\s*\\d+\\s*(,\\s*\\d+\\s*)?\\))?$");
+ private static final Pattern STRING_REGEX = Pattern.compile("^(character varying|(long|medium)text)|((varchar|char|text|clob|binary|bit)(\\(\\d+\\))?)$");
+ private static final Pattern YYYYMMDD = Pattern.compile("\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}");
+ private static final DateTimeFormatter YYYYMMDD_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
+
+ @Override
+ public Schema convertToSchema(TableMeta meta) {
+ List fields = new ArrayList<>(meta.getFields().size() + 4);
+ fields.add(new Field(Constants.HUDI_DELETE_KEY_NAME, BOOLEAN_SCHEMA, null, false));
+ fields.add(new Field(Constants.UNION_KEY_NAME, STRING_SCHEMA, null, ""));
+
+ for (TableMeta.FieldMeta field : meta.getFields()) {
+ fields.add(new Field(field.getName(), convertType(field.getType(), field.getLength(), field.getScala()), null, JsonProperties.NULL_VALUE));
+ }
+
+ fields.add(new Field(Constants.UPDATE_TIMESTAMP_KEY_NAME, LONG_SCHEMA, null, -1));
+ fields.add(new Field(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME, STRING_SCHEMA, null, ""));
+ return Schema.createRecord(meta.getTable(), null, null, false, fields);
+ }
+
+ private Schema convertType(String type, Long length, Integer scala) {
+ type = type.trim().toLowerCase();
+ if (BOOLEAN_REGEX.matcher(type).matches()) {
+ return NULLABLE_BOOLEAN_SCHEMA;
+ } else if (STRING_REGEX.matcher(type).matches() || DATE_REGEX.matcher(type).matches()) {
+ return NULLABLE_STRING_SCHEMA;
+ } else if (INT_REGEX.matcher(type).matches()) {
+ return NULLABLE_INT_SCHEMA;
+ } else if (LONG_REGEX.matcher(type).matches()) {
+ return NULLABLE_LONG_SCHEMA;
+ } else if (FLOAT_REGEX.matcher(type).matches()) {
+ return NULLABLE_FLOAT_SCHEMA;
+ } else if (DOUBLE_REGEX.matcher(type).matches()) {
+ return NULLABLE_DOUBLE_SCHEMA;
+ } else if (FIXED_REGEX.matcher(type).matches()) {
+ return NULLABLE_DECIMAL_SCHEMA.apply(length.intValue(), 0);
+ } else if (DECIMAL_REGEX.matcher(type).matches() || NUMERIC_REGEX.matcher(type).matches()) {
+ if (ObjectUtil.isNull(scala)) {
+ return NULLABLE_DECIMAL_SCHEMA.apply(length.intValue(), 6);
+ } else {
+ return NULLABLE_DECIMAL_SCHEMA.apply(length.intValue(), scala);
+ }
+ } else {
+ throw new RuntimeException(Constants.LOG_POINT_FIELD_TYPE_NOT_FOUND + " Cannot find correct type for source type: " + type + " length: " + length);
+ }
+ }
+
+ @Override
+ public GenericRowData convertToGenericRowData(TableMeta meta, Schema schema, Map data) {
+ List fields = schema.getFields();
+ GenericRowData row = new GenericRowData(fields.size());
+ for (int index = 0; index < fields.size(); index++) {
+ Field field = fields.get(index);
+ Object value = data.getOrDefault(Constants.FIELD_COVERT.apply(meta, field.name()), null);
+ row.setField(index, covertValue(field.schema(), value));
+ }
+ return row;
+ }
+
+ private Object covertValue(Schema schema, Object value) {
+ if (ObjectUtil.isNull(value)) {
+ return value;
+ } else if (NULLABLE_BOOLEAN_SCHEMA.equals(schema) || BOOLEAN_SCHEMA.equals(schema)) {
+ return value instanceof String ? Boolean.valueOf((String) value) : value;
+ } else if (NULLABLE_INT_SCHEMA.equals(schema) || INT_SCHEMA.equals(schema)) {
+ return value instanceof String ? Integer.valueOf((String) value) : value;
+ } else if (NULLABLE_LONG_SCHEMA.equals(schema) || LONG_SCHEMA.equals(schema)) {
+ return value instanceof String ? Long.valueOf((String) value) : value;
+ } else if (NULLABLE_FLOAT_SCHEMA.equals(schema) || FLOAT_SCHEMA.equals(schema)) {
+ return value instanceof String ? Float.valueOf((String) value) : value;
+ } else if (NULLABLE_DOUBLE_SCHEMA.equals(schema) || DOUBLE_SCHEMA.equals(schema)) {
+ return value instanceof String ? Double.valueOf((String) value) : value;
+ } else if (NULLABLE_STRING_SCHEMA.equals(schema) || STRING_SCHEMA.equals(schema)) {
+ return StringData.fromString((String) value);
+ } else {
+ for (Schema type : schema.getTypes()) {
+ if (type.getLogicalType() instanceof LogicalTypes.Decimal) {
+ LogicalTypes.Decimal decimalType = (LogicalTypes.Decimal) type.getLogicalType();
+ int precision = decimalType.getPrecision();
+ int scala = decimalType.getScale();
+ return DecimalData.fromBigDecimal(new BigDecimal((String) value), precision, scala);
+ }
+ }
+ return value;
+ }
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/AbstractMetric.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/AbstractMetric.java
new file mode 100644
index 0000000..d0c333a
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/AbstractMetric.java
@@ -0,0 +1,123 @@
+package com.lanyuanxiaoyao.service.sync.metrics;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.http.HttpResponse;
+import cn.hutool.http.HttpUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.utils.JacksonUtils;
+import dev.failsafe.Failsafe;
+import dev.failsafe.RetryPolicy;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.atomic.LongAdder;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 指标基础类
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public abstract class AbstractMetric implements Metric {
+ private static final Logger logger = LoggerFactory.getLogger(AbstractMetric.class);
+ private static final ObjectMapper MAPPER = JacksonUtils.getMapper();
+ private static final RetryPolicy PUBLISH_RETRY = RetryPolicy.builder()
+ .handle(Exception.class)
+ .withDelay(Duration.ofSeconds(1))
+ .withMaxAttempts(5)
+ .build();
+ private final GlobalConfiguration globalConfiguration;
+ private final List lineCache = ListUtil.toList();
+ private final LongAdder autoPublishCount = new LongAdder();
+ private List requests = new ArrayList<>();
+
+ public AbstractMetric(GlobalConfiguration globalConfiguration) {
+ this.globalConfiguration = globalConfiguration;
+ }
+
+ public void setRequests(HttpMetricsRequest... requests) {
+ setRequests(ListUtil.toList(requests));
+ }
+
+ public void setRequests(List requests) {
+ this.requests = requests;
+ }
+
+ public void addRequest(HttpMetricsRequest request) {
+ this.requests.add(request);
+ }
+
+ public void addRequests(HttpMetricsRequest... requests) {
+ addRequests(ListUtil.toList(requests));
+ }
+
+ public void addRequests(List requests) {
+ this.requests.addAll(requests);
+ }
+
+ @Override
+ public void addTag(String key, String value) {
+ requests.forEach(request -> request.addTag(key, value));
+ }
+
+ @Override
+ public void makePoint(boolean autoPublish, int batch) {
+ if (!globalConfiguration.getMetricEnable()) {
+ return;
+ }
+ if (autoPublish) {
+ if (autoPublishCount.sum() >= batch) {
+ publish();
+ }
+ autoPublishCount.increment();
+ }
+ makePoint();
+ }
+
+ public synchronized void publish() {
+ if (!globalConfiguration.getMetricEnable()) {
+ return;
+ }
+ try {
+ requests.stream()
+ .filter(request -> !request.isEmpty())
+ .map(request -> {
+ try {
+ String data = MAPPER.writeValueAsString(request);
+ request.clear();
+ return data;
+ } catch (JsonProcessingException e) {
+ logger.warn("Parse metrics failure: " + request, e);
+ }
+ return null;
+ })
+ .filter(Objects::nonNull)
+ .forEach(lineCache::add);
+ if (lineCache.isEmpty()) {
+ return;
+ }
+ String lines = String.join("\n", lineCache);
+ logger.debug("Push metrics: \n{}", lines);
+ HttpResponse response = Failsafe.with(PUBLISH_RETRY)
+ .get(() -> HttpUtil.createPost(globalConfiguration.getMetricPublishUrl())
+ .body(lines)
+ .basicAuth(Constants.VICTORIA_USERNAME, Constants.VICTORIA_PASSWORD)
+ .timeout(globalConfiguration.getMetricPublishTimeout())
+ .execute());
+ if (response.isOk()) {
+ logger.debug("Metrics push success");
+ }
+ } catch (Throwable throwable) {
+ logger.warn("Push metrics failure, url: " + globalConfiguration.getMetricPublishUrl(), throwable);
+ } finally {
+ lineCache.clear();
+ autoPublishCount.reset();
+ }
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/CountMetric.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/CountMetric.java
new file mode 100644
index 0000000..3cc9055
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/CountMetric.java
@@ -0,0 +1,67 @@
+package com.lanyuanxiaoyao.service.sync.metrics;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.utils.MetricsUtils;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.LongAdder;
+
+/**
+ * 基础类
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public class CountMetric extends AbstractMetric {
+ private final LongAdder count = new LongAdder();
+
+ private final HttpMetricsRequest countMetrics;
+
+ public CountMetric(GlobalConfiguration globalConfiguration, String name) {
+ this(globalConfiguration, name, MapUtil.empty());
+ }
+
+ public CountMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta) {
+ this(globalConfiguration, name, MapUtil.builder(MetricsUtils.commonTags(job, meta)).build());
+ }
+
+ public CountMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta, String extraTagKey, String extraTagValue) {
+ this(globalConfiguration, name, job, meta, MapUtil.of(extraTagKey, extraTagValue));
+ }
+
+ public CountMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta, Map tags) {
+ this(globalConfiguration, name, MapUtil.builder(MetricsUtils.commonTags(job, meta))
+ .putAll(tags)
+ .build());
+ }
+
+ public CountMetric(GlobalConfiguration globalConfiguration, String name, Map tags) {
+ super(globalConfiguration);
+ countMetrics = new HttpMetricsRequest(
+ name + "_count",
+ MapUtil.builder().putAll(tags).build()
+ );
+ setRequests(countMetrics);
+ }
+
+ public void increment() {
+ count.increment();
+ }
+
+ @Override
+ public void makePoint() {
+ double count = this.count.doubleValue();
+ if (count != 0) {
+ countMetrics.addMetric(count);
+ }
+ }
+
+ @Override
+ public List getMetrics() {
+ return ListUtil.toList(countMetrics);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/HttpMetricsRequest.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/HttpMetricsRequest.java
new file mode 100644
index 0000000..43cd4d7
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/HttpMetricsRequest.java
@@ -0,0 +1,88 @@
+package com.lanyuanxiaoyao.service.sync.metrics;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
+import java.beans.Transient;
+import java.io.Serializable;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * 指标实体类
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public class HttpMetricsRequest implements Serializable {
+ private final Map metric;
+ private final List values;
+ private final List timestamps;
+
+ private final Lock lock = new ReentrantLock();
+
+ public HttpMetricsRequest(String name, Map metrics) {
+ this.metric = MapUtil.builder()
+ .put("__name__", name)
+ .build();
+ this.metric.putAll(metrics);
+ this.values = Collections.synchronizedList(ListUtil.list(true));
+ this.timestamps = Collections.synchronizedList(ListUtil.list(true));
+ }
+
+ public void addTag(String key, String value) {
+ this.metric.put(key, value);
+ }
+
+ public void addMetric(Double value) {
+ addMetric(value, Instant.now().toEpochMilli());
+ }
+
+ public void addMetric(Double value, Long timestamp) {
+ synchronized (this) {
+ values.add(value);
+ timestamps.add(timestamp);
+ }
+ }
+
+ public void clear() {
+ synchronized (this) {
+ this.values.clear();
+ this.timestamps.clear();
+ }
+ }
+
+ @Transient
+ public boolean isEmpty() {
+ return this.values.isEmpty() && this.timestamps.isEmpty();
+ }
+
+ @Transient
+ public boolean isNonEmpty() {
+ return !isEmpty();
+ }
+
+ public Map getMetric() {
+ return metric;
+ }
+
+ public List getValues() {
+ return values;
+ }
+
+ public List getTimestamps() {
+ return timestamps;
+ }
+
+ @Override
+ public String toString() {
+ return "MetricsItem{" +
+ "metrics=" + metric +
+ ", values=" + values +
+ ", timestamps=" + timestamps +
+ '}';
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/MessageSizeSizeMetric.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/MessageSizeSizeMetric.java
new file mode 100644
index 0000000..9acf771
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/MessageSizeSizeMetric.java
@@ -0,0 +1,77 @@
+package com.lanyuanxiaoyao.service.sync.metrics;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.utils.MetricsUtils;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.DoubleAdder;
+import java.util.concurrent.atomic.LongAdder;
+
+/**
+ * 基础类
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public class MessageSizeSizeMetric extends AbstractMetric {
+ private final LongAdder count = new LongAdder();
+ private final DoubleAdder size = new DoubleAdder();
+
+ private final HttpMetricsRequest sizeMetrics;
+ private final HttpMetricsRequest perMessageSizeMetrics;
+
+ public MessageSizeSizeMetric(GlobalConfiguration globalConfiguration, String name) {
+ this(globalConfiguration, name, MapUtil.empty());
+ }
+
+ public MessageSizeSizeMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta) {
+ this(globalConfiguration, name, MapUtil.builder(MetricsUtils.commonTags(job, meta)).build());
+ }
+
+ public MessageSizeSizeMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta, String extraTagKey, String extraTagValue) {
+ this(globalConfiguration, name, job, meta, MapUtil.of(extraTagKey, extraTagValue));
+ }
+
+ public MessageSizeSizeMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta, Map tags) {
+ this(globalConfiguration, name, MapUtil.builder(MetricsUtils.commonTags(job, meta))
+ .putAll(tags)
+ .build());
+ }
+
+ public MessageSizeSizeMetric(GlobalConfiguration globalConfiguration, String name, Map tags) {
+ super(globalConfiguration);
+ sizeMetrics = new HttpMetricsRequest(
+ name + "_total",
+ MapUtil.builder().putAll(tags).build()
+ );
+ perMessageSizeMetrics = new HttpMetricsRequest(
+ name + "_per_message",
+ MapUtil.builder().putAll(tags).build()
+ );
+ setRequests(sizeMetrics, perMessageSizeMetrics);
+ }
+
+ public void increment(long size) {
+ this.count.increment();
+ this.size.add(size);
+ }
+
+ @Override
+ public void makePoint() {
+ double count = this.count.doubleValue();
+ double size = this.size.doubleValue();
+ if (size != 0 && count != 0) {
+ sizeMetrics.addMetric(size);
+ perMessageSizeMetrics.addMetric(size / count);
+ }
+ }
+
+ @Override
+ public List getMetrics() {
+ return ListUtil.toList(sizeMetrics);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/Metric.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/Metric.java
new file mode 100644
index 0000000..145b539
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/Metric.java
@@ -0,0 +1,20 @@
+package com.lanyuanxiaoyao.service.sync.metrics;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * 指标类定义
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public interface Metric extends Serializable {
+ void addTag(String key, String value);
+
+ void makePoint(boolean autoPublish, int batch);
+
+ void makePoint();
+
+ List getMetrics();
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/RateMetric.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/RateMetric.java
new file mode 100644
index 0000000..173e152
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/metrics/RateMetric.java
@@ -0,0 +1,87 @@
+package com.lanyuanxiaoyao.service.sync.metrics;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.utils.MetricsUtils;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.LongAdder;
+
+/**
+ * 基础类
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-13
+ */
+public class RateMetric extends AbstractMetric {
+ private final LongAdder count = new LongAdder();
+
+ private final HttpMetricsRequest countMetrics;
+ private final HttpMetricsRequest millisecondMetrics;
+ private final HttpMetricsRequest perSecondMetrics;
+
+ private final Instant startInstant;
+
+ public RateMetric(GlobalConfiguration globalConfiguration, String name) {
+ this(globalConfiguration, name, MapUtil.empty());
+ }
+
+ public RateMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta) {
+ this(globalConfiguration, name, MapUtil.builder(MetricsUtils.commonTags(job, meta)).build());
+ }
+
+ public RateMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta, String extraTagKey, String extraTagValue) {
+ this(globalConfiguration, name, job, meta, MapUtil.of(extraTagKey, extraTagValue));
+ }
+
+ public RateMetric(GlobalConfiguration globalConfiguration, String name, FlinkJob job, TableMeta meta, Map tags) {
+ this(globalConfiguration, name, MapUtil.builder(MetricsUtils.commonTags(job, meta))
+ .putAll(tags)
+ .build());
+ }
+
+ public RateMetric(GlobalConfiguration globalConfiguration, String name, Map tags) {
+ super(globalConfiguration);
+
+ startInstant = Instant.now();
+
+ countMetrics = new HttpMetricsRequest(
+ name + "_count",
+ MapUtil.builder().putAll(tags).build()
+ );
+ millisecondMetrics = new HttpMetricsRequest(
+ name + "_millisecond",
+ MapUtil.builder().putAll(tags).build()
+ );
+ perSecondMetrics = new HttpMetricsRequest(
+ name + "_per_millisecond",
+ MapUtil.builder().putAll(tags).build()
+ );
+ setRequests(countMetrics, millisecondMetrics, perSecondMetrics);
+ }
+
+ public void increment() {
+ count.increment();
+ }
+
+ @Override
+ public void makePoint() {
+ double count = this.count.doubleValue();
+ if (count != 0) {
+ long millis = Duration.between(startInstant, Instant.now()).toMillis();
+ countMetrics.addMetric(count);
+ millisecondMetrics.addMetric((double) millis);
+ perSecondMetrics.addMetric(count / millis);
+ }
+ }
+
+ @Override
+ public List getMetrics() {
+ return ListUtil.toList(countMetrics, millisecondMetrics, perSecondMetrics);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/ArgumentsUtils.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/ArgumentsUtils.java
new file mode 100644
index 0000000..bc525c0
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/ArgumentsUtils.java
@@ -0,0 +1,98 @@
+package com.lanyuanxiaoyao.service.sync.utils;
+
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.exception.MissingArgumentException;
+import java.util.List;
+import org.apache.flink.api.java.utils.ParameterTool;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.type.TypeReference;
+
+/**
+ * 入参解析相关内容
+ *
+ * @author ZhangJiacheng
+ * @date 2022-03-10
+ */
+public class ArgumentsUtils {
+ public static long getJobId(String[] args) throws MissingArgumentException {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.JOB_ID)) {
+ throw new MissingArgumentException(Constants.JOB_ID);
+ }
+ return argsTool.getLong(Constants.JOB_ID);
+ }
+
+ public static String getTable(String[] args) throws MissingArgumentException {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.TABLE_NAME)) {
+ throw new MissingArgumentException(Constants.TABLE_NAME);
+ }
+ return argsTool.get(Constants.TABLE_NAME);
+ }
+
+ public static Boolean getServiceMode(String[] args) throws Exception {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.SERVICE_MODE)) {
+ return false;
+ }
+ return argsTool.getBoolean(Constants.SERVICE_MODE);
+ }
+
+ public static String getMessageId(String[] args) throws MissingArgumentException {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.MESSAGE_ID)) {
+ throw new MissingArgumentException(Constants.MESSAGE_ID);
+ }
+ return argsTool.get(Constants.MESSAGE_ID);
+ }
+
+ public static TableMeta getTableMeta(String[] args) throws Exception {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.TABLE_META)) {
+ throw new MissingArgumentException(Constants.TABLE_META);
+ }
+ return JacksonUtils.getMapper().readValue(argsTool.get(Constants.TABLE_META), TableMeta.class);
+ }
+
+ public static List getTableMetaList(String[] args) throws Exception {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.TABLE_META_LIST)) {
+ throw new MissingArgumentException(Constants.TABLE_META_LIST);
+ }
+ return JacksonUtils.getMapper().readValue(argsTool.get(Constants.TABLE_META_LIST), new TypeReference>() {});
+ }
+
+ public static FlinkJob getFlinkJob(String[] args) throws MissingArgumentException, JsonProcessingException {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.FLINK_JOB)) {
+ throw new MissingArgumentException(Constants.FLINK_JOB);
+ }
+ return JacksonUtils.getMapper().readValue(argsTool.get(Constants.FLINK_JOB), FlinkJob.class);
+ }
+
+ public static String getInstants(String[] args) throws JsonProcessingException {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.INSTANTS)) {
+ return "";
+ }
+ return argsTool.get(Constants.INSTANTS);
+ }
+
+ public static Boolean getBetaMode(String[] args) {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.BETA)) {
+ return false;
+ }
+ return argsTool.getBoolean(Constants.BETA);
+ }
+
+ public static String getCluster(String[] args) {
+ ParameterTool argsTool = ParameterTool.fromArgs(args);
+ if (!argsTool.has(Constants.CLUSTER)) {
+ return "";
+ }
+ return argsTool.get(Constants.CLUSTER);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/EnvUtils.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/EnvUtils.java
new file mode 100644
index 0000000..eac2794
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/EnvUtils.java
@@ -0,0 +1,22 @@
+package com.lanyuanxiaoyao.service.sync.utils;
+
+import com.lanyuanxiaoyao.service.common.Constants;
+
+/**
+ * 环境变量相关参数
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-21
+ */
+public class EnvUtils {
+ public static void setEnv() {
+ }
+
+ public static void setCompactionEnv() {
+ System.setProperty(Constants.METRICS_LABEL_RUN_TYPE, Constants.METRICS_RUN_TYPE_COMPACTION);
+ }
+
+ public static void setSyncEnv() {
+ System.setProperty(Constants.METRICS_LABEL_RUN_TYPE, Constants.METRICS_RUN_TYPE_SYNC);
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/JacksonUtils.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/JacksonUtils.java
new file mode 100644
index 0000000..afd5d60
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/JacksonUtils.java
@@ -0,0 +1,31 @@
+package com.lanyuanxiaoyao.service.sync.utils;
+
+import cn.hutool.core.util.ObjectUtil;
+import java.io.Serializable;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.DeserializationFeature;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.MapperFeature;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Json 解析相关工具
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-12
+ */
+public class JacksonUtils implements Serializable {
+ private static final Logger logger = LoggerFactory.getLogger(JacksonUtils.class);
+
+ private static ObjectMapper INSTANCE = null;
+
+ public static ObjectMapper getMapper() {
+ if (ObjectUtil.isNull(INSTANCE)) {
+ INSTANCE = new ObjectMapper();
+ INSTANCE.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true);
+ INSTANCE.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+ INSTANCE.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
+ }
+ return INSTANCE;
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/LoadBalance.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/LoadBalance.java
new file mode 100644
index 0000000..51182b0
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/LoadBalance.java
@@ -0,0 +1,29 @@
+package com.lanyuanxiaoyao.service.sync.utils;
+
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.RandomUtil;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 动态平衡获取指标 url
+ *
+ * @author ZhangJiacheng
+ * @date 2022-07-05
+ */
+public class LoadBalance {
+ private static final Logger logger = LoggerFactory.getLogger(LoadBalance.class);
+ private static String[] urls = null;
+ private static int length = 0;
+
+ public static String getCustomPublishUrl(GlobalConfiguration globalConfiguration) {
+ if (ObjectUtil.isNull(urls)) {
+ urls = globalConfiguration.getMetricsPublishCustomUrl().split(",");
+ length = urls.length;
+ }
+ String url = urls[RandomUtil.randomInt(length)];
+ logger.info("Random url: {}", url);
+ return url;
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/MetricsUtils.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/MetricsUtils.java
new file mode 100644
index 0000000..b089628
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/MetricsUtils.java
@@ -0,0 +1,64 @@
+package com.lanyuanxiaoyao.service.sync.utils;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.map.MapUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.metrics.AbstractMetric;
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 指标工具
+ *
+ * @author ZhangJiacheng
+ * @date 2022-06-12
+ */
+public class MetricsUtils implements Serializable {
+ private static final Logger logger = LoggerFactory.getLogger(MetricsUtils.class);
+
+ public static Map commonTags(FlinkJob job, TableMeta meta) {
+ return MapUtil.builder()
+ .put(Constants.METRICS_LABEL_FLINK_JOB_ID, job.getId().toString())
+ .put(Constants.METRICS_LABEL_FLINK_JOB_NAME, job.getName())
+ .put(Constants.METRICS_LABEL_SCHEMA, meta.getSchema())
+ .put(Constants.METRICS_LABEL_TABLE, meta.getTable())
+ .put(Constants.METRICS_LABEL_ALIAS, meta.getAlias())
+ .build();
+ }
+
+ @SafeVarargs
+ public static void createMakePointTimer(GlobalConfiguration globalConfiguration, T... metrics) {
+ createMakePointTimer(globalConfiguration, ListUtil.toList(metrics));
+ }
+
+ public static void createMakePointTimer(GlobalConfiguration globalConfiguration, List metrics) {
+ logger.info("Create timer: {}", metrics);
+ new Timer().schedule(new TimerTask() {
+ @Override
+ public void run() {
+ for (AbstractMetric metric : metrics) {
+ metric.makePoint(true, globalConfiguration.getMetricPublishBatch());
+ }
+ }
+ }, globalConfiguration.getMetricPublishDelay(), globalConfiguration.getMetricPublishPeriod());
+ }
+
+ @SafeVarargs
+ public static void publishAllMetrics(T... metrics) {
+ publishAllMetrics(ListUtil.toList(metrics));
+ }
+
+ public static void publishAllMetrics(List metrics) {
+ for (AbstractMetric metric : metrics) {
+ metric.publish();
+ }
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/StatusUtils.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/StatusUtils.java
new file mode 100644
index 0000000..3f782e5
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/StatusUtils.java
@@ -0,0 +1,271 @@
+package com.lanyuanxiaoyao.service.sync.utils;
+
+import cn.hutool.core.util.EnumUtil;
+import cn.hutool.core.util.ObjectUtil;
+import cn.hutool.core.util.StrUtil;
+import cn.hutool.http.HttpUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.configuration.RetryPolicyProvider;
+import dev.failsafe.Failsafe;
+import java.time.Instant;
+import java.util.Map;
+import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.hudi.common.model.HoodieCommitMetadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * 状态输出
+ *
+ * @author ZhangJiacheng
+ * @date 2022-07-05
+ */
+public class StatusUtils {
+ private static final Logger logger = LoggerFactory.getLogger(StatusUtils.class);
+
+ private static final ObjectMapper MAPPER = JacksonUtils.getMapper();
+ private static final int HTTP_TIMEOUT = (int) Constants.MINUTE;
+
+ public static void syncStart(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta) {
+ logger.info("Enter method: syncStart[configuration, flinkJob, tableMeta]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta);
+ try {
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() ->
+ HttpUtil.createGet(
+ StrUtil.format(
+ "{}/api/sync_start?flink_job_id={}&alias={}&database={}&schema={}&table={}&cluster={}&application_id={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ tableMeta.getSource(),
+ tableMeta.getSchema(),
+ tableMeta.getTable(),
+ configuration.getCluster(),
+ configuration.getApplicationId()
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ );
+ } catch (Exception e) {
+ logger.warn("sync start metrics submit failure");
+ }
+ }
+
+ public static void syncCheckpoint(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, String messageId, Long publishTime) {
+ logger.info("Enter method: syncCheckpoint[configuration, flinkJob, tableMeta, messageId, publishTime]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta + "," + "messageId:" + messageId + "," + "publishTime:" + publishTime);
+ try {
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() -> HttpUtil.createGet(
+ StrUtil.format(
+ "{}/api/sync_checkpoint_state?flink_job_id={}&alias={}&message_id={}&publish_time={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ messageId,
+ publishTime
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ );
+ } catch (Exception e) {
+ logger.warn("sync checkpoint metrics submit failure");
+ }
+ }
+
+ public static void syncOperation(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta) {
+ syncOperation(configuration, flinkJob, tableMeta, null);
+ }
+
+ public static void syncOperation(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, Long operationTime) {
+ logger.info("Enter method: syncOperation[configuration, flinkJob, tableMeta, operationTime]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta + "," + "operationTime:" + operationTime);
+ try {
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() -> {
+ if (ObjectUtil.isNull(operationTime)) {
+ HttpUtil.createGet(
+ StrUtil.format(
+ "{}/api/sync_operation_state?flink_job_id={}&alias={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias()
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .timeout(HTTP_TIMEOUT)
+ .execute();
+ } else {
+ HttpUtil.createGet(
+ StrUtil.format(
+ "{}/api/sync_operation_state?flink_job_id={}&alias={}&operation_time={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ operationTime
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .timeout(HTTP_TIMEOUT)
+ .execute();
+ }
+ });
+ } catch (Exception e) {
+ logger.warn("sync operation metrics submit failure");
+ }
+ }
+
+ public static void compactionStart(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta) {
+ logger.info("Enter method: compactionStart[configuration, flinkJob, tableMeta]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta);
+ try {
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() ->
+ HttpUtil.createGet(
+ StrUtil.format(
+ "{}/api/compaction_start?flink_job_id={}&alias={}&type={}&cluster={}&application_id={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ EnumUtil.toString(tableMeta.getSourceType()),
+ configuration.getCluster(),
+ configuration.getApplicationId()
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ );
+ } catch (Exception e) {
+ logger.warn("compaction start metrics submit failure");
+ }
+ }
+
+ public static void compactionPreCommit(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, String instant, Map metadata) {
+ logger.info("Enter method: compactionPreCommit[configuration, flinkJob, tableMeta, instant, metadata]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta + "," + "instant:" + instant + "," + "metadata:" + metadata);
+ try {
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() -> HttpUtil.createPost(
+ StrUtil.format(
+ "{}/api/compaction_pre_commit?flink_job_id={}&alias={}&instant={}&cluster={}&application_id={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ instant,
+ configuration.getCluster(),
+ configuration.getApplicationId()
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .body(MAPPER.writeValueAsString(metadata))
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ );
+ } catch (Exception e) {
+ logger.warn("compaction pre commit metrics submit failure");
+ }
+ }
+
+ public static void compactionCommit(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, String instant, HoodieCommitMetadata metadata) {
+ logger.info("Enter method: compactionCommit[configuration, flinkJob, tableMeta, instant, metadata]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta + "," + "instant:" + instant + "," + "metadata:" + metadata);
+ try {
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() -> HttpUtil.createPost(
+ StrUtil.format(
+ "{}/api/compaction_commit?flink_job_id={}&alias={}&instant={}&cluster={}&application_id={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ instant,
+ configuration.getCluster(),
+ configuration.getApplicationId()
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .body(MAPPER.writeValueAsString(metadata))
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ );
+ } catch (Exception e) {
+ logger.warn("compaction commit metrics submit failure");
+ }
+ }
+
+ public static void compactionFinish(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, String message, Exception exception) {
+ logger.info("Enter method: compactionFinish[configuration, flinkJob, tableMeta, message, exception]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta + "," + "message:" + message + "," + "exception:" + exception);
+ try {
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() -> {
+ boolean success = (exception == null);
+ HttpUtil.createPost(StrUtil.format(
+ "{}/api/compaction_finish?flink_job_id={}&alias={}&time={}&state={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ Instant.now().toEpochMilli(),
+ success
+ ))
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .body(success ? message == null ? "" : message : exception.toString(), "text/plain")
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ .close();
+ });
+ } catch (Exception e) {
+ logger.warn("compaction finish metrics submit failure");
+ }
+ }
+
+ public static void versionUpdate(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, String version, String opts) {
+ logger.info("Enter method: versionUpdate[configuration, flinkJob, tableMeta, version, opts]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta + "," + "version:" + version + "," + "opts:" + opts);
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() ->
+ HttpUtil.createGet(
+ StrUtil.format(
+ "{}/api/version_update?flink_job_id={}&alias={}&version={}&opts={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ version,
+ opts
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ );
+ }
+
+ public static void compactionLatestOpTs(GlobalConfiguration configuration, FlinkJob flinkJob, TableMeta tableMeta, Long latestOpTs) {
+ logger.info("Enter method: compactionLatestOpTs[configuration, flinkJob, tableMeta, latestOpTs]. " + "configuration:" + configuration + "," + "flinkJob:" + flinkJob + "," + "tableMeta:" + tableMeta + "," + "latestOpTs:" + latestOpTs);
+ Failsafe.with(RetryPolicyProvider.HTTP_RETRY)
+ .run(() ->
+ HttpUtil.createGet(
+ StrUtil.format(
+ "{}/api/compaction_latest_operation_time?flink_job_id={}&alias={}&latest_op_ts={}",
+ LoadBalance.getCustomPublishUrl(configuration),
+ flinkJob.getId(),
+ tableMeta.getAlias(),
+ latestOpTs
+ )
+ )
+ .header(Constants.API_HEADER_NAME, Constants.API_VERSION)
+ .basicAuth(Constants.SPRING_SECURITY_USERNAME, Constants.SPRING_SECURITY_PASSWORD_PLAIN)
+ .timeout(HTTP_TIMEOUT)
+ .execute()
+ );
+ }
+}
diff --git a/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/SyncUtils.java b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/SyncUtils.java
new file mode 100644
index 0000000..bb84c8f
--- /dev/null
+++ b/utils/sync/src/main/java/com/lanyuanxiaoyao/service/sync/utils/SyncUtils.java
@@ -0,0 +1,257 @@
+package com.lanyuanxiaoyao.service.sync.utils;
+
+import cn.hutool.core.collection.ListUtil;
+import cn.hutool.core.util.EnumUtil;
+import cn.hutool.core.util.StrUtil;
+import com.lanyuanxiaoyao.service.common.Constants;
+import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
+import com.lanyuanxiaoyao.service.common.entity.Record;
+import com.lanyuanxiaoyao.service.common.entity.TableMeta;
+import com.lanyuanxiaoyao.service.common.utils.TableMetaHelper;
+import com.lanyuanxiaoyao.service.sync.configuration.DefaultPartitionNameKeyGenerator;
+import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
+import com.lanyuanxiaoyao.service.sync.configuration.TraceOverwriteWithLatestAvroPayload;
+import com.lanyuanxiaoyao.service.sync.configuration.TraceWriteStatus;
+import com.lanyuanxiaoyao.service.sync.functions.OperationTypeFilter;
+import com.lanyuanxiaoyao.service.sync.functions.Record2RowDataFunction;
+import com.lanyuanxiaoyao.service.sync.functions.type.TypeConverter;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import org.apache.flink.api.common.typeinfo.TypeHint;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.streaming.api.datastream.DataStream;
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
+import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
+import org.apache.flink.table.data.RowData;
+import org.apache.flink.table.types.logical.RowType;
+import org.apache.hudi.common.model.HoodieCleaningPolicy;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
+import org.apache.hudi.common.table.view.FileSystemViewStorageType;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieCompactionConfig;
+import org.apache.hudi.config.HoodieStorageConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsConfig;
+import org.apache.hudi.config.metrics.HoodieMetricsVictoriaConfig;
+import org.apache.hudi.configuration.FlinkOptions;
+import org.apache.hudi.configuration.OptionsResolver;
+import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.metrics.MetricsReporterType;
+import org.apache.hudi.org.apache.avro.Schema;
+import org.apache.hudi.sink.utils.Pipelines;
+import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy;
+import org.apache.hudi.util.AvroSchemaConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.lanyuanxiaoyao.service.common.Constants.HOUR;
+
+/**
+ * Flink 相关的工具
+ *
+ * @author lanyuanxiaoyao
+ * @version 0.0.2
+ * @date 2022-04-20
+ */
+@SuppressWarnings("UnusedAssignment")
+public class SyncUtils {
+ private static final Logger logger = LoggerFactory.getLogger(SyncUtils.class);
+
+ private static final long K = 1024;
+ private static final long M = 1024 * K;
+ private static final long G = 1024 * M;
+
+ public static Schema avroSchemaWithExtraFields(TableMeta meta) {
+ return TypeConverter.getInstance(meta).convertToSchema(meta);
+ }
+
+ public static Configuration getSyncFlinkConfiguration(GlobalConfiguration globalConfiguration, Configuration inputConfiguration, FlinkJob flinkJob, TableMeta tableMeta, Schema schema, Integer defaultParallelism) {
+ Configuration configuration = new Configuration();
+ if (inputConfiguration != null) {
+ configuration = inputConfiguration;
+ }
+
+ configuration.setBoolean(HoodieMetricsConfig.TURN_METRICS_ON.key(), false);
+ configuration.setString(HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key(), MetricsReporterType.VICTORIA.name());
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_ENDPOINT.key(), globalConfiguration.getMetricPublishPrometheusUrl());
+ configuration.setInteger(HoodieMetricsVictoriaConfig.VICTORIA_TIMEOUT.key(), 60000);
+ configuration.setBoolean(HoodieMetricsVictoriaConfig.VICTORIA_BASIC_AUTH_ENABLE.key(), true);
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_BASIC_AUTH_USERNAME.key(), Constants.VICTORIA_USERNAME);
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_BASIC_AUTH_PASSWORD.key(), Constants.VICTORIA_PASSWORD);
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_TAGS.key(), ListUtil.toList(
+ Pair.of(Constants.METRICS_LABEL_RUN_TYPE, Constants.METRICS_RUN_TYPE_SYNC),
+ Pair.of(Constants.METRICS_LABEL_FLINK_JOB_ID, flinkJob.getId()),
+ Pair.of(Constants.METRICS_LABEL_FLINK_JOB_NAME, flinkJob.getName().replaceAll("\\s", "_")),
+ Pair.of(Constants.METRICS_LABEL_SCHEMA, tableMeta.getSchema()),
+ Pair.of(Constants.METRICS_LABEL_TABLE, tableMeta.getTable()),
+ Pair.of(Constants.METRICS_LABEL_ALIAS, tableMeta.getAlias())
+ ).stream().map(pair -> StrUtil.format("{}={}", pair.getLeft(), pair.getRight())).collect(Collectors.joining(";")));
+
+ return getFlinkConfiguration(configuration, tableMeta, schema, defaultParallelism);
+ }
+
+ public static Configuration getCompactionFlinkConfiguration(GlobalConfiguration globalConfiguration, Configuration inputConfiguration, FlinkJob flinkJob, TableMeta tableMeta, Schema schema, Integer defaultParallelism) {
+ Configuration configuration = new Configuration();
+ if (inputConfiguration != null) {
+ configuration = inputConfiguration;
+ }
+
+ configuration.setBoolean(HoodieMetricsConfig.TURN_METRICS_ON.key(), false);
+ configuration.setString(HoodieMetricsConfig.METRICS_REPORTER_TYPE_VALUE.key(), MetricsReporterType.VICTORIA.name());
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_ENDPOINT.key(), globalConfiguration.getMetricPublishPrometheusUrl());
+ configuration.setInteger(HoodieMetricsVictoriaConfig.VICTORIA_TIMEOUT.key(), 60000);
+ configuration.setBoolean(HoodieMetricsVictoriaConfig.VICTORIA_BASIC_AUTH_ENABLE.key(), true);
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_BASIC_AUTH_USERNAME.key(), Constants.VICTORIA_USERNAME);
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_BASIC_AUTH_PASSWORD.key(), Constants.VICTORIA_PASSWORD);
+ configuration.setString(HoodieMetricsVictoriaConfig.VICTORIA_TAGS.key(), ListUtil.toList(
+ Pair.of(Constants.METRICS_LABEL_RUN_TYPE, Constants.METRICS_RUN_TYPE_COMPACTION),
+ Pair.of(Constants.METRICS_LABEL_FLINK_JOB_ID, flinkJob.getId()),
+ Pair.of(Constants.METRICS_LABEL_FLINK_JOB_NAME, flinkJob.getName().replaceAll("\\s", "_")),
+ Pair.of(Constants.METRICS_LABEL_SCHEMA, tableMeta.getSchema()),
+ Pair.of(Constants.METRICS_LABEL_TABLE, tableMeta.getTable()),
+ Pair.of(Constants.METRICS_LABEL_ALIAS, tableMeta.getAlias())
+ ).stream().map(pair -> StrUtil.format("{}={}", pair.getLeft(), pair.getRight())).collect(Collectors.joining(";")));
+
+ return getFlinkConfiguration(configuration, tableMeta, schema, defaultParallelism);
+ }
+
+ public static Configuration getFlinkConfiguration(Configuration inputConfiguration, TableMeta tableMeta, Schema schema, Integer defaultParallelism) {
+ Configuration configuration = new Configuration();
+ if (inputConfiguration != null) {
+ configuration = inputConfiguration;
+ }
+ String tableType = tableMeta.getHudi().getTargetTableType();
+ logger.info("Hudi table type: {}", tableMeta.getHudi().getTargetTableType());
+ // 基本信息
+ configuration.setString(FlinkOptions.TABLE_NAME, tableMeta.getHudi().getTargetTable());
+ configuration.setString(FlinkOptions.TABLE_TYPE, tableType);
+ configuration.setString(FlinkOptions.PATH, tableMeta.getHudi().getTargetHdfsPath());
+ configuration.setString(FlinkOptions.RECORD_KEY_FIELD, Constants.UNION_KEY_NAME);
+ configuration.setBoolean(FlinkOptions.PRE_COMBINE, false);
+ if (TableMetaHelper.existsTag(tableMeta, Constants.TAGS_PRE_COMBINE)) {
+ configuration.setBoolean(FlinkOptions.PRE_COMBINE, true);
+ }
+ configuration.setString(FlinkOptions.PRECOMBINE_FIELD, Constants.UPDATE_TIMESTAMP_KEY_NAME);
+ configuration.setString(FlinkOptions.SOURCE_AVRO_SCHEMA, schema.toString());
+
+ if (TableMetaHelper.existsTag(tableMeta, Constants.TAGS_NO_IGNORE_FAILED)) {
+ configuration.setBoolean(FlinkOptions.IGNORE_FAILED, false);
+ }
+
+ configuration.setString(FlinkOptions.PARTITION_DEFAULT_NAME, "default");
+ configuration.setString(FlinkOptions.KEYGEN_CLASS_NAME, DefaultPartitionNameKeyGenerator.class.getName());
+
+ Optional partitionPath = TableMetaHelper.getPartitionField(tableMeta);
+ logger.info("Partition field: {}", partitionPath.orElse(""));
+ if (partitionPath.isPresent()) {
+ configuration.setString(FlinkOptions.PARTITION_PATH_FIELD, partitionPath.get());
+ }
+
+ if (TableMetaHelper.existsTag(tableMeta, Constants.TAGS_TRACE_LATEST_OP_TS)) {
+ logger.info("Enable trace latest op ts");
+ configuration.setString(FlinkOptions.PAYLOAD_CLASS_NAME, TraceOverwriteWithLatestAvroPayload.class.getName());
+ configuration.setString(HoodieWriteConfig.WRITE_STATUS_CLASS_NAME.key(), TraceWriteStatus.class.getName());
+ }
+
+ configuration.setBoolean(FlinkOptions.METADATA_ENABLED, false);
+ configuration.setInteger(HoodieStorageConfig.LOGFILE_DATA_BLOCK_MAX_SIZE.key(), Integer.MAX_VALUE);
+ configuration.setString(FileSystemViewStorageConfig.SECONDARY_VIEW_TYPE.key(), FileSystemViewStorageType.SPILLABLE_DISK.name());
+
+ // Write
+ configuration.setInteger(FlinkOptions.WRITE_TASKS, tableMeta.getHudi().getWriteTasks() == 0 ? defaultParallelism : tableMeta.getHudi().getWriteTasks());
+ configuration.setInteger(FlinkOptions.WRITE_MERGE_MAX_MEMORY, 0);
+ configuration.setDouble(FlinkOptions.WRITE_TASK_MAX_SIZE, tableMeta.getHudi().getWriteTaskMaxMemory() == 0 ? FlinkOptions.WRITE_TASK_MAX_SIZE.defaultValue() : tableMeta.getHudi().getWriteTaskMaxMemory());
+ configuration.setDouble(FlinkOptions.WRITE_BATCH_SIZE, tableMeta.getHudi().getWriteBatchSize() == 0 ? FlinkOptions.WRITE_BATCH_SIZE.defaultValue() : tableMeta.getHudi().getWriteBatchSize());
+ configuration.setLong(FlinkOptions.WRITE_RATE_LIMIT, tableMeta.getHudi().getWriteRateLimit());
+ configuration.setLong(FlinkOptions.WRITE_COMMIT_ACK_TIMEOUT, HOUR);
+
+ // 索引
+ configuration.setString(FlinkOptions.INDEX_TYPE, HoodieIndex.IndexType.BUCKET.name());
+ configuration.setInteger(FlinkOptions.BUCKET_INDEX_NUM_BUCKETS, tableMeta.getHudi().getBucketIndexNumber() == 0 ? 50 : tableMeta.getHudi().getBucketIndexNumber());
+ configuration.setString(FlinkOptions.INDEX_KEY_FIELD, Constants.UNION_KEY_NAME);
+
+ configuration.setBoolean(FlinkOptions.INDEX_BOOTSTRAP_ENABLED, false);
+ configuration.setBoolean(FlinkOptions.INDEX_GLOBAL_ENABLED, false);
+ configuration.setDouble(FlinkOptions.INDEX_STATE_TTL, -1);
+
+ // 增大 就 OOM
+ // configuration.setDouble(HoodieMemoryConfig.MAX_DFS_STREAM_BUFFER_SIZE.key(), 64 * M);
+ // 增大 就 OOM
+ // configuration.setDouble(HoodieStorageConfig.LOGFILE_DATA_BLOCK_MAX_SIZE.key(), 128 * M);
+
+ // Compaction
+ configuration.setBoolean(FlinkOptions.COMPACTION_ASYNC_ENABLED, false);
+ if (EnumUtil.equals(HoodieTableType.COPY_ON_WRITE, tableType)) {
+ configuration.setBoolean(FlinkOptions.COMPACTION_ASYNC_ENABLED, true);
+ }
+ configuration.setBoolean(FlinkOptions.COMPACTION_SCHEDULE_ENABLED, true);
+ configuration.setInteger(FlinkOptions.COMPACTION_TASKS, tableMeta.getHudi().getCompactionTasks());
+ configuration.setString(FlinkOptions.COMPACTION_TRIGGER_STRATEGY, StrUtil.isBlank(tableMeta.getHudi().getCompactionStrategy()) ? FlinkOptions.NUM_OR_TIME : tableMeta.getHudi().getCompactionStrategy());
+ configuration.setInteger(FlinkOptions.COMPACTION_MAX_MEMORY, 1024);
+ configuration.setInteger(FlinkOptions.COMPACTION_DELTA_SECONDS, tableMeta.getHudi().getCompactionDeltaSeconds() == 0 ? 15 * 60 : tableMeta.getHudi().getCompactionDeltaSeconds());
+ configuration.setInteger(FlinkOptions.COMPACTION_DELTA_COMMITS, tableMeta.getHudi().getCompactionDeltaCommits() == 0 ? 5 : tableMeta.getHudi().getCompactionDeltaCommits());
+
+ configuration.setString(HoodieCompactionConfig.COMPACTION_STRATEGY.key(), UnBoundedCompactionStrategy.class.getName());
+ // configuration.setString(HoodieCompactionConfig.COMPACTION_STRATEGY.key(), CombineAllCompactionStrategy.class.getName());
+ // configuration.setBoolean(FlinkOptions.COMPACTION_SCHEDULE_ENABLED, true);
+ // configuration.setInteger(FlinkOptions.COMPACTION_TASKS, tableMeta.getHudi().getCompactionTasks() == 0 ? defaultParallelism : tableMeta.getHudi().getCompactionTasks());
+ // configuration.setInteger(FlinkOptions.COMPACTION_MAX_MEMORY, tableMeta.getHudi().getCompactionMaxMemory());
+ // configuration.setString(FlinkOptions.COMPACTION_TRIGGER_STRATEGY, tableMeta.getHudi().getCompactionStrategy());
+ // configuration.setInteger(FlinkOptions.COMPACTION_DELTA_COMMITS, tableMeta.getHudi().getCompactionDeltaCommits());
+ // configuration.setInteger(FlinkOptions.COMPACTION_DELTA_SECONDS, tableMeta.getHudi().getCompactionDeltaSeconds());
+
+ // 时间线保留个数
+ configuration.setInteger(FlinkOptions.CLEAN_RETAIN_COMMITS, tableMeta.getHudi().getKeepCommitVersion());
+ // 时间线归档最小保留个数，要比上一个参数大
+ configuration.setInteger(FlinkOptions.ARCHIVE_MIN_COMMITS, tableMeta.getHudi().getKeepCommitVersion() + 50);
+ // 时间线归档最大保留个数，要比上一个参数大
+ configuration.setInteger(FlinkOptions.ARCHIVE_MAX_COMMITS, tableMeta.getHudi().getKeepCommitVersion() + 100);
+ // log文件和data文件保留版本数
+ configuration.setString(FlinkOptions.CLEAN_POLICY, HoodieCleaningPolicy.KEEP_LATEST_FILE_VERSIONS.name());
+ configuration.setInteger(FlinkOptions.CLEAN_RETAIN_FILE_VERSIONS, tableMeta.getHudi().getKeepFileVersion());
+
+ // 关闭一个内置的 http 服务
+ // configuration.setBoolean(HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_ENABLE.key(), false);
+
+ return configuration;
+ }
+
+ public static void sinkToHoodieByTable(GlobalConfiguration globalConfiguration, FlinkJob flinkJob, TableMeta tableMeta, StreamExecutionEnvironment environment, DataStream inputDataStream) {
+ Schema schema = avroSchemaWithExtraFields(tableMeta);
+ DataStream dataStream = inputDataStream
+ .filter(new OperationTypeFilter(globalConfiguration, flinkJob, tableMeta))
+ .name("Count operation type")
+ .map(new Record2RowDataFunction(globalConfiguration, flinkJob, tableMeta), TypeInformation.of(new TypeHint>() {
+ }))
+ .name("Covert Row ( " + tableMeta.getSchema() + "-" + tableMeta.getTable() + " )")
+ .flatMap((list, collector) -> list.forEach(collector::collect), TypeInformation.of(RowData.class))
+ .filter(Objects::nonNull)
+ .name("Filter not null");
+
+ RowType rowType = (RowType) AvroSchemaConverter.convertToDataType(schema).getLogicalType();
+ StreamTableEnvironment tableEnvironment = StreamTableEnvironment.create(
+ environment,
+ EnvironmentSettings.newInstance()
+ .inStreamingMode()
+ .useBlinkPlanner()
+ .build()
+ );
+ Configuration configuration = tableEnvironment.getConfig().getConfiguration();
+ int parallelism = configuration.getInteger("parallelism", 1);
+ configuration = getSyncFlinkConfiguration(globalConfiguration, configuration, flinkJob, tableMeta, schema, parallelism);
+
+ DataStream hoodieRecordDataStream = Pipelines.bootstrap(configuration, rowType, parallelism, dataStream);
+ DataStream