lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(all): 迁移common、sync、executor项目

Browse Source
This commit is contained in:
兰缘小妖
2024-02-29 15:32:14 +08:00
parent 0683068a02
commit 5a2e9fdfb8
73 changed files with 10204 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
service-common/pom.xml Normal file
View File

@@ -0,0 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>hudi-service</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<artifactId>service-common</artifactId>
<dependencies>
<!-- Common 包不要引入第三方依赖，避免冲突，一些简单的工具类自己手动实现，复杂或不必要 common 的流程不要放在 common 包里实现 -->
<!-- hutool 系列是一个无三方依赖的工具包，建议使用，但同样也是能不用就不用，保持纯净 -->
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
</dependency>
<!-- 用于提供有限的SQL构造，避免引入复杂的ORM框架 -->
<dependency>
<groupId>io.github.dragons96</groupId>
<artifactId>sql-builder</artifactId>
<version>0.0.5.3</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>

211
service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java Normal file
View File

@@ -0,0 +1,211 @@
package com.lanyuanxiaoyao.service.common;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import java.time.format.DateTimeFormatter;
import java.util.function.BiFunction;
/**
* 常量
*
* @author ZhangJiacheng
* @version 0.0.1
* @date 2021-12-03
*/
public interface Constants {
// String DATABASE_NAME = "hudi_collect_build";
// String DATABASE_NAME = "hudi_collect_build_2";
String DATABASE_NAME = "hudi_collect_build_b12";
String API_HEADER_NAME = "Api-Version";
String API_VERSION = "1";
/**
* 联合主键
*/
String UNION_KEY_NAME = "_key";
/**
* 源端最后操作时间
*/
String LATEST_OPERATION_TIMESTAMP_KEY_NAME = "latest_op_ts";
/**
* 记录下游入库时间
*/
String UPDATE_TIMESTAMP_KEY_NAME = "update_ts";
/**
* Hudi 删除标记字段
*/
String HUDI_DELETE_KEY_NAME = "_hoodie_is_deleted";
String PULSAR_SUBSCRIPTION_NAME_PREFIX = "Hudi_Sync_Pulsar_Reader";
String VERSION_UPDATE_KEY = "versionUpdate";
String VERSION_KEY = "version";
String DELETE = "D";
String INSERT = "I";
String UPDATE = "U";
String DDL = "ddl";
String UNKNOWN = "unknown";
String CITY_ID = "CITY_ID";
String INCLUDE = "INCLUDE";
String EXCLUDE = "EXCLUDE";
String JOB_ID = "job-id";
String SERVICE_MODE = "service-mode";
String FLINK_JOB = "flink-job";
String TABLE_META = "table-meta";
String TABLE_META_LIST = "table-meta-list";
String MESSAGE_ID = "message-id";
String INSTANTS = "instants";
String BETA = "beta";
String CLUSTER = "cluster";
String COW = "COPY_ON_WRITE";
String MOR = "MERGE_ON_READ";
String FLINK_JOB_OPTION = "-" + FLINK_JOB;
String TABLE_META_OPTION = "-" + TABLE_META;
String TABLE_META_LIST_OPTION = "-" + TABLE_META_LIST;
String INSTANTS_OPTION = "-" + INSTANTS;
String BETA_OPTION = "-" + BETA;
String CLUSTER_OPTION = "-" + CLUSTER;
String SPRING_SECURITY_AUTHORITY = "Anonymous";
String SPRING_SECURITY_USERNAME = "AxhEbscwsJDbYMH2";
String SPRING_SECURITY_PASSWORD = "{noop}cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4";
String SPRING_SECURITY_PASSWORD_PLAIN = "cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4";
String VICTORIA_USERNAME = "EsCFVuNkiDWv7PKmcF";
String VICTORIA_PASSWORD = "Abf%x9ocS^iKr3tgrd";
String SCHEMA_NAME = "schema";
String TABLE_NAME = "table";
String DATA_TIME = "data-time";
String DATA_PARENT_PATH = "data-parent-path";
String METRICS_PREFIX = "metrics_hudi";
String METRICS_YARN_PREFIX = METRICS_PREFIX + "_yarn";
String METRICS_YARN_JOB = METRICS_YARN_PREFIX + "_job";
String METRICS_YARN_TABLE = METRICS_YARN_PREFIX + "_table";
String METRICS_SYNC_PREFIX = METRICS_PREFIX + "_sync";
String METRICS_SYNC_SOURCE_LATENCY = METRICS_SYNC_PREFIX + "_source_latency";
String METRICS_SYNC_LATENCY = METRICS_SYNC_PREFIX + "_latency";
String METRICS_SYNC_FLINK_JOB_ID = METRICS_SYNC_PREFIX + "_flink_job_id";
String METRICS_SYNC_SOURCE_MESSAGE_RECEIVE = METRICS_SYNC_PREFIX + "_source_message_receive";
String METRICS_SYNC_SOURCE_MESSAGE_SIZE_RECEIVE_BYTES = METRICS_SYNC_PREFIX + "_source_message_receive_bytes";
String METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE = METRICS_SYNC_PREFIX + "_source_operation_type_receive";
String METRICS_SYNC_SOURCE_CHANGE_FILTER = METRICS_SYNC_PREFIX + "_source_change_filter";
String METRICS_SYNC_SOURCE_CHANGE_PARTITION = METRICS_SYNC_PREFIX + "_source_change_partition";
String METRICS_SYNC_SOURCE_BACK_LOGS = METRICS_SYNC_PREFIX + "_source_back_logs";
String METRICS_LABEL_FLINK_JOB_ID = "flink_job_id";
String METRICS_LABEL_FLINK_JOB_NAME = "flink_job_name";
String METRICS_LABEL_FLINK_NATIVE_JOB_ID = "flink_native_job_id";
String METRICS_LABEL_FLINK_NATIVE_TASK_NAME = "flink_native_task_name";
String METRICS_LABEL_FLINK_PARALLEL_ID = "flink_parallel_id";
String METRICS_LABEL_RUN_TYPE = "run_type";
String METRICS_LABEL_EXECUTOR_VERSION = "executor_version";
String METRICS_LABEL_CLUSTER = "cluster";
String METRICS_RUN_TYPE_SYNC = "sync";
String METRICS_RUN_TYPE_COMPACTION = "compaction";
String METRICS_LABEL_SCHEMA = "schema";
String METRICS_LABEL_TABLE = "table";
String METRICS_LABEL_STATUS = "status";
String METRICS_LABEL_TOPIC = "topic";
String METRICS_LABEL_BATCH_ID = "batch_id";
String METRICS_LABEL_ALIAS = "alias";
String METRICS_LABEL_APPLICATION_ID = "application_id";
String METRICS_STATUS_RUNNING = "running";
String METRICS_STATUS_STOPPED = "stopped";
String METRICS_LABEL_TYPE = "type";
String LOKI_PUSH_URL = "loki_push_url";
DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
String OPERATION_DONE = "操作完成";
String OPERATION_CANCEL = "操作取消";
String FETCHING_DATA = "Fetching Data";
String COMPACTION_STATUS_SCHEDULE = "SCHEDULE";
String COMPACTION_STATUS_START = "START";
String COMPACTION_STATUS_FINISH = "FINISH";
String COMPACTION_STATUS_FAILURE = "FAILURE";
long SECOND = 1000;
long HALF_MINUTE = 30 * SECOND;
long MINUTE = 60 * SECOND;
long HALF_HOUR = 30 * MINUTE;
long HOUR = 60 * MINUTE;
long KB = 1024;
long MB = 1024 * KB;
long GB = 1024 * MB;
long TB = 1024 * GB;
String TAG_SPLIT = ";";
String TAG_OPERATOR = "=";
String EVENT = "event";
String FROM_COMMAND_UTIL = "command util";
String FROM_COMPACTOR = "compactor";
int COMMAND_RENDER_WIDTH = 500;
String LOG_FLINK_JOB_ID_LABEL = "LOG_FLINK_JOB_ID_LABEL";
String LOG_FLINK_JOB_ID = "flink_job_id";
String LOG_ALIAS_LABEL = "LOG_ALIAS_LABEL";
String LOG_ALIAS = "alias";
String LOG_JOB_ID_LABEL = "LOG_JOB_ID_LABEL";
String LOG_JOB_ID = "job_id";
String LOG_POINT_PREFIX = "LOP-";
String LOG_POINT_MESSAGE_ID_EMPTY = LOG_POINT_PREFIX + "000001";
String LOG_POINT_CHECKPOINT_INITIAL = LOG_POINT_PREFIX + "000002";
String LOG_POINT_CHECKPOINT_INITIAL_MESSAGE_ID = LOG_POINT_PREFIX + "000003";
String LOG_POINT_PULSAR_SOURCE_BOOTSTRAP_MESSAGE_ID = LOG_POINT_PREFIX + "000004";
String LOG_POINT_PULSAR_SOURCE_GET_MESSAGE_ID_ERROR = LOG_POINT_PREFIX + "000005";
String LOG_POINT_FIELD_TYPE_NOT_FOUND = LOG_POINT_PREFIX + "000006";
String TAGS_NO_COMPACT = "NO_COMPACT";
String TAGS_PULSAR_BACKUP = "PULSAR_BACKUP";
String TAGS_NO_PRE_COMBINE = "NO_PRE_COMBINE";
String TAGS_PRE_COMBINE = "PRE_COMBINE";
String TAGS_NO_IGNORE_FAILED = "NO_IGNORE_FAILED";
String TAGS_DISABLE_CHAINING = "DISABLE_CHAINING";
String TAGS_TRACE_LATEST_OP_TS = "TRACE_LATEST_OP_TS";
String TAGS_SOURCE_READER = "SOURCE_READER";
String TAGS_USE_TEST_JAR = "USE_TEST_JAR";
String TAGS_ODS = "ODS";
String TAGS_ODS_FOCUS = "ODS_FOCUS";
String COMPACTION_QUEUE_PRE = "compaction-queue-pre";
String COMPACTION_QUEUE_B1 = "compaction-queue-b1";
String COMPACTION_QUEUE_B5 = "compaction-queue-b5";
String COMPACTION_QUEUE_A4 = "compaction-queue-a4";
String COMPACTION_QUEUE_B12 = "compaction-queue-b12";
String CLUSTER_B1 = "b1";
String CLUSTER_B5 = "b5";
String CLUSTER_A4 = "a4";
String CLUSTER_B12 = "b12";
String SCHEDULE_JOB_FAIL_COUNT = "SCHEDULE_JOB_FAIL_COUNT";
String SCHEDULE_RECOMMEND = "schedule_recommend";
String SCHEDULE_FORCE = "schedule_force";
BiFunction<TableMeta, String, String> FIELD_COVERT = (tableMeta, field) -> {
if (TableMeta.SourceType.TELEPG.equals(tableMeta.getSourceType())) {
return field.toLowerCase();
} else {
return field;
}
};
}

1713
service-common/src/main/java/com/lanyuanxiaoyao/service/common/SQLConstants.java Normal file
View File

File diff suppressed because it is too large Load Diff

123
service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/FlinkJob.java Normal file
View File

@@ -0,0 +1,123 @@
package com.lanyuanxiaoyao.service.common.entity;
import java.io.Serializable;
/**
* Flink Job
*
* @author ZhangJiacheng
* @version 0.0.1
* @date 2021-12-08
*/
public class FlinkJob implements Serializable {
private Long id;
private String name;
private RunMode runMode;
private TableMeta.YarnMeta oneInOneSyncYarn;
public FlinkJob() {
}
public FlinkJob(Builder builder) {
this.id = builder.id;
this.name = builder.name;
this.runMode = builder.runMode;
this.oneInOneSyncYarn = builder.oneInOneSyncYarn;
}
public static Builder builder() {
return new Builder();
}
public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public RunMode getRunMode() {
return runMode;
}
public void setRunMode(RunMode runMode) {
this.runMode = runMode;
}
public TableMeta.YarnMeta getOneInOneSyncYarn() {
return oneInOneSyncYarn;
}
public void setOneInOneSyncYarn(TableMeta.YarnMeta oneInOneSyncYarn) {
this.oneInOneSyncYarn = oneInOneSyncYarn;
}
@Override
public String toString() {
return "FlinkJob{" +
"id=" + id +
", name='" + name + '\'' +
", runMode=" + runMode +
", oneInOneSyncYarn=" + oneInOneSyncYarn +
'}';
}
public enum RunMode {
/**
* 所有表放在同一个任务中
*/
ALL_IN_ONE,
/**
* 每个表放在单独的任务中
*/
ONE_IN_ONE,
/**
* 针对 ACCT 小表，将同一个表放在同一个子任务中
*/
ALL_IN_ONE_BY_TABLE,
ALL_IN_ONE_BY_SCHEMA,
}
public static final class Builder {
private Long id;
private String name;
private RunMode runMode;
private TableMeta.YarnMeta oneInOneSyncYarn;
private Builder() {}
public Builder id(Long id) {
this.id = id;
return this;
}
public Builder name(String name) {
this.name = name;
return this;
}
public Builder runMode(RunMode runMode) {
this.runMode = runMode;
return this;
}
public Builder oneInOneSyncYarn(TableMeta.YarnMeta oneInOneSyncYarn) {
this.oneInOneSyncYarn = oneInOneSyncYarn;
return this;
}
public FlinkJob build() {
return new FlinkJob(this);
}
}
}

197
service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/Record.java Normal file
View File

@@ -0,0 +1,197 @@
package com.lanyuanxiaoyao.service.common.entity;
import java.io.Serializable;
import java.util.Map;
/**
* 从 Pulsar 读取的消息封装
*
* @author lanyuanxiaoyao
* @version 0.0.1
* @date 2021-11-25
*/
public class Record implements Serializable {
private Source source;
private Statement statement;
public Record() {
}
public Source getSource() {
return source;
}
public void setSource(Source source) {
this.source = source;
}
public Statement getStatement() {
return statement;
}
public void setStatement(Statement statement) {
this.statement = statement;
}
@Override
public String toString() {
return "Record{" +
"source=" + source +
", statement=" + statement +
'}';
}
public static class Source implements Serializable {
private String sourceId;
private String sourceType;
private String sourcePos;
private String currentTs;
public Source() {
}
public String getSourceId() {
return sourceId;
}
public void setSourceId(String sourceId) {
this.sourceId = sourceId;
}
public String getSourceType() {
return sourceType;
}
public void setSourceType(String sourceType) {
this.sourceType = sourceType;
}
public String getSourcePos() {
return sourcePos;
}
public void setSourcePos(String sourcePos) {
this.sourcePos = sourcePos;
}
public String getCurrentTs() {
return currentTs;
}
public void setCurrentTs(String currentTs) {
this.currentTs = currentTs;
}
@Override
public String toString() {
return "Source{" +
"sourceId='" + sourceId + '\'' +
", sourceType='" + sourceType + '\'' +
", sourcePos='" + sourcePos + '\'' +
", currentTs='" + currentTs + '\'' +
'}';
}
}
public static class Statement implements Serializable {
private String schema;
private String table;
private String opStatement;
private String opType;
private String op;
private String opTs;
private String version;
private Map<String, Object> before;
private Map<String, Object> after;
public Statement() {
}
public String getSchema() {
return schema;
}
public void setSchema(String schema) {
this.schema = schema;
}
public String getTable() {
return table;
}
public void setTable(String table) {
this.table = table;
}
public String getOpStatement() {
return opStatement;
}
public void setOpStatement(String opStatement) {
this.opStatement = opStatement;
}
public String getOpType() {
return opType;
}
public void setOpType(String opType) {
this.opType = opType;
}
public String getOp() {
return op;
}
public void setOp(String op) {
this.op = op;
}
public String getOpTs() {
return opTs;
}
public void setOpTs(String opTs) {
this.opTs = opTs;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public Map<String, Object> getBefore() {
return before;
}
public void setBefore(Map<String, Object> before) {
this.before = before;
}
public Map<String, Object> getAfter() {
return after;
}
public void setAfter(Map<String, Object> after) {
this.after = after;
}
@Override
public String toString() {
return "Statement{" +
"schema='" + schema + '\'' +
", table='" + table + '\'' +
", opStatement='" + opStatement + '\'' +
", opType='" + opType + '\'' +
", op='" + op + '\'' +
", opTs='" + opTs + '\'' +
", version='" + version + '\'' +
", before=" + before +
", after=" + after +
'}';
}
}
}

121
service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/RunMeta.java Normal file
View File

@@ -0,0 +1,121 @@
package com.lanyuanxiaoyao.service.common.entity;
import com.lanyuanxiaoyao.service.common.utils.NameHelper;
import java.io.Serializable;
/**
* 运行时参数
*
* @author ZhangJiacheng
* @date 2023-05-11
*/
@SuppressWarnings("FieldMayBeFinal")
public class RunMeta implements Serializable {
private String cluster;
private Long flinkJobId;
private String alias;
private String flinkJobName;
private String host;
private String applicationId;
private String containerId;
private String containerPath;
private String runType;
private String executorVersion;
private String jvmPid;
private String applicationProxy;
private String subscriptionName;
public RunMeta() {
this.flinkJobName = System.getenv("flink_job_name");
this.host = System.getenv("NM_HOST");
this.applicationId = System.getenv("_APP_ID");
this.containerId = System.getenv("CONTAINER_ID");
this.containerPath = System.getenv("PWD");
this.runType = System.getenv("run_type");
this.executorVersion = System.getenv("executor_version");
this.jvmPid = System.getenv("JVM_PID");
this.applicationProxy = System.getenv("APPLICATION_WEB_PROXY_BASE");
}
public RunMeta(String cluster, Long flinkJobId) {
this();
this.cluster = cluster;
this.flinkJobId = flinkJobId;
}
public RunMeta(String cluster, Long flinkJobId, String alias) {
this(cluster, flinkJobId);
this.alias = alias;
this.subscriptionName = NameHelper.pulsarSubscriptionName(flinkJobId, alias);
}
public String getCluster() {
return cluster;
}
public Long getFlinkJobId() {
return flinkJobId;
}
public String getAlias() {
return alias;
}
public String getFlinkJobName() {
return flinkJobName;
}
public String getHost() {
return host;
}
public String getApplicationId() {
return applicationId;
}
public String getContainerId() {
return containerId;
}
public String getContainerPath() {
return containerPath;
}
public String getRunType() {
return runType;
}
public String getExecutorVersion() {
return executorVersion;
}
public String getJvmPid() {
return jvmPid;
}
public String getApplicationProxy() {
return applicationProxy;
}
public String getSubscriptionName() {
return subscriptionName;
}
@Override
public String toString() {
return "RunMeta{" +
"cluster='" + cluster + '\'' +
", flinkJobId='" + flinkJobId + '\'' +
", flinkJobName='" + flinkJobName + '\'' +
", host='" + host + '\'' +
", applicationId='" + applicationId + '\'' +
", containerId='" + containerId + '\'' +
", containerPath='" + containerPath + '\'' +
", runType='" + runType + '\'' +
", executorVersion='" + executorVersion + '\'' +
", jvmPid='" + jvmPid + '\'' +
", applicationProxy='" + applicationProxy + '\'' +
", subscriptionName='" + subscriptionName + '\'' +
'}';
}
}

259
service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/SyncState.java Normal file
View File

@@ -0,0 +1,259 @@
package com.lanyuanxiaoyao.service.common.entity;
import java.io.Serializable;
/**
* 同步压缩状态表类
*
* @author ZhangJiacheng
* @date 2023-04-24
*/
public class SyncState implements Serializable {
private Long flinkJobId;
private String alias;
private String messageId;
private Long sourceStartTime;
private Long sourceCheckpointTime;
private Long sourcePublishTime;
private Long sourceOperationTime;
private Long compactionStartTime;
private Long compactionFinishTime;
private String compactionApplicationId;
private String compactionStatus;
private Long compactionStatusTime;
private Long compactionLatestOperationTime;
public SyncState() {
}
public SyncState(Builder builder) {
this.flinkJobId = builder.flinkJobId;
this.alias = builder.alias;
this.messageId = builder.messageId;
this.sourceStartTime = builder.sourceStartTime;
this.sourceCheckpointTime = builder.sourceCheckpointTime;
this.sourcePublishTime = builder.sourcePublishTime;
this.sourceOperationTime = builder.sourceOperationTime;
this.compactionStartTime = builder.compactionStartTime;
this.compactionFinishTime = builder.compactionFinishTime;
this.compactionApplicationId = builder.compactionApplicationId;
this.compactionStatus = builder.compactionStatus;
this.compactionStatusTime = builder.compactionStatusTime;
this.compactionLatestOperationTime = builder.compactionLatestOperationTime;
}
public static Builder builder() {
return new Builder();
}
public Long getFlinkJobId() {
return flinkJobId;
}
public void setFlinkJobId(Long flinkJobId) {
this.flinkJobId = flinkJobId;
}
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
public String getMessageId() {
return messageId;
}
public void setMessageId(String messageId) {
this.messageId = messageId;
}
public Long getSourceStartTime() {
return sourceStartTime;
}
public void setSourceStartTime(Long sourceStartTime) {
this.sourceStartTime = sourceStartTime;
}
public Long getSourceCheckpointTime() {
return sourceCheckpointTime;
}
public void setSourceCheckpointTime(Long sourceCheckpointTime) {
this.sourceCheckpointTime = sourceCheckpointTime;
}
public Long getSourcePublishTime() {
return sourcePublishTime;
}
public void setSourcePublishTime(Long sourcePublishTime) {
this.sourcePublishTime = sourcePublishTime;
}
public Long getSourceOperationTime() {
return sourceOperationTime;
}
public void setSourceOperationTime(Long sourceOperationTime) {
this.sourceOperationTime = sourceOperationTime;
}
public Long getCompactionStartTime() {
return compactionStartTime;
}
public void setCompactionStartTime(Long compactionStartTime) {
this.compactionStartTime = compactionStartTime;
}
public Long getCompactionFinishTime() {
return compactionFinishTime;
}
public void setCompactionFinishTime(Long compactionFinishTime) {
this.compactionFinishTime = compactionFinishTime;
}
public String getCompactionApplicationId() {
return compactionApplicationId;
}
public void setCompactionApplicationId(String compactionApplicationId) {
this.compactionApplicationId = compactionApplicationId;
}
public String getCompactionStatus() {
return compactionStatus;
}
public void setCompactionStatus(String compactionStatus) {
this.compactionStatus = compactionStatus;
}
public Long getCompactionStatusTime() {
return compactionStatusTime;
}
public void setCompactionStatusTime(Long compactionStatusTime) {
this.compactionStatusTime = compactionStatusTime;
}
public Long getCompactionLatestOperationTime() {
return compactionLatestOperationTime;
}
public void setCompactionLatestOperationTime(Long compactionLatestOperationTime) {
this.compactionLatestOperationTime = compactionLatestOperationTime;
}
@Override
public String toString() {
return "SyncState{" +
"flinkJobId=" + flinkJobId +
", alias='" + alias + '\'' +
", messageId='" + messageId + '\'' +
", sourceStartTime=" + sourceStartTime +
", sourceCheckpointTime=" + sourceCheckpointTime +
", sourcePublishTime=" + sourcePublishTime +
", sourceOperationTime=" + sourceOperationTime +
", compactionStartTime=" + compactionStartTime +
", compactionFinishTime=" + compactionFinishTime +
", compactionApplicationId='" + compactionApplicationId + '\'' +
", compactionStatus='" + compactionStatus + '\'' +
", compactionStatusTime=" + compactionStatusTime +
", compactionLatestOperationTime=" + compactionLatestOperationTime +
'}';
}
public static final class Builder {
private Long flinkJobId;
private String alias;
private String messageId;
private Long sourceStartTime;
private Long sourceCheckpointTime;
private Long sourcePublishTime;
private Long sourceOperationTime;
private Long compactionStartTime;
private Long compactionFinishTime;
private String compactionApplicationId;
private String compactionStatus;
private Long compactionStatusTime;
private Long compactionLatestOperationTime;
private Builder() {
}
public Builder flinkJobId(Long flinkJobId) {
this.flinkJobId = flinkJobId;
return this;
}
public Builder alias(String alias) {
this.alias = alias;
return this;
}
public Builder messageId(String messageId) {
this.messageId = messageId;
return this;
}
public Builder sourceStartTime(Long sourceStartTime) {
this.sourceStartTime = sourceStartTime;
return this;
}
public Builder sourceCheckpointTime(Long sourceCheckpointTime) {
this.sourceCheckpointTime = sourceCheckpointTime;
return this;
}
public Builder sourcePublishTime(Long sourcePublishTime) {
this.sourcePublishTime = sourcePublishTime;
return this;
}
public Builder sourceOperationTime(Long sourceOperationTime) {
this.sourceOperationTime = sourceOperationTime;
return this;
}
public Builder compactionStartTime(Long compactionStartTime) {
this.compactionStartTime = compactionStartTime;
return this;
}
public Builder compactionFinishTime(Long compactionFinishTime) {
this.compactionFinishTime = compactionFinishTime;
return this;
}
public Builder compactionApplicationId(String compactionApplicationId) {
this.compactionApplicationId = compactionApplicationId;
return this;
}
public Builder compactionStatus(String compactionStatus) {
this.compactionStatus = compactionStatus;
return this;
}
public Builder compactionStatusTime(Long compactionStatusTime) {
this.compactionStatusTime = compactionStatusTime;
return this;
}
public Builder compactionLatestOperationTime(Long compactionLatestOperationTime) {
this.compactionLatestOperationTime = compactionLatestOperationTime;
return this;
}
public SyncState build() {
return new SyncState(this);
}
}
}

2390
service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/TableMeta.java Normal file
View File

File diff suppressed because it is too large Load Diff

144
service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleJob.java Normal file
View File

@@ -0,0 +1,144 @@
package com.lanyuanxiaoyao.service.common.entity.compaction;
/**
* 压缩调度任务
*
* @author ZhangJiacheng
* @date 2022-09-26
*/
public class ScheduleJob {
private String id;
private Long flinkJobId;
private String alias;
private String batch;
private String status;
private String comment;
public ScheduleJob() {
}
public ScheduleJob(String id, Long flinkJobId, String alias, String batch, String status, String comment) {
this.id = id;
this.flinkJobId = flinkJobId;
this.alias = alias;
this.batch = batch;
this.status = status;
this.comment = comment;
}
public ScheduleJob(Builder builder) {
this.id = builder.id;
this.flinkJobId = builder.flinkJobId;
this.alias = builder.alias;
this.batch = builder.batch;
this.status = builder.status;
this.comment = builder.comment;
}
public static Builder builder() {
return new Builder();
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public Long getFlinkJobId() {
return flinkJobId;
}
public void setFlinkJobId(Long flinkJobId) {
this.flinkJobId = flinkJobId;
}
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
public String getBatch() {
return batch;
}
public void setBatch(String batch) {
this.batch = batch;
}
public String getStatus() {
return status;
}
public void setStatus(String status) {
this.status = status;
}
public String getComment() {
return comment;
}
public void setComment(String comment) {
this.comment = comment;
}
@Override
public String toString() {
return "ScheduleJob{" +
"id=" + id +
", flinkJobId=" + flinkJobId +
", alias='" + alias + '\'' +
'}';
}
public static final class Builder {
private String id;
private Long flinkJobId;
private String alias;
private String batch;
private String status;
private String comment;
private Builder() {
}
public Builder id(String id) {
this.id = id;
return this;
}
public Builder flinkJobId(Long flinkJobId) {
this.flinkJobId = flinkJobId;
return this;
}
public Builder alias(String alias) {
this.alias = alias;
return this;
}
public Builder batch(String batch) {
this.batch = batch;
return this;
}
public Builder status(String status) {
this.status = status;
return this;
}
public Builder comment(String comment) {
this.comment = comment;
return this;
}
public ScheduleJob build() {
return new ScheduleJob(this);
}
}
}

15
service-common/src/main/java/com/lanyuanxiaoyao/service/common/entity/compaction/ScheduleQueue.java Normal file
View File

@@ -0,0 +1,15 @@
package com.lanyuanxiaoyao.service.common.entity.compaction;
import java.util.Deque;
/**
* 队列
*
* @author ZhangJiacheng
* @date 2022-09-26
*/
public interface ScheduleQueue extends Deque<ScheduleJob> {
Iterable<ScheduleJob> poll(int limit);
Iterable<ScheduleJob> pollWithoutSame(int limit);
}

11
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/CheckpointRootPathNotFoundException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* @author ZhangJiacheng
* @date 2022-05-23
*/
public class CheckpointRootPathNotFoundException extends RuntimeException{
public CheckpointRootPathNotFoundException() {
super("Checkpoint root path not found");
}
}

29
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ConfigException.java Normal file
View File

@@ -0,0 +1,29 @@
package com.lanyuanxiaoyao.service.common.exception;
import java.util.function.Supplier;
/**
* 配置异常
*
* @author ZhangJiacheng
* @date 2022-05-16
*/
public class ConfigException extends Exception {
public ConfigException(String message) {
super(message);
}
public static void check(String message, Supplier<Boolean> checkFunction) throws ConfigException {
if (checkFunction.get()) {
throw new ConfigException(message);
}
}
public static void checkQuiet(String message, Supplier<Boolean> checkFunction) {
try {
check(message, checkFunction);
} catch (ConfigException e) {
throw new RuntimeException(e);
}
}
}

15
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/FlinkJobNotFoundException.java Normal file
View File

@@ -0,0 +1,15 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* @author ZhangJiacheng
* @date 2022-05-23
*/
public class FlinkJobNotFoundException extends RuntimeException {
public FlinkJobNotFoundException() {
super("Flink job not found");
}
public FlinkJobNotFoundException(Long flinkJobId) {
super("Flink job " + flinkJobId + " not found");
}
}

14
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MessageIdEmptyException.java Normal file
View File

@@ -0,0 +1,14 @@
package com.lanyuanxiaoyao.service.common.exception;
import com.lanyuanxiaoyao.service.common.Constants;
/**
* @author ZhangJiacheng
* @date 2022-05-23
*/
public class MessageIdEmptyException extends RuntimeException {
public MessageIdEmptyException() {
super(Constants.LOG_POINT_MESSAGE_ID_EMPTY + " Message id is empty");
}
}

13
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/MissingArgumentException.java Normal file
View File

@@ -0,0 +1,13 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* 缺参数异常
*
* @author ZhangJiacheng
* @date 2022-06-20
*/
public class MissingArgumentException extends Exception {
public MissingArgumentException(String argumentName) {
super("Argument: '" + argumentName + "' is not found");
}
}

11
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PartitionPathNotFoundException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* @author ZhangJiacheng
* @date 2022-05-23
*/
public class PartitionPathNotFoundException extends RuntimeException{
public PartitionPathNotFoundException() {
super("Partition path not found");
}
}

11
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/PulsarInfoNotFoundException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* @author ZhangJiacheng
* @date 2023-04-28
*/
public class PulsarInfoNotFoundException extends RuntimeException {
public PulsarInfoNotFoundException(String message) {
super(message);
}
}

11
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/SyncStateNotFoundException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* @author ZhangJiacheng
* @date 2022-05-23
*/
public class SyncStateNotFoundException extends RuntimeException{
public SyncStateNotFoundException() {
super("Sync state not found");
}
}

11
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/TableMetaNotFoundException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* @author ZhangJiacheng
* @date 2022-05-23
*/
public class TableMetaNotFoundException extends RuntimeException{
public TableMetaNotFoundException() {
super("Table meta not found");
}
}

11
service-common/src/main/java/com/lanyuanxiaoyao/service/common/exception/ZookeeperUrlNotFoundException.java Normal file
View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.common.exception;
/**
* @author ZhangJiacheng
* @date 2022-05-23
*/
public class ZookeeperUrlNotFoundException extends RuntimeException{
public ZookeeperUrlNotFoundException() {
super("Zookeeper url not found");
}
}

86
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/FlinkJobHelper.java Normal file
View File

@@ -0,0 +1,86 @@
package com.lanyuanxiaoyao.service.common.utils;
import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
/**
* Flink Job 工具类
*
* @author ZhangJiacheng
* @version 0.0.1
* @date 2021-12-08
*/
public class FlinkJobHelper {
public static String allFlinkJobSql(String database) {
// language=MySQL
return "select tafjc.id,\n" +
" tafjc.name,\n" +
" tafjc.run_mode,\n" +
" tayjc.job_manager_memory,\n" +
" tayjc.task_manager_memory\n" +
"from `" + database + "`.tb_app_flink_job_config tafjc\n" +
" left join\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc on tafjc.one_in_one_yarn_job_id = tayjc.id and tayjc.status = 'y'\n" +
"where tafjc.status = 'y'";
}
public static String flinkJobSql(String database) {
// language=MySQL
return "select tafjc.id,\n" +
" tafjc.name,\n" +
" tafjc.run_mode,\n" +
" tayjc.job_manager_memory,\n" +
" tayjc.task_manager_memory\n" +
"from `" + database + "`.tb_app_flink_job_config tafjc\n" +
" left join\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc on tafjc.one_in_one_yarn_job_id = tayjc.id and tayjc.status = 'y'\n" +
"where tafjc.id = ?\n" +
" and tafjc.status = 'y'";
}
public static List<FlinkJob> from(ResultSet rs) throws SQLException {
List<FlinkJob> results = new ArrayList<>();
while (rs.next()) {
String runModeText = rs.getString(3);
FlinkJob.RunMode mode;
try {
mode = FlinkJob.RunMode.valueOf(runModeText);
} catch (IllegalArgumentException e) {
mode = FlinkJob.RunMode.ALL_IN_ONE;
}
TableMeta.YarnMeta yarnMeta = TableMeta.YarnMeta.builder()
.jobManagerMemory(rs.getInt(4))
.taskManagerMemory(rs.getInt(5))
.build();
results.add(
FlinkJob.builder()
.id(rs.getLong(1))
.name(rs.getString(2))
.runMode(mode)
.oneInOneSyncYarn(yarnMeta)
.build()
);
}
return results;
}
public static Optional<FlinkJob> fromOne(ResultSet rs) throws SQLException {
List<FlinkJob> results = from(rs);
if (results.size() < 1) {
return Optional.empty();
} else if (results.size() > 1) {
throw new SQLException("Found more than 1 records");
} else {
return Optional.of(results.get(0));
}
}
public static boolean isOneInOneMode(FlinkJob job) {
return FlinkJob.RunMode.ONE_IN_ONE.equals(job.getRunMode());
}
}

117
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/LogHelper.java Normal file
View File

@@ -0,0 +1,117 @@
package com.lanyuanxiaoyao.service.common.utils;
import cn.hutool.core.collection.ListUtil;
import cn.hutool.core.map.MapBuilder;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.common.Constants;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.MDC;
/**
* 日志相关
*
* @author ZhangJiacheng
* @date 2023-12-25
*/
public class LogHelper {
private static String generateLog(LogPoint point, String template, Object[] args) {
if (ObjectUtil.isEmpty(template)) {
return point.toString();
}
if (ObjectUtil.isEmpty(args)) {
return point + " " + template;
}
Object[] items = new Object[args.length + 1];
items[0] = point;
System.arraycopy(args, 0, items, 1, args.length);
return StrUtil.format("{} " + template, items);
}
public static void info(Logger logger, LogPoint point) {
logger.info(generateLog(point, null, null));
}
public static void info(Logger logger, LogPoint point, String template) {
logger.info(generateLog(point, template, null));
}
public static void info(Logger logger, LogPoint point, String template, Object... args) {
logger.info(generateLog(point, template, args));
}
public static void debug(Logger logger, LogPoint point, String template, Object... args) {
logger.debug(generateLog(point, template, args));
}
public static void warn(Logger logger, LogPoint point, String template, Object... args) {
logger.warn(generateLog(point, template, args));
}
public static void error(Logger logger, LogPoint point, String template, Object... args) {
logger.error(generateLog(point, template, args));
}
public static void setMdc(Map<String, String> mdcList) {
mdcList.forEach(MDC::put);
}
public static void setMdc(String... items) {
if (items.length % 2 != 0) {
throw new IllegalArgumentException("Items must key-value");
}
MapBuilder<String, String> builder = MapUtil.builder();
for (int i = 0, j = 1; j < items.length; i++, j++) {
builder.put(items[i], items[j]);
}
setMdc(builder.build());
}
public static void removeMdc(List<String> mdcList) {
mdcList.forEach(MDC::remove);
}
public static void removeMdc(String... names) {
removeMdc(ListUtil.of(names));
}
public static void setMdcFlinkJobAndAlias(Long flinkJobId, String alias) {
setMdc(MapUtil.<String, String>builder()
.put(Constants.LOG_FLINK_JOB_ID_LABEL, flinkJobId.toString())
.put(Constants.LOG_ALIAS_LABEL, alias)
.build());
}
public static void removeMdcFlinkJobAndAlias(Long flinkJobId, String alias) {
removeMdc(ListUtil.of(Constants.LOG_FLINK_JOB_ID_LABEL, Constants.LOG_ALIAS_LABEL));
}
public enum LogPoint {
PULSAR_SOURCE_CHECKPOINT_INITIAL(100),
PULSAR_SOURCE_CHECKPOINT_INITIAL_MESSAGE_ID(101),
MESSAGE_ID_EMPTY(1),
CHECKPOINT_INITIAL(2),
CHECKPOINT_INITIAL_MESSAGE_ID(3),
PULSAR_SOURCE_BOOTSTRAP_MESSAGE_ID(4),
PULSAR_SOURCE_BOOTSTRAP_GET_MESSAGE_ERROR(5),
FIELD_TYPE_NOT_FOUND(6),
VERSION_UPDATE(7),
CHECKPOINT_START(8),
CHECKPOINT_COMPLETE(9);
private final Integer code;
LogPoint(Integer code) {
this.code = code;
}
@Override
public String toString() {
return String.format("LOP-%06d", code);
}
}
}

37
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/MapHelper.java Normal file
View File

@@ -0,0 +1,37 @@
package com.lanyuanxiaoyao.service.common.utils;
import java.util.Map;
/**
* Map工具类
*
* @author ZhangJiacheng
* @date 2023-03-20
*/
public class MapHelper {
// 如果大小写同时存在，那么这个忽略大小写的方案就会首先返回大写键的值
public static Object getWithoutCase(Map<String, ?> map, String key) {
String upperKey = key.toUpperCase(), lowerKey = key.toLowerCase();
return map.containsKey(upperKey) ? map.get(upperKey) : map.containsKey(lowerKey) ? map.get(lowerKey) : map.get(key);
}
public static String getStringWithoutCase(Map<String, ?> map, String key) {
return (String) getWithoutCase(map, key);
}
public static Integer getIntWithoutCase(Map<String, ?> map, String key) {
return (Integer) getWithoutCase(map, key);
}
public static Long getLongWithoutCase(Map<String, ?> map, String key) {
return (Long) getWithoutCase(map, key);
}
public static Double getDoubleWithoutCase(Map<String, ?> map, String key) {
return (Double) getWithoutCase(map, key);
}
public static Float getFloatWithoutCase(Map<String, ?> map, String key) {
return (Float) getWithoutCase(map, key);
}
}

103
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/NameHelper.java Normal file
View File

@@ -0,0 +1,103 @@
package com.lanyuanxiaoyao.service.common.utils;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.common.Constants;
/**
* 命名相关工具
*
* @author ZhangJiacheng
* @date 2022-06-24
*/
public class NameHelper {
public static String pulsarSubscriptionName(Long flinkJobId, String alias) {
return Constants.PULSAR_SUBSCRIPTION_NAME_PREFIX + "_" + flinkJobId + "_" + alias + "_20230425";
}
// Sync job name
public static final String SYNC_JOB_NAME_REGEX = "^Sync_(\\d+?)_(.+)$";
public static boolean isSyncJob(String name) {
return StrUtil.isNotBlank(name) && name.matches(SYNC_JOB_NAME_REGEX);
}
public static String syncJobName(Long flinkJobId, String flinkJobName) {
return "Sync_" + flinkJobId + "_" + flinkJobName;
}
public static final String COMPACTION_JOB_NAME_REGEX = "^Compaction_(\\d+?)_(.+?)$";
public static boolean isCompactionJob(String name) {
return StrUtil.isNotBlank(name) && name.matches(COMPACTION_JOB_NAME_REGEX);
}
public static String compactionJobName(Long flinkJobId, String alias) {
return "Compaction_" + flinkJobId + "_" + alias;
}
// flink job name
public static String syncFlinkName(Long flinkJobId, String flinkJobName) {
return flinkJobName + " (ID: " + flinkJobId + ")";
}
public static String syncFlinkName(Long flinkJobId, String flinkJobName, String alias) {
return flinkJobName + " " + alias + " (ID: " + flinkJobId + ")";
}
public static String compactionFlinkName(Long flinkJobId, String schema, String alias) {
return schema + " " + alias + " (ID: " + flinkJobId + ")";
}
// sync state name
public static String syncStateName(Long flinkJobId, String alias) {
return flinkJobId + "-" + alias;
}
// zk lock name
public static final String ZK_ROOT_PATH = "/hudi";
public static final String ZK_LOCK_PATH = ZK_ROOT_PATH + "/lock";
public static final String ZK_LAUNCHER_LOCK_PATH = ZK_LOCK_PATH + "/launcher";
public static final String ZK_RUNNING_LOCK_PATH = ZK_LOCK_PATH + "/running";
public static final String ZK_SYNC_SUFFIX_PATH = "/sync";
public static final String ZK_SYNC_LAUNCHER_LOCK_PATH = ZK_LAUNCHER_LOCK_PATH + ZK_SYNC_SUFFIX_PATH;
public static final String ZK_SYNC_RUNNING_LOCK_PATH = ZK_RUNNING_LOCK_PATH + ZK_SYNC_SUFFIX_PATH;
public static String syncLockName(Long flinkJobId, String alias) {
if (StrUtil.isNotBlank(alias)) {
return "sync_lock_" + flinkJobId + "_" + alias;
}
return "sync_lock_" + flinkJobId;
}
public static String syncLauncherLockPath(Long flinkJobId) {
return ZK_SYNC_LAUNCHER_LOCK_PATH + "/" + syncLockName(flinkJobId, null);
}
public static String syncRunningLockPath(Long flinkJobId) {
return syncRunningLockPath(flinkJobId, null);
}
public static String syncRunningLockPath(Long flinkJobId, String alias) {
return ZK_SYNC_RUNNING_LOCK_PATH + "/" + syncLockName(flinkJobId, alias);
}
public static final String ZK_COMPACTION_SUFFIX_PATH = "/compaction";
public static final String ZK_COMPACTION_LAUNCHER_LOCK_PATH = ZK_LAUNCHER_LOCK_PATH + ZK_COMPACTION_SUFFIX_PATH;
public static final String ZK_COMPACTION_RUNNING_LOCK_PATH = ZK_RUNNING_LOCK_PATH + ZK_COMPACTION_SUFFIX_PATH;
public static String compactionLockName(Long flinkJobId, String alias) {
return "compaction_lock_" + flinkJobId + "_" + alias;
}
public static String compactionLauncherLockPath(Long flinkJobId, String alias) {
return ZK_COMPACTION_LAUNCHER_LOCK_PATH + "/" + compactionLockName(flinkJobId, alias);
}
public static String compactionRunningLockPath(Long flinkJobId, String alias) {
return ZK_COMPACTION_RUNNING_LOCK_PATH + "/" + compactionLockName(flinkJobId, alias);
}
}

127
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/RecordHelper.java Normal file
View File

@@ -0,0 +1,127 @@
package com.lanyuanxiaoyao.service.common.utils;
import com.lanyuanxiaoyao.service.common.Constants;
import com.lanyuanxiaoyao.service.common.entity.Record;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Record 工具类
*
* @author ZhangJiacheng
* @version 0.0.1
* @date 2021-12-03
*/
public class RecordHelper {
public static Boolean isNotVersionUpdateRecord(String record) {
return !isVersionUpdateRecord(record);
}
public static Boolean isVersionUpdateRecord(String record) {
return record.contains(Constants.VERSION_UPDATE_KEY);
}
public static Boolean isNotVersionUpdateRecord(Record record) {
return !isVersionUpdateRecord(record);
}
public static Boolean isVersionUpdateRecord(Record record) {
// Record{source=Source{sourceId='versionUpdate', sourceType='null', sourcePos='null', currentTs='2022-11-15 22:17:44'}, statement=Statement{schema='crm_ivpn_cust', table='customer', opStatement='null', opType='version', op='null', opTs='2022-11-15 00:17:43', version='20220925', before=null, after=null}}
return Constants.VERSION_UPDATE_KEY.equals(record.getSource().getSourceId())
&& Constants.VERSION_KEY.equals(record.getStatement().getOpType());
}
/**
* 获取当前的 Statement, 即 source 和 target 选不为空的操作
*
* @param record 消息
* @return 更新的字段内容
*/
public static Map<String, Object> getCurrentStatement(Record record) {
Map<String, Object> before = record.getStatement().getBefore();
Map<String, Object> after = record.getStatement().getAfter();
return (after == null ? before : after.isEmpty() ? before : after);
}
public static Optional<Map<String, Object>> getBeforeStatement(Record record) {
return Optional.ofNullable(record.getStatement().getBefore());
}
public static Optional<Map<String, Object>> getAfterStatement(Record record) {
return Optional.ofNullable(record.getStatement().getAfter());
}
private static Boolean isMapEmpty(Map<?, ?> map) {
return map == null || map.isEmpty();
}
public static Boolean isChangeField(TableMeta meta, Record record, Function<TableMeta, Optional<String>> fieldGetter) {
Map<String, Object> before = record.getStatement().getBefore();
Map<String, Object> after = record.getStatement().getAfter();
Optional<String> field = fieldGetter.apply(meta);
if (isMapEmpty(before) || isMapEmpty(after) || !field.isPresent()) {
return false;
}
Object beforeField = before.getOrDefault(field.get(), null);
Object afterField = after.getOrDefault(field.get(), null);
if (beforeField == null || afterField == null) {
return false;
}
return !Objects.equals(String.valueOf(beforeField), String.valueOf(afterField));
}
public static Map<String, Object> addExtraMetadata(Map<String, Object> current, TableMeta tableMeta, Record record) {
String operationType = record.getStatement().getOpType();
return addExtraMetadata(current, tableMeta, record, Constants.DELETE.equals(operationType));
}
public static Map<String, Object> addExtraMetadata(Map<String, Object> current, TableMeta tableMeta, Record record, Boolean isDelete) {
Map<String, Object> newMap = new HashMap<>(current);
newMap.put(Constants.UNION_KEY_NAME, RecordHelper.createUnionKey(tableMeta, current));
newMap.put(Constants.UPDATE_TIMESTAMP_KEY_NAME, SnowFlakeHelper.next());
newMap.put(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME, record.getStatement().getOpTs());
newMap.put(Constants.HUDI_DELETE_KEY_NAME, isDelete);
return newMap;
}
public static String createUnionKey(TableMeta tableMeta, Record record) {
return createUnionKey(tableMeta, getCurrentStatement(record));
}
private static final String PRIMARY_KEY_NOT_FOUND = "Primary Key Not Found";
/**
* 生成基于主键和分片键的联合主键
*
* @param tableMeta 表信息
* @param fields 字段
* @return 联合主键值
*/
public static String createUnionKey(TableMeta tableMeta, Map<String, Object> fields) {
if (tableMeta.getPrimaryKeys().isEmpty()) {
throw new RuntimeException(PRIMARY_KEY_NOT_FOUND);
}
if (Objects.isNull(fields)) {
throw new RuntimeException("Fields cannot be null");
}
List<String> primaryKeys = tableMeta.getPrimaryKeys()
.stream()
.map(key -> Optional.ofNullable(fields.get(Constants.FIELD_COVERT.apply(tableMeta, key.getName())))
.orElseThrow(() -> new RuntimeException(PRIMARY_KEY_NOT_FOUND + " " + fields))
.toString())
.collect(Collectors.toList());
String primaryKey = String.join("-", primaryKeys);
if (tableMeta.getPartitionKeys().isEmpty()) {
return primaryKey;
} else {
List<String> partitionKeys = tableMeta.getPartitionKeys()
.stream()
.map(key -> fields.get(Constants.FIELD_COVERT.apply(tableMeta, key.getName())).toString())
.collect(Collectors.toList());
String partitionKey = String.join("-", partitionKeys);
return primaryKey + "_" + partitionKey;
}
}
}

72
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SnowFlakeHelper.java Normal file
View File

@@ -0,0 +1,72 @@
package com.lanyuanxiaoyao.service.common.utils;
/**
* 雪花算(pi)法(jiu)
*
* @author ZhangJiacheng
* @date 2020-06-05
*/
public class SnowFlakeHelper {
/**
* 起始的时间戳
*/
private final static long START_TIMESTAMP = 1;
/**
* 序列号占用的位数
*/
private final static long SEQUENCE_BIT = 11;
/**
* 序列号最大值
*/
private final static long MAX_SEQUENCE_BIT = ~(-1 << SEQUENCE_BIT);
/**
* 时间戳值向左位移
*/
private final static long TIMESTAMP_OFFSET = SEQUENCE_BIT;
/**
* 序列号
*/
private static long sequence = 0;
/**
* 上一次时间戳
*/
private static long lastTimestamp = -1;
public static synchronized long next() {
long currentTimestamp = nowTimestamp();
if (currentTimestamp < lastTimestamp) {
throw new RuntimeException("Clock have moved backwards.");
}
if (currentTimestamp == lastTimestamp) {
// 相同毫秒内, 序列号自增
sequence = (sequence + 1) & MAX_SEQUENCE_BIT;
// 同一毫秒的序列数已经达到最大
if (sequence == 0) {
currentTimestamp = nextTimestamp();
}
} else {
// 不同毫秒内, 序列号置为0
sequence = 0;
}
lastTimestamp = currentTimestamp;
return (currentTimestamp - START_TIMESTAMP) << TIMESTAMP_OFFSET | sequence;
}
private static long nextTimestamp() {
long milli = nowTimestamp();
while (milli <= lastTimestamp) {
milli = nowTimestamp();
}
return milli;
}
private static long nowTimestamp() {
return System.currentTimeMillis();
}
}

82
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/SyncStateHelper.java Normal file
View File

@@ -0,0 +1,82 @@
package com.lanyuanxiaoyao.service.common.utils;
import com.lanyuanxiaoyao.service.common.entity.SyncState;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
/**
* @author ZhangJiacheng
* @date 2023-04-24
*/
public class SyncStateHelper {
public static String allSyncStateSql(String database) {
// language=MySQL
return "select info.flink_job_id,\n" +
" info.alias,\n" +
" state.message_id,\n" +
" state.source_start_time,\n" +
" state.source_checkpoint_time,\n" +
" state.source_publish_time,\n" +
" state.source_op_time,\n" +
" state.compaction_start_time,\n" +
" state.compaction_finish_time,\n" +
" state.compaction_application_id,\n" +
" state.compaction_status,\n" +
" state.compaction_status_time,\n" +
" state.compaction_latest_op_ts\n" +
"from " + database + ".tb_app_hudi_sync_state state,\n" +
" " + database + ".tb_app_collect_table_info info\n" +
"where state.id = concat(info.flink_job_id, '-', info.alias)\n" +
" and info.status = 'y'";
}
public static String syncStateSql(String database) {
// language=MySQL
return "select info.flink_job_id,\n" +
" info.alias,\n" +
" state.message_id,\n" +
" state.source_start_time,\n" +
" state.source_checkpoint_time,\n" +
" state.source_publish_time,\n" +
" state.source_op_time,\n" +
" state.compaction_start_time,\n" +
" state.compaction_finish_time,\n" +
" state.compaction_application_id,\n" +
" state.compaction_status,\n" +
" state.compaction_status_time,\n" +
" state.compaction_latest_op_ts\n" +
"from " + database + ".tb_app_hudi_sync_state state,\n" +
" " + database + ".tb_app_collect_table_info info\n" +
"where state.id = concat(info.flink_job_id, '-', info.alias)\n" +
" and info.flink_job_id = ?\n" +
" and info.alias = ?\n" +
" and info.status = 'y'";
}
public static List<SyncState> from(ResultSet rs) throws SQLException {
List<SyncState> results = new ArrayList<>();
Function<Timestamp, Long> dateConvertor = timestamp -> timestamp == null ? 0 : timestamp.getTime();
while (rs.next()) {
results.add(SyncState.builder()
.flinkJobId(rs.getLong(1))
.alias(rs.getString(2))
.messageId(rs.getString(3))
.sourceStartTime(dateConvertor.apply(rs.getTimestamp(4)))
.sourceCheckpointTime(dateConvertor.apply(rs.getTimestamp(5)))
.sourcePublishTime(dateConvertor.apply(rs.getTimestamp(6)))
.sourceOperationTime(dateConvertor.apply(rs.getTimestamp(7)))
.compactionStartTime(dateConvertor.apply(rs.getTimestamp(8)))
.compactionFinishTime(dateConvertor.apply(rs.getTimestamp(9)))
.compactionApplicationId(rs.getString(10))
.compactionStatus(rs.getString(11))
.compactionStatusTime(dateConvertor.apply(rs.getTimestamp(12)))
.compactionLatestOperationTime(dateConvertor.apply(rs.getTimestamp(13)))
.build());
}
return results;
}
}

576
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/TableMetaHelper.java Normal file
View File

@@ -0,0 +1,576 @@
package com.lanyuanxiaoyao.service.common.utils;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import com.lanyuanxiaoyao.service.common.exception.ConfigException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.stream.Collectors;
/**
* Table Meta 工具类
*
* @author ZhangJiacheng
* @version 0.0.1
* @date 2021-12-01
*/
public class TableMetaHelper {
//private static final AES AES = new AES(Mode.CBC, Padding.NoPadding, "6fa22c779ec14b98".getBytes(), "6fa22c779ec14b98".getBytes());
public static String tableMetaSql(String database) {
return tableMetaSql(database, true, false);
}
/*
SqlBuilder.select(
DataSource.DS_NAME,
DataSource.SCHEMA_NAME,
DataSourceTable.TABLE_NAME,
DataSourceTable.TABLE_TYPE,
DataSourceTableField.FIELD_NAME,
DataSourceTableField.FIELD_SEQ,
DataSourceTableField.FIELD_TYPE,
DataSourceTableField.PRIMARY_KEY,
DataSourceTableField.PARTITION_KEY,
DataSourceTableField.LENGTH,
TbAppCollectTableInfo.TGT_DB,
TbAppCollectTableInfo.TGT_TABLE,
TbAppCollectTableInfo.TGT_TABLE_TYPE,
TbAppCollectTableInfo.TGT_HDFS_PATH,
TbAppHudiJobConfig.WRITE_TASKS,
TbAppHudiJobConfig.WRITE_OPERATION,
TbAppHudiJobConfig.WRITE_TASK_MAX_MEMORY,
TbAppHudiJobConfig.WRITE_BATCH_SIZE,
TbAppHudiJobConfig.WRITE_RATE_LIMIT,
TbAppCollectTableInfo.BUCKET_NUMBER,
TbAppHudiJobConfig.COMPACTION_STRATEGY,
TbAppHudiJobConfig.COMPACTION_TASKS,
TbAppHudiJobConfig.COMPACTION_DELTA_COMMITS,
TbAppHudiJobConfig.COMPACTION_DELTA_SECONDS,
TbAppHudiJobConfig.COMPACTION_ASYNC_ENABLED,
TbAppHudiJobConfig.COMPACTION_MAX_MEMORY,
TbAppHudiJobConfig.CONFIGS,
TbAppCollectTableInfo.FILTER_FIELD,
TbAppCollectTableInfo.FILTER_VALUES,
TbAppCollectTableInfo.FILTER_TYPE,
TbAppCollectTableInfo.SRC_TOPIC,
TbAppCollectTableInfo.SRC_PULSAR_ADDR,
Alias.of(TbAppYarnJobConfigSync.JOB_MANAGER_MEMORY, "sync_job_manager_memory"),
Alias.of(TbAppYarnJobConfigSync.TASK_MANAGER_MEMORY, "sync_task_manager_memory"),
Alias.of(TbAppYarnJobConfigCompaction.JOB_MANAGER_MEMORY, "compaction_job_manager_memory"),
Alias.of(TbAppYarnJobConfigCompaction.TASK_MANAGER_MEMORY, "compaction_task_manger_momory"),
TbAppCollectTableInfo.PARTITION_FIELD,
TbAppHudiSyncState.MESSAGE_ID,
TbAppGlobalConfig.METRIC_PUBLISH_URL,
TbAppGlobalConfig.METRIC_PROMETHEUS_URL,
TbAppGlobalConfig.METRIC_API_URL,
TbAppGlobalConfig.METRIC_PUBLISH_DELAY,
TbAppGlobalConfig.METRIC_PUBLISH_PERIOD,
TbAppGlobalConfig.METRIC_PUBLISH_TIMEOUT,
TbAppGlobalConfig.METRIC_PUBLISH_BATCH,
Alias.of(TbAppFlinkJobConfig.ID, "job_id"),
Alias.of(TbAppFlinkJobConfig.NAME, "job_name"),
TbAppGlobalConfig.CHECKPOINT_ROOT_PATH,
TbAppHudiJobConfig.SOURCE_TASKS,
TbAppCollectTableInfo.ALIAS,
DataSource.CONNECTION,
TbAppCollectTableInfo.PRIORITY,
DataSource.DS_TYPE,
TbAppHudiJobConfig.KEEP_FILE_VERSION,
TbAppHudiJobConfig.KEEP_COMMIT_VERSION,
TbAppCollectTableInfo.TAGS,
TbAppGlobalConfig.ZK_URL
)
.from(
DataSource._alias_,
DataSourceTable._alias_,
DataSourceTableField._alias_,
TbAppFlinkJobConfig._alias_,
TbAppHudiJobConfig._alias_,
TbAppYarnJobConfigSync._alias_,
TbAppYarnJobConfigCompaction._alias_,
TbAppGlobalConfig._alias_,
TbAppCollectTableInfo._alias_
)
.leftJoin(TbAppHudiSyncState._alias_)
.onEq(TbAppHudiSyncState.ID, Column.as(StrUtil.format("concat({}, '-', {})", TbAppCollectTableInfo.FLINK_JOB_ID, TbAppCollectTableInfo.ALIAS)))
.whereEq(DataSource.DS_ROLE, "src")
.andEq(DataSource.DS_STATE, "y")
.andEq(DataSource.RECORD_STATE, "y")
.andEq(DataSourceTable.DS_ID, Column.as(DataSource.DS_ID))
.andEq(DataSourceTable.RECORD_STATE, "y")
.andEq(DataSourceTableField.TABLE_ID, Column.as(DataSourceTable.TABLE_ID))
.andEq(DataSourceTableField.RECORD_STATE, "y")
.andIn(DataSource.DS_TYPE, "udal", "telepg")
.andEq(DataSource.DS_NAME, Column.as(TbAppCollectTableInfo.SRC_DB))
.andEq(DataSource.SCHEMA_NAME, Column.as(TbAppCollectTableInfo.SRC_SCHEMA))
.andEq(DataSourceTable.TABLE_NAME, Column.as(TbAppCollectTableInfo.SRC_TABLE))
.andEq(TbAppCollectTableInfo.FLINK_JOB_ID, Column.as(TbAppFlinkJobConfig.ID))
.andEq(TbAppCollectTableInfo.HUDI_JOB_ID, Column.as(TbAppHudiJobConfig.ID))
.andEq(TbAppCollectTableInfo.SYNC_YARN_JOB_ID, Column.as(TbAppYarnJobConfigSync.ID))
.andEq(TbAppCollectTableInfo.COMPACTION_YARN_JOB_ID, Column.as(TbAppYarnJobConfigCompaction.ID))
.andEq(TbAppCollectTableInfo.CONFIG_ID, Column.as(TbAppGlobalConfig.ID))
.andEq(TbAppFlinkJobConfig.ID, 1542097984132706304L)
.andEq(TbAppCollectTableInfo.ALIAS, "crm_cfguse_channel")
.andEq(TbAppCollectTableInfo.STATUS, "y")
.andEq(TbAppFlinkJobConfig.STATUS, "y")
.andEq(TbAppHudiJobConfig.STATUS, "y")
.andEq(TbAppYarnJobConfigSync.STATUS, "y")
.andEq(TbAppYarnJobConfigCompaction.STATUS, "y")
.orderBy(DataSourceTableField.FIELD_SEQ)
.build()
*/
public static String tableMetaSql(String database, Boolean filterByFlinkJobId, Boolean filterByAlias) {
// language=MySQL
return "select dst.ds_name,\n" +
" dst.schema_name,\n" +
" dst.table_name,\n" +
" dst.table_type,\n" +
" dstf.field_name,\n" +
" dstf.field_seq,\n" +
" dstf.field_type,\n" +
" dstf.primary_key,\n" +
" dstf.partition_key,\n" +
" dstf.length,\n" +
" tacti.tgt_db,\n" +
" tacti.tgt_table,\n" +
" tacti.tgt_table_type,\n" +
" tacti.tgt_hdfs_path,\n" +
" tajhc.write_tasks,\n" +
" tajhc.write_operation,\n" +
" tajhc.write_task_max_memory,\n" +
" tajhc.write_batch_size,\n" +
" tajhc.write_rate_limit,\n" +
" tacti.bucket_number,\n" +
" tajhc.compaction_strategy,\n" +
" tajhc.compaction_tasks,\n" +
" tajhc.compaction_delta_commits,\n" +
" tajhc.compaction_delta_seconds,\n" +
" tajhc.compaction_async_enabled,\n" +
" tajhc.compaction_max_memory,\n" +
" tajhc.configs,\n" +
" tacti.filter_field,\n" +
" tacti.filter_values,\n" +
" tacti.filter_type,\n" +
" tacti.src_topic,\n" +
" tacti.src_pulsar_addr,\n" +
" tayjc_sync.job_manager_memory as sync_job_manager_memory,\n" +
" tayjc_sync.task_manager_memory as sync_task_manager_memory,\n" +
" tayjc_compaction.job_manager_memory as compaction_job_manager_memory,\n" +
" tayjc_compaction.task_manager_memory as compaction_task_manger_momory,\n" +
" tacti.partition_field,\n" +
" tahss.message_id,\n" +
" tagc.metric_publish_url,\n" +
" tagc.metric_prometheus_url,\n" +
" tagc.metric_api_url,\n" +
" tagc.metric_publish_delay,\n" +
" tagc.metric_publish_period,\n" +
" tagc.metric_publish_timeout,\n" +
" tagc.metric_publish_batch,\n" +
" tafjc.id as job_id,\n" +
" tafjc.name as job_name,\n" +
" tagc.checkpoint_root_path,\n" +
" tajhc.source_tasks,\n" +
" tacti.alias,\n" +
" dst.connection,\n" +
" tacti.priority,\n" +
" dst.ds_type,\n" +
" tajhc.keep_file_version,\n" +
" tajhc.keep_commit_version,\n" +
" tacti.tags,\n" +
" tagc.zk_url,\n" +
" tacti.version,\n" +
" dstf.scale\n" +
"from `" + database + "`.tb_app_collect_table_info tacti\n" +
" left join `" + database + "`.tb_app_hudi_sync_state tahss\n" +
" on tahss.id = concat(tacti.flink_job_id, '-', tacti.alias),\n" +
" `" + database + "`.tb_app_flink_job_config tafjc,\n" +
" `" + database + "`.tb_app_hudi_job_config tajhc,\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc_sync,\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc_compaction,\n" +
" `" + database + "`.tb_app_global_config tagc,\n" +
" `" + database + "`.tb_app_hudi_compaction_schedule tahcs,\n" +
" `iap-datahub`.data_source_table_field dstf,\n" +
" (select ds.*, dst.table_id, dst.table_name, dst.table_type\n" +
" from `iap-datahub`.data_source_table dst,\n" +
" (select ds.ds_id, ds.ds_name, ds.ds_type, ds.schema_name, ds.connection\n" +
" from `iap-datahub`.data_source ds\n" +
" where ds.ds_role = 'src'\n" +
" and ds.ds_state = 'y'\n" +
" and ds.record_state = 'y') ds\n" +
" where dst.ds_id = ds.ds_id\n" +
" and dst.record_state = 'y') dst\n" +
"where dstf.table_id = dst.table_id\n" +
" and dstf.record_state = 'y'\n" +
" and dst.ds_type in ('udal', 'telepg')\n" +
" and dst.ds_name = tacti.src_db\n" +
" and dst.schema_name = tacti.src_schema\n" +
" and dst.table_name = tacti.src_table\n" +
" and tacti.flink_job_id = tafjc.id\n" +
" and tacti.hudi_job_id = tajhc.id\n" +
" and tacti.sync_yarn_job_id = tayjc_sync.id\n" +
" and tacti.compaction_yarn_job_id = tayjc_compaction.id\n" +
" and tacti.config_id = tagc.id\n" +
" and tacti.schedule_id = tahcs.id\n" +
(filterByFlinkJobId ? " and tafjc.id = ?\n" : "") +
(filterByAlias ? " and tacti.alias = ?\n" : "") +
" and tacti.status = 'y'\n" +
" and tafjc.status = 'y'\n" +
" and tajhc.status = 'y'\n" +
" and tayjc_sync.status = 'y'\n" +
" and tayjc_compaction.status = 'y'\n" +
"order by dstf.field_seq;";
}
public static List<TableMeta> from(ResultSet rs) throws SQLException {
List<TableMeta> results = new ArrayList<>();
List<TableMeta.RowMeta> metaList = new ArrayList<>();
while (rs.next( )) {
metaList.add(
TableMeta.RowMeta.builder()
.dsName(rs.getString(1))
.schemaName(rs.getString(2))
.tableName(rs.getString(3))
.tableType(rs.getString(4))
.fieldName(rs.getString(5))
.fieldSeq(rs.getInt(6))
.fieldType(rs.getString(7))
.primaryKey(rs.getString(8))
.partitionKey(rs.getString(9))
.length(rs.getLong(10))
.tgtDb(rs.getString(11))
.tgtTable(rs.getString(12))
.tgtTableType(rs.getString(13))
.tgtHdfsPath(rs.getString(14))
.writeTasks(rs.getInt(15))
.writeOperation(rs.getString(16))
.writeTaskMaxMemory(rs.getInt(17))
.writeBatchSize(rs.getInt(18))
.writeRateLimit(rs.getInt(19))
.bucketIndexNumber(rs.getInt(20))
.compactionStrategy(rs.getString(21))
.compactionTasks(rs.getInt(22))
.compactionDeltaCommits(rs.getInt(23))
.compactionDeltaSeconds(rs.getInt(24))
.compactionAsyncEnabled(rs.getString(25))
.compactionMaxMemory(rs.getInt(26))
.configs(rs.getString(27))
.filterField(rs.getString(28))
.filterValues(rs.getString(29))
.filterType(rs.getString(30))
.topic(rs.getString(31))
.pulsarAddress(rs.getString(32))
.syncJobManagerMemory(rs.getInt(33))
.syncTaskManagerMemory(rs.getInt(34))
.compactionJobManagerMemory(rs.getInt(35))
.compactionTaskManagerMemory(rs.getInt(36))
.partitionField(rs.getString(37))
.messageId(rs.getString(38))
.metricPublishUrl(rs.getString(39))
.metricPrometheusUrl(rs.getString(40))
.metricApiUrl(rs.getString(41))
.metricPublishDelay(rs.getInt(42))
.metricPublishPeriod(rs.getInt(43))
.metricPublishTimeout(rs.getInt(44))
.metricPublishBatch(rs.getInt(45))
.jobId(rs.getLong(46))
.jobName(rs.getString(47))
.checkpointRootPath(rs.getString(48))
.sourceTasks(rs.getInt(49))
.alias(rs.getString(50))
.connection(rs.getString(51))
.priority(rs.getInt(52))
.sourceType(rs.getString(53))
.keepFileVersion(rs.getInt(54))
.keepCommitVersion(rs.getInt(55))
.tags(rs.getString(56))
.zookeeperUrl(rs.getString(57))
.version(rs.getInt(58))
.scala(rs.getInt(59))
.build()
);
}
metaList.stream()
.collect(Collectors.groupingBy(TableMeta.RowMeta::getAlias))
.values()
.stream()
.flatMap(schemaRowMetas -> schemaRowMetas
.stream()
.collect(Collectors.groupingBy(TableMeta.RowMeta::getTableName))
.values()
.stream()
.map(tableRowMetas -> {
try {
return fromRowMetas(tableRowMetas);
} catch (Exception e) {
throw new RuntimeException(e);
}
}))
.forEach(results::add);
return results;
}
private static void checkMoreThanOne(String fieldName, Collection<?> collection) throws ConfigException {
ConfigException.check(fieldName + " cannot be more than 1", () -> collection.size() > 1);
}
private static void checkEmpty(String fieldName, Collection<?> collection) throws ConfigException {
ConfigException.check(fieldName + " cannot be empty", collection::isEmpty);
}
private static void checkEmptyOrMoreThanOne(String fieldName, Collection<?> collection) throws ConfigException {
checkEmpty(fieldName, collection);
checkMoreThanOne(fieldName, collection);
}
public static TableMeta fromRowMetas(List<TableMeta.RowMeta> metaList) throws Exception {
List<String> aliasList = metaList.stream()
.map(TableMeta.RowMeta::getAlias)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("alias", aliasList);
String alias = aliasList.get(0);
List<String> sourceTypeList = metaList.stream()
.map(TableMeta.RowMeta::getSourceType)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("source_type", sourceTypeList);
String sourceTypeText = sourceTypeList.get(0).toUpperCase();
TableMeta.SourceType sourceType;
try {
sourceType = TableMeta.SourceType.valueOf(sourceTypeText);
} catch (IllegalArgumentException e) {
throw new Exception("Cannot parse source type " + sourceTypeText);
}
List<String> dsNames = metaList.stream()
.map(TableMeta.RowMeta::getDsName)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("ds_name", dsNames);
String dataSource = dsNames.get(0);
List<String> schemaNames = metaList.stream()
.map(TableMeta.RowMeta::getSchemaName)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("schema_name", schemaNames);
String schema = schemaNames.get(0);
List<String> tableNames = metaList.stream()
.map(TableMeta.RowMeta::getTableName)
.distinct()
.collect(Collectors.toList());
// 每次只能获取 1 张表的元信息
checkMoreThanOne("table_name", tableNames);
checkEmpty("table_name", tableNames);
String table = tableNames.get(0);
List<String> tableTypes = metaList.stream()
.map(TableMeta.RowMeta::getTableType)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("table_type", tableTypes);
String type = tableTypes.get(0);
List<String> filterFields = metaList.stream()
.map(TableMeta.RowMeta::getFilterField)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("filter_field", filterFields);
String filterField = filterFields.get(0);
List<String> filterValueList = metaList.stream()
.map(TableMeta.RowMeta::getFilterValues)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("filter_values", filterValueList);
String filterValuesText = filterValueList.get(0);
List<String> filterValues = (filterValuesText == null || filterValuesText.isEmpty())
? Collections.emptyList()
: Arrays.asList(filterValuesText.split(","));
List<String> filterTypes = metaList.stream()
.map(TableMeta.RowMeta::getFilterType)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("filter_field", filterFields);
TableMeta.FilterType filterType;
try {
filterType = TableMeta.FilterType.valueOf(filterTypes.get(0));
} catch (IllegalArgumentException e) {
filterType = TableMeta.FilterType.NONE;
}
List<String> topics = metaList.stream()
.map(TableMeta.RowMeta::getTopic)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("topic", topics);
String topic = topics.get(0);
List<String> pulsarAddresses = metaList.stream()
.map(TableMeta.RowMeta::getPulsarAddress)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("pulsar address", pulsarAddresses);
String pulsarAddress = pulsarAddresses.get(0);
List<Integer> priorities = metaList.stream()
.map(TableMeta.RowMeta::getPriority)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("priority", priorities);
Integer priority = priorities.get(0);
List<String> tagTexts = metaList.stream()
.map(TableMeta.RowMeta::getTags)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("tags", tagTexts);
String tagText = tagTexts.get(0) == null ? "" : tagTexts.get(0);
List<String> tags = Arrays.asList(tagText.split(","));
List<Integer> versions = metaList.stream()
.map(TableMeta.RowMeta::getVersion)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("version", versions);
Integer version = versions.get(0);
// 获取 Hudi 配置, 因为查出来同一张表的配置都相同, 所以直接取第一条即可
TableMeta.RowMeta example = metaList.get(0);
TableMeta.HudiMeta hudiMeta = TableMeta.HudiMeta.builder()
.targetDataSource(example.getTgtDb())
.targetTable(example.getTgtTable())
.targetTableType(example.getTgtTableType())
.targetHdfsPath(example.getTgtHdfsPath())
.sourceTasks(example.getSourceTasks())
.writeTasks(example.getWriteTasks())
.writeOperation(example.getWriteOperation())
.writeTaskMaxMemory(example.getWriteTaskMaxMemory())
.writeBatchSize(example.getWriteBatchSize())
.writeRateLimit(example.getWriteRateLimit())
.bucketIndexNumber(example.getBucketIndexNumber())
.compactionStrategy(example.getCompactionStrategy())
.compactionTasks(example.getCompactionTasks())
.compactionDeltaCommits(example.getCompactionDeltaCommits())
.compactionDeltaSeconds(example.getCompactionDeltaSeconds())
.compactionAsyncEnabled(example.getCompactionAsyncEnabled())
.compactionMaxMemory(example.getCompactionMaxMemory())
.configs(example.getConfigs())
.keepFileVersion(example.getKeepFileVersion())
.keepCommitVersion(example.getKeepCommitVersion())
.build();
TableMeta.YarnMeta syncYarnMeta = TableMeta.YarnMeta.builder()
.jobManagerMemory(example.getSyncJobManagerMemory())
.taskManagerMemory(example.getSyncTaskManagerMemory())
.build();
TableMeta.YarnMeta compactionYarnMeta = TableMeta.YarnMeta.builder()
.jobManagerMemory(example.getCompactionJobManagerMemory())
.taskManagerMemory(example.getCompactionTaskManagerMemory())
.build();
TableMeta.ConfigMeta configMeta = TableMeta.ConfigMeta.builder()
.messageId(example.getMessageId())
.metricPublishUrl(example.getMetricPublishUrl())
.metricPrometheusUrl(example.getMetricPrometheusUrl())
.metricApiUrl(example.getMetricApiUrl())
.metricPublishDelay(example.getMetricPublishDelay())
.metricPublishPeriod(example.getMetricPublishPeriod())
.metricPublishTimeout(example.getMetricPublishTimeout())
.metricPublishBatch(example.getMetricPublishBatch())
.checkpointRootPath(example.getCheckpointRootPath())
.zookeeperUrl(example.getZookeeperUrl())
.build();
TableMeta.JobMeta jobMeta = TableMeta.JobMeta.builder()
.id(example.getJobId())
.name(example.getJobName())
.build();
TableMeta.ConnectionMeta connectionMeta = null;
String connectionText = example.getConnection();
if (connectionText != null && !connectionText.isEmpty()) {
JSONObject connectionObj = JSONUtil.parseObj(connectionText);
connectionMeta = TableMeta.ConnectionMeta.builder()
.url(connectionObj.getStr("jdbc_url"))
.user(connectionObj.getStr("jdbc_user"))
.password(connectionObj.getStr("jdbc_password"))
.driver(connectionObj.getStr("jdbc_driver"))
.build();
}
List<String> partitionFields = metaList.stream()
.map(TableMeta.RowMeta::getPartitionField)
.distinct()
.collect(Collectors.toList());
checkEmptyOrMoreThanOne("partition_field", filterFields);
String partitionField = partitionFields.get(0);
List<TableMeta.FieldMeta> primaryKeys = new ArrayList<>(), partitionKeys = new ArrayList<>();
List<TableMeta.FieldMeta> fieldMetaList = new ArrayList<>(metaList.size());
for (TableMeta.RowMeta rowMeta : metaList) {
boolean isPrimaryKey = "y".equals(rowMeta.getPrimaryKey());
boolean isPartitionKey = "y".equals(rowMeta.getPartitionKey());
TableMeta.FieldMeta fieldMeta = TableMeta.FieldMeta.builder()
.name(rowMeta.getFieldName().toUpperCase(Locale.ROOT))
.sequence(rowMeta.getFieldSeq())
.type(rowMeta.getFieldType())
.isPrimaryKey(isPrimaryKey)
.partitionKey(isPartitionKey)
.length(rowMeta.getLength())
.scala(rowMeta.getScala())
.build();
if (isPrimaryKey) {
primaryKeys.add(fieldMeta);
}
if (isPartitionKey) {
partitionKeys.add(fieldMeta);
}
fieldMetaList.add(fieldMeta);
}
return TableMeta.builder()
.alias(alias)
.source(dataSource)
.schema(schema)
.table(table)
.type(type)
.primaryKeys(primaryKeys)
.partitionKeys(partitionKeys)
.hudi(hudiMeta)
.fields(fieldMetaList)
.filterField(filterField)
.filterValues(filterValues)
.filterType(filterType)
.topic(topic)
.pulsarAddress(pulsarAddress)
.syncYarn(syncYarnMeta)
.compactionYarn(compactionYarnMeta)
.partitionField(partitionField)
.config(configMeta)
.job(jobMeta)
.connection(connectionMeta)
.priority(priority)
.sourceType(sourceType)
.tags(tags)
.version(version)
.build();
}
public static Optional<String> getPartitionField(TableMeta meta) {
if (meta.getPartitionField() == null || "".equals(meta.getPartitionField())) {
return Optional.empty();
} else {
return meta.getFields()
.stream()
.map(TableMeta.FieldMeta::getName)
.filter(name -> meta.getPartitionField().equalsIgnoreCase(name))
.findFirst();
}
}
public static Optional<String> getFilterField(TableMeta meta) {
if (meta.getFilterField() == null || "".equals(meta.getFilterField())) {
return Optional.empty();
} else {
return meta.getFields()
.stream()
.map(TableMeta.FieldMeta::getName)
.filter(name -> meta.getFilterField().equalsIgnoreCase(name))
.findFirst();
}
}
public static boolean existsTag(TableMeta meta, String tag) {
return meta.getTags() != null && meta.getTags().contains(tag);
}
}