lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(all): 迁移common、sync、executor项目

Browse Source
This commit is contained in:
兰缘小妖
2024-02-29 15:32:14 +08:00
parent 0683068a02
commit 5a2e9fdfb8
73 changed files with 10204 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
service-common/src/main/java/com/lanyuanxiaoyao/service/common/Constants.java Normal file
View File

@@ -0,0 +1,211 @@
package com.lanyuanxiaoyao.service.common;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import java.time.format.DateTimeFormatter;
import java.util.function.BiFunction;
/**
* 常量
*
* @author ZhangJiacheng
* @version 0.0.1
* @date 2021-12-03
*/
public interface Constants {
// String DATABASE_NAME = "hudi_collect_build";
// String DATABASE_NAME = "hudi_collect_build_2";
String DATABASE_NAME = "hudi_collect_build_b12";
String API_HEADER_NAME = "Api-Version";
String API_VERSION = "1";
/**
* 联合主键
*/
String UNION_KEY_NAME = "_key";
/**
* 源端最后操作时间
*/
String LATEST_OPERATION_TIMESTAMP_KEY_NAME = "latest_op_ts";
/**
* 记录下游入库时间
*/
String UPDATE_TIMESTAMP_KEY_NAME = "update_ts";
/**
* Hudi 删除标记字段
*/
String HUDI_DELETE_KEY_NAME = "_hoodie_is_deleted";
String PULSAR_SUBSCRIPTION_NAME_PREFIX = "Hudi_Sync_Pulsar_Reader";
String VERSION_UPDATE_KEY = "versionUpdate";
String VERSION_KEY = "version";
String DELETE = "D";
String INSERT = "I";
String UPDATE = "U";
String DDL = "ddl";
String UNKNOWN = "unknown";
String CITY_ID = "CITY_ID";
String INCLUDE = "INCLUDE";
String EXCLUDE = "EXCLUDE";
String JOB_ID = "job-id";
String SERVICE_MODE = "service-mode";
String FLINK_JOB = "flink-job";
String TABLE_META = "table-meta";
String TABLE_META_LIST = "table-meta-list";
String MESSAGE_ID = "message-id";
String INSTANTS = "instants";
String BETA = "beta";
String CLUSTER = "cluster";
String COW = "COPY_ON_WRITE";
String MOR = "MERGE_ON_READ";
String FLINK_JOB_OPTION = "-" + FLINK_JOB;
String TABLE_META_OPTION = "-" + TABLE_META;
String TABLE_META_LIST_OPTION = "-" + TABLE_META_LIST;
String INSTANTS_OPTION = "-" + INSTANTS;
String BETA_OPTION = "-" + BETA;
String CLUSTER_OPTION = "-" + CLUSTER;
String SPRING_SECURITY_AUTHORITY = "Anonymous";
String SPRING_SECURITY_USERNAME = "AxhEbscwsJDbYMH2";
String SPRING_SECURITY_PASSWORD = "{noop}cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4";
String SPRING_SECURITY_PASSWORD_PLAIN = "cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4";
String VICTORIA_USERNAME = "EsCFVuNkiDWv7PKmcF";
String VICTORIA_PASSWORD = "Abf%x9ocS^iKr3tgrd";
String SCHEMA_NAME = "schema";
String TABLE_NAME = "table";
String DATA_TIME = "data-time";
String DATA_PARENT_PATH = "data-parent-path";
String METRICS_PREFIX = "metrics_hudi";
String METRICS_YARN_PREFIX = METRICS_PREFIX + "_yarn";
String METRICS_YARN_JOB = METRICS_YARN_PREFIX + "_job";
String METRICS_YARN_TABLE = METRICS_YARN_PREFIX + "_table";
String METRICS_SYNC_PREFIX = METRICS_PREFIX + "_sync";
String METRICS_SYNC_SOURCE_LATENCY = METRICS_SYNC_PREFIX + "_source_latency";
String METRICS_SYNC_LATENCY = METRICS_SYNC_PREFIX + "_latency";
String METRICS_SYNC_FLINK_JOB_ID = METRICS_SYNC_PREFIX + "_flink_job_id";
String METRICS_SYNC_SOURCE_MESSAGE_RECEIVE = METRICS_SYNC_PREFIX + "_source_message_receive";
String METRICS_SYNC_SOURCE_MESSAGE_SIZE_RECEIVE_BYTES = METRICS_SYNC_PREFIX + "_source_message_receive_bytes";
String METRICS_SYNC_SOURCE_OPERATION_TYPE_RECEIVE = METRICS_SYNC_PREFIX + "_source_operation_type_receive";
String METRICS_SYNC_SOURCE_CHANGE_FILTER = METRICS_SYNC_PREFIX + "_source_change_filter";
String METRICS_SYNC_SOURCE_CHANGE_PARTITION = METRICS_SYNC_PREFIX + "_source_change_partition";
String METRICS_SYNC_SOURCE_BACK_LOGS = METRICS_SYNC_PREFIX + "_source_back_logs";
String METRICS_LABEL_FLINK_JOB_ID = "flink_job_id";
String METRICS_LABEL_FLINK_JOB_NAME = "flink_job_name";
String METRICS_LABEL_FLINK_NATIVE_JOB_ID = "flink_native_job_id";
String METRICS_LABEL_FLINK_NATIVE_TASK_NAME = "flink_native_task_name";
String METRICS_LABEL_FLINK_PARALLEL_ID = "flink_parallel_id";
String METRICS_LABEL_RUN_TYPE = "run_type";
String METRICS_LABEL_EXECUTOR_VERSION = "executor_version";
String METRICS_LABEL_CLUSTER = "cluster";
String METRICS_RUN_TYPE_SYNC = "sync";
String METRICS_RUN_TYPE_COMPACTION = "compaction";
String METRICS_LABEL_SCHEMA = "schema";
String METRICS_LABEL_TABLE = "table";
String METRICS_LABEL_STATUS = "status";
String METRICS_LABEL_TOPIC = "topic";
String METRICS_LABEL_BATCH_ID = "batch_id";
String METRICS_LABEL_ALIAS = "alias";
String METRICS_LABEL_APPLICATION_ID = "application_id";
String METRICS_STATUS_RUNNING = "running";
String METRICS_STATUS_STOPPED = "stopped";
String METRICS_LABEL_TYPE = "type";
String LOKI_PUSH_URL = "loki_push_url";
DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
String OPERATION_DONE = "操作完成";
String OPERATION_CANCEL = "操作取消";
String FETCHING_DATA = "Fetching Data";
String COMPACTION_STATUS_SCHEDULE = "SCHEDULE";
String COMPACTION_STATUS_START = "START";
String COMPACTION_STATUS_FINISH = "FINISH";
String COMPACTION_STATUS_FAILURE = "FAILURE";
long SECOND = 1000;
long HALF_MINUTE = 30 * SECOND;
long MINUTE = 60 * SECOND;
long HALF_HOUR = 30 * MINUTE;
long HOUR = 60 * MINUTE;
long KB = 1024;
long MB = 1024 * KB;
long GB = 1024 * MB;
long TB = 1024 * GB;
String TAG_SPLIT = ";";
String TAG_OPERATOR = "=";
String EVENT = "event";
String FROM_COMMAND_UTIL = "command util";
String FROM_COMPACTOR = "compactor";
int COMMAND_RENDER_WIDTH = 500;
String LOG_FLINK_JOB_ID_LABEL = "LOG_FLINK_JOB_ID_LABEL";
String LOG_FLINK_JOB_ID = "flink_job_id";
String LOG_ALIAS_LABEL = "LOG_ALIAS_LABEL";
String LOG_ALIAS = "alias";
String LOG_JOB_ID_LABEL = "LOG_JOB_ID_LABEL";
String LOG_JOB_ID = "job_id";
String LOG_POINT_PREFIX = "LOP-";
String LOG_POINT_MESSAGE_ID_EMPTY = LOG_POINT_PREFIX + "000001";
String LOG_POINT_CHECKPOINT_INITIAL = LOG_POINT_PREFIX + "000002";
String LOG_POINT_CHECKPOINT_INITIAL_MESSAGE_ID = LOG_POINT_PREFIX + "000003";
String LOG_POINT_PULSAR_SOURCE_BOOTSTRAP_MESSAGE_ID = LOG_POINT_PREFIX + "000004";
String LOG_POINT_PULSAR_SOURCE_GET_MESSAGE_ID_ERROR = LOG_POINT_PREFIX + "000005";
String LOG_POINT_FIELD_TYPE_NOT_FOUND = LOG_POINT_PREFIX + "000006";
String TAGS_NO_COMPACT = "NO_COMPACT";
String TAGS_PULSAR_BACKUP = "PULSAR_BACKUP";
String TAGS_NO_PRE_COMBINE = "NO_PRE_COMBINE";
String TAGS_PRE_COMBINE = "PRE_COMBINE";
String TAGS_NO_IGNORE_FAILED = "NO_IGNORE_FAILED";
String TAGS_DISABLE_CHAINING = "DISABLE_CHAINING";
String TAGS_TRACE_LATEST_OP_TS = "TRACE_LATEST_OP_TS";
String TAGS_SOURCE_READER = "SOURCE_READER";
String TAGS_USE_TEST_JAR = "USE_TEST_JAR";
String TAGS_ODS = "ODS";
String TAGS_ODS_FOCUS = "ODS_FOCUS";
String COMPACTION_QUEUE_PRE = "compaction-queue-pre";
String COMPACTION_QUEUE_B1 = "compaction-queue-b1";
String COMPACTION_QUEUE_B5 = "compaction-queue-b5";
String COMPACTION_QUEUE_A4 = "compaction-queue-a4";
String COMPACTION_QUEUE_B12 = "compaction-queue-b12";
String CLUSTER_B1 = "b1";
String CLUSTER_B5 = "b5";
String CLUSTER_A4 = "a4";
String CLUSTER_B12 = "b12";
String SCHEDULE_JOB_FAIL_COUNT = "SCHEDULE_JOB_FAIL_COUNT";
String SCHEDULE_RECOMMEND = "schedule_recommend";
String SCHEDULE_FORCE = "schedule_force";
BiFunction<TableMeta, String, String> FIELD_COVERT = (tableMeta, field) -> {
if (TableMeta.SourceType.TELEPG.equals(tableMeta.getSourceType())) {
return field.toLowerCase();
} else {
return field;
}
};
}