lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(scheduler): 使用简化的table_meta加速查询

Browse Source
This commit is contained in:
v-zhangjc9
2024-06-03 18:35:33 +08:00
parent 6f2fce4359
commit 24e4672dc3
7 changed files with 181 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

216
service-common/src/main/java/com/lanyuanxiaoyao/service/common/utils/TableMetaHelper.java
View File

@@ -6,7 +6,13 @@ import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import com.lanyuanxiaoyao.service.common.exception.ConfigException;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
import java.util.stream.Collectors;
/**
@@ -17,7 +23,7 @@ import java.util.stream.Collectors;
* @date 2021-12-01
*/
public class TableMetaHelper {
//private static final AES AES = new AES(Mode.CBC, Padding.NoPadding, "6fa22c779ec14b98".getBytes(), "6fa22c779ec14b98".getBytes());
// private static final AES AES = new AES(Mode.CBC, Padding.NoPadding, "6fa22c779ec14b98".getBytes(), "6fa22c779ec14b98".getBytes());
public static String tableMetaSql(String database) {
return tableMetaSql(database, true, false);
@@ -125,109 +131,109 @@ public class TableMetaHelper {
public static String tableMetaSql(String database, Boolean filterByFlinkJobId, Boolean filterByAlias) {
// language=MySQL
return "select dst.ds_name,\n" +
" dst.schema_name,\n" +
" dst.table_name,\n" +
" dst.table_type,\n" +
" dstf.field_name,\n" +
" dstf.field_seq,\n" +
" dstf.field_type,\n" +
" dstf.primary_key,\n" +
" dstf.partition_key,\n" +
" dstf.length,\n" +
" tacti.tgt_db,\n" +
" tacti.tgt_table,\n" +
" tacti.tgt_table_type,\n" +
" tacti.tgt_hdfs_path,\n" +
" tajhc.write_tasks,\n" +
" tajhc.write_operation,\n" +
" tajhc.write_task_max_memory,\n" +
" tajhc.write_batch_size,\n" +
" tajhc.write_rate_limit,\n" +
" tacti.bucket_number,\n" +
" tajhc.compaction_strategy,\n" +
" tajhc.compaction_tasks,\n" +
" tajhc.compaction_delta_commits,\n" +
" tajhc.compaction_delta_seconds,\n" +
" tajhc.compaction_async_enabled,\n" +
" tajhc.compaction_max_memory,\n" +
" tajhc.configs,\n" +
" tacti.filter_field,\n" +
" tacti.filter_values,\n" +
" tacti.filter_type,\n" +
" tacti.src_topic,\n" +
" tacti.src_pulsar_addr,\n" +
" tayjc_sync.job_manager_memory as sync_job_manager_memory,\n" +
" tayjc_sync.task_manager_memory as sync_task_manager_memory,\n" +
" tayjc_compaction.job_manager_memory as compaction_job_manager_memory,\n" +
" tayjc_compaction.task_manager_memory as compaction_task_manger_momory,\n" +
" tacti.partition_field,\n" +
" tahss.message_id,\n" +
" tagc.metric_publish_url,\n" +
" tagc.metric_prometheus_url,\n" +
" tagc.metric_api_url,\n" +
" tagc.metric_publish_delay,\n" +
" tagc.metric_publish_period,\n" +
" tagc.metric_publish_timeout,\n" +
" tagc.metric_publish_batch,\n" +
" tafjc.id as job_id,\n" +
" tafjc.name as job_name,\n" +
" tagc.checkpoint_root_path,\n" +
" tajhc.source_tasks,\n" +
" tacti.alias,\n" +
" dst.connection,\n" +
" tacti.priority,\n" +
" dst.ds_type,\n" +
" tajhc.keep_file_version,\n" +
" tajhc.keep_commit_version,\n" +
" tacti.tags,\n" +
" tagc.zk_url,\n" +
" tacti.version,\n" +
" dstf.scale\n" +
"from `" + database + "`.tb_app_collect_table_info tacti\n" +
" left join `" + database + "`.tb_app_hudi_sync_state tahss\n" +
" on tahss.id = concat(tacti.flink_job_id, '-', tacti.alias),\n" +
" `" + database + "`.tb_app_flink_job_config tafjc,\n" +
" `" + database + "`.tb_app_hudi_job_config tajhc,\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc_sync,\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc_compaction,\n" +
" `" + database + "`.tb_app_global_config tagc,\n" +
" `" + database + "`.tb_app_hudi_compaction_schedule tahcs,\n" +
" `iap-datahub`.data_source_table_field dstf,\n" +
" (select ds.*, dst.table_id, dst.table_name, dst.table_type\n" +
" from `iap-datahub`.data_source_table dst,\n" +
" (select ds.ds_id, ds.ds_name, ds.ds_type, ds.schema_name, ds.connection\n" +
" from `iap-datahub`.data_source ds\n" +
" where ds.ds_role = 'src'\n" +
" and ds.ds_state = 'y'\n" +
" and ds.record_state = 'y') ds\n" +
" where dst.ds_id = ds.ds_id\n" +
" and dst.record_state = 'y') dst\n" +
"where dstf.table_id = dst.table_id\n" +
" and dstf.record_state = 'y'\n" +
" and dst.ds_type in ('udal', 'telepg')\n" +
" and dst.ds_name = tacti.src_db\n" +
" and dst.schema_name = tacti.src_schema\n" +
" and dst.table_name = tacti.src_table\n" +
" and tacti.flink_job_id = tafjc.id\n" +
" and tacti.hudi_job_id = tajhc.id\n" +
" and tacti.sync_yarn_job_id = tayjc_sync.id\n" +
" and tacti.compaction_yarn_job_id = tayjc_compaction.id\n" +
" and tacti.config_id = tagc.id\n" +
" and tacti.schedule_id = tahcs.id\n" +
(filterByFlinkJobId ? " and tafjc.id = ?\n" : "") +
(filterByAlias ? " and tacti.alias = ?\n" : "") +
" and tacti.status = 'y'\n" +
" and tafjc.status = 'y'\n" +
" and tajhc.status = 'y'\n" +
" and tayjc_sync.status = 'y'\n" +
" and tayjc_compaction.status = 'y'\n" +
"order by dstf.field_seq;";
" dst.schema_name,\n" +
" dst.table_name,\n" +
" dst.table_type,\n" +
" dstf.field_name,\n" +
" dstf.field_seq,\n" +
" dstf.field_type,\n" +
" dstf.primary_key,\n" +
" dstf.partition_key,\n" +
" dstf.length,\n" +
" tacti.tgt_db,\n" +
" tacti.tgt_table,\n" +
" tacti.tgt_table_type,\n" +
" tacti.tgt_hdfs_path,\n" +
" tajhc.write_tasks,\n" +
" tajhc.write_operation,\n" +
" tajhc.write_task_max_memory,\n" +
" tajhc.write_batch_size,\n" +
" tajhc.write_rate_limit,\n" +
" tacti.bucket_number,\n" +
" tajhc.compaction_strategy,\n" +
" tajhc.compaction_tasks,\n" +
" tajhc.compaction_delta_commits,\n" +
" tajhc.compaction_delta_seconds,\n" +
" tajhc.compaction_async_enabled,\n" +
" tajhc.compaction_max_memory,\n" +
" tajhc.configs,\n" +
" tacti.filter_field,\n" +
" tacti.filter_values,\n" +
" tacti.filter_type,\n" +
" tacti.src_topic,\n" +
" tacti.src_pulsar_addr,\n" +
" tayjc_sync.job_manager_memory as sync_job_manager_memory,\n" +
" tayjc_sync.task_manager_memory as sync_task_manager_memory,\n" +
" tayjc_compaction.job_manager_memory as compaction_job_manager_memory,\n" +
" tayjc_compaction.task_manager_memory as compaction_task_manger_momory,\n" +
" tacti.partition_field,\n" +
" tahss.message_id,\n" +
" tagc.metric_publish_url,\n" +
" tagc.metric_prometheus_url,\n" +
" tagc.metric_api_url,\n" +
" tagc.metric_publish_delay,\n" +
" tagc.metric_publish_period,\n" +
" tagc.metric_publish_timeout,\n" +
" tagc.metric_publish_batch,\n" +
" tafjc.id as job_id,\n" +
" tafjc.name as job_name,\n" +
" tagc.checkpoint_root_path,\n" +
" tajhc.source_tasks,\n" +
" tacti.alias,\n" +
" dst.connection,\n" +
" tacti.priority,\n" +
" dst.ds_type,\n" +
" tajhc.keep_file_version,\n" +
" tajhc.keep_commit_version,\n" +
" tacti.tags,\n" +
" tagc.zk_url,\n" +
" tacti.version,\n" +
" dstf.scale\n" +
"from `" + database + "`.tb_app_collect_table_info tacti\n" +
" left join `" + database + "`.tb_app_hudi_sync_state tahss\n" +
" on tahss.id = concat(tacti.flink_job_id, '-', tacti.alias),\n" +
" `" + database + "`.tb_app_flink_job_config tafjc,\n" +
" `" + database + "`.tb_app_hudi_job_config tajhc,\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc_sync,\n" +
" `" + database + "`.tb_app_yarn_job_config tayjc_compaction,\n" +
" `" + database + "`.tb_app_global_config tagc,\n" +
" `" + database + "`.tb_app_hudi_compaction_schedule tahcs,\n" +
" `iap-datahub`.data_source_table_field dstf,\n" +
" (select ds.*, dst.table_id, dst.table_name, dst.table_type\n" +
" from `iap-datahub`.data_source_table dst,\n" +
" (select ds.ds_id, ds.ds_name, ds.ds_type, ds.schema_name, ds.connection\n" +
" from `iap-datahub`.data_source ds\n" +
" where ds.ds_role = 'src'\n" +
" and ds.ds_state = 'y'\n" +
" and ds.record_state = 'y') ds\n" +
" where dst.ds_id = ds.ds_id\n" +
" and dst.record_state = 'y') dst\n" +
"where dstf.table_id = dst.table_id\n" +
" and dstf.record_state = 'y'\n" +
" and dst.ds_type in ('udal', 'telepg')\n" +
" and dst.ds_name = tacti.src_db\n" +
" and dst.schema_name = tacti.src_schema\n" +
" and dst.table_name = tacti.src_table\n" +
" and tacti.flink_job_id = tafjc.id\n" +
" and tacti.hudi_job_id = tajhc.id\n" +
" and tacti.sync_yarn_job_id = tayjc_sync.id\n" +
" and tacti.compaction_yarn_job_id = tayjc_compaction.id\n" +
" and tacti.config_id = tagc.id\n" +
" and tacti.schedule_id = tahcs.id\n" +
(filterByFlinkJobId ? " and tafjc.id = ?\n" : "") +
(filterByAlias ? " and tacti.alias = ?\n" : "") +
" and tacti.status = 'y'\n" +
" and tafjc.status = 'y'\n" +
" and tajhc.status = 'y'\n" +
" and tayjc_sync.status = 'y'\n" +
" and tayjc_compaction.status = 'y'\n" +
"order by dstf.field_seq;";
}
public static List<TableMeta> from(ResultSet rs) throws SQLException {
List<TableMeta> results = new ArrayList<>();
List<TableMeta.RowMeta> metaList = new ArrayList<>();
while (rs.next( )) {
while (rs.next()) {
metaList.add(
TableMeta.RowMeta.builder()
.dsName(rs.getString(1))
@@ -571,6 +577,14 @@ public class TableMetaHelper {
}
public static boolean existsTag(TableMeta meta, String tag) {
return meta.getTags() != null && meta.getTags().contains(tag);
return existsTag(meta.getTags(), tag);
}
public static boolean existsTag(String sourceTags, String tag) {
return existsTag(Arrays.asList(sourceTags.split(",")), tag);
}
public static boolean existsTag(List<String> sourceTags, String tag) {
return sourceTags != null && sourceTags.contains(tag);
}
}