feature(info-query): 增加hudi表类型过滤

This commit is contained in:
2023-07-11 17:43:43 +08:00
parent 2ac1e0be2c
commit 5abfe2d017
7 changed files with 201 additions and 88 deletions

View File

@@ -7,10 +7,7 @@ import com.eshore.odcp.hudi.connector.entity.FlinkJob;
import com.eshore.odcp.hudi.connector.entity.SyncState; import com.eshore.odcp.hudi.connector.entity.SyncState;
import com.eshore.odcp.hudi.connector.entity.TableMeta; import com.eshore.odcp.hudi.connector.entity.TableMeta;
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse; import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
import com.lanyuanxiaoyao.service.configuration.entity.info.CompactionMetrics; import com.lanyuanxiaoyao.service.configuration.entity.info.*;
import com.lanyuanxiaoyao.service.configuration.entity.info.JobAndMetas;
import com.lanyuanxiaoyao.service.configuration.entity.info.JobIdAndAlias;
import com.lanyuanxiaoyao.service.configuration.entity.info.VersionUpdated;
import java.util.Map; import java.util.Map;
import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.list.ImmutableList;
@@ -20,98 +17,101 @@ import org.eclipse.collections.api.list.ImmutableList;
* @author lanyuanxiaoyao * @author lanyuanxiaoyao
* @date 2023-04-24 * @date 2023-04-24
*/ */
@BaseRequest(baseURL = "http://service-info-query/info") @BaseRequest(baseURL = "http://service-info-query")
public interface InfoService { public interface InfoService {
@Get("/table_count") @Get("/logs")
ImmutableList<SQLLine> sqlLogs();
@Get("/info/table_count")
Long tableCount(); Long tableCount();
@Get("/table_focus_count") @Get("/info/table_focus_count")
Long tableFocusCount(); Long tableFocusCount();
@Get("/hudi_count") @Get("/info/hudi_count")
Long hudiCount(); Long hudiCount();
@Get("/hudi_focus_count") @Get("/info/hudi_focus_count")
Long hudiFocusCount(); Long hudiFocusCount();
@Get("/hive_count") @Get("/info/hive_count")
Long hiveCount(); Long hiveCount();
@Get("/hive_focus_count") @Get("/info/hive_focus_count")
Long hiveFocusCount(); Long hiveFocusCount();
@Get("/un_receive_version_normal_table") @Get("/info/un_receive_version_normal_table")
ImmutableList<JobIdAndAlias> unReceiveVersionNormalTable(@Query("version") String version); ImmutableList<JobIdAndAlias> unReceiveVersionNormalTable(@Query("version") String version);
@Get("/un_receive_version_normal_table_count") @Get("/info/un_receive_version_normal_table_count")
Long unReceiveVersionNormalTableCount(@Query("version") String version); Long unReceiveVersionNormalTableCount(@Query("version") String version);
@Get("/un_receive_version_focus_table") @Get("/info/un_receive_version_focus_table")
ImmutableList<JobIdAndAlias> unReceiveVersionFocusTable(@Query("version") String version); ImmutableList<JobIdAndAlias> unReceiveVersionFocusTable(@Query("version") String version);
@Get("/un_receive_version_focus_table_count") @Get("/info/un_receive_version_focus_table_count")
Long unReceiveVersionFocusTableCount(@Query("version") String version); Long unReceiveVersionFocusTableCount(@Query("version") String version);
@Get("/un_scheduled_normal_table") @Get("/info/un_scheduled_normal_table")
ImmutableList<JobIdAndAlias> unScheduledNormalTable(@Query("version") String version); ImmutableList<JobIdAndAlias> unScheduledNormalTable(@Query("version") String version);
@Get("/un_scheduled_normal_table_count") @Get("/info/un_scheduled_normal_table_count")
Long unScheduledNormalTableCount(@Query("version") String version); Long unScheduledNormalTableCount(@Query("version") String version);
@Get("/un_scheduled_focus_table") @Get("/info/un_scheduled_focus_table")
ImmutableList<JobIdAndAlias> unScheduledFocusTable(@Query("version") String version); ImmutableList<JobIdAndAlias> unScheduledFocusTable(@Query("version") String version);
@Get("/un_scheduled_focus_table_count") @Get("/info/un_scheduled_focus_table_count")
Long unScheduledFocusTableCount(@Query("version") String version); Long unScheduledFocusTableCount(@Query("version") String version);
@Get("/job_id_alias") @Get("/info/job_id_alias")
PageResponse<JobIdAndAlias> jobIdAndAlias(@Query Map<String, Object> queryMap); PageResponse<JobIdAndAlias> jobIdAndAlias(@Query Map<String, Object> queryMap);
@Get("/version_tables") @Get("/info/version_tables")
PageResponse<VersionUpdated> versionTables(@Query Map<String, Object> queryMap); PageResponse<VersionUpdated> versionTables(@Query Map<String, Object> queryMap);
@Get("/compaction_metrics") @Get("/info/compaction_metrics")
PageResponse<CompactionMetrics> compactionMetrics(@Query Map<String, Object> queryMap); PageResponse<CompactionMetrics> compactionMetrics(@Query Map<String, Object> queryMap);
@Get("/job_metas") @Get("/info/job_metas")
ImmutableList<JobAndMetas> jobAndMetas(); ImmutableList<JobAndMetas> jobAndMetas();
@Get("/flink_job/list") @Get("/info/flink_job/list")
ImmutableList<FlinkJob> flinkJobList(); ImmutableList<FlinkJob> flinkJobList();
@Get("/flink_job/detail") @Get("/info/flink_job/detail")
FlinkJob flinkJobDetail(@Query("flink_job_id") Long flinkJobId); FlinkJob flinkJobDetail(@Query("flink_job_id") Long flinkJobId);
@Get("/table_meta/list") @Get("/info/table_meta/list")
ImmutableList<TableMeta> tableMetaList(); ImmutableList<TableMeta> tableMetaList();
@Get("/table_meta/list") @Get("/info/table_meta/list")
ImmutableList<TableMeta> tableMetaList(@Query("flink_job_id") Long flinkJobId); ImmutableList<TableMeta> tableMetaList(@Query("flink_job_id") Long flinkJobId);
@Get("/table_meta/detail") @Get("/info/table_meta/detail")
TableMeta tableMetaDetail(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias); TableMeta tableMetaDetail(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias);
@Get("/sync_state/detail") @Get("/info/sync_state/detail")
SyncState syncStateDetail(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias); SyncState syncStateDetail(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias);
@Get("/non_updated_version_tables") @Get("/info/non_updated_version_tables")
ImmutableList<String> nonUpdatedVersionTables(); ImmutableList<String> nonUpdatedVersionTables();
@Get("/updated_version_tables") @Get("/info/updated_version_tables")
ImmutableList<String> updatedVersionTables(); ImmutableList<String> updatedVersionTables();
@Get("/exists_table") @Get("/info/exists_table")
Boolean existsTable(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias); Boolean existsTable(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias);
@Get("/non_exists_table") @Get("/info/non_exists_table")
Boolean nonExistsTable(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias); Boolean nonExistsTable(@Query("flink_job_id") Long flinkJobId, @Query("alias") String alias);
@Get("/all_flink_job_id") @Get("/info/all_flink_job_id")
ImmutableList<Long> allFlinkJobId(); ImmutableList<Long> allFlinkJobId();
@Get("/all_alias") @Get("/info/all_alias")
ImmutableList<String> allAlias(); ImmutableList<String> allAlias();
@Get("/all_hdfs") @Get("/info/all_hdfs")
ImmutableList<String> allHdfs(); ImmutableList<String> allHdfs();
} }

View File

@@ -45,6 +45,7 @@ public class InfoController {
@RequestParam(value = "alias", required = false) String alias, @RequestParam(value = "alias", required = false) String alias,
@RequestParam(value = "order", required = false) String order, @RequestParam(value = "order", required = false) String order,
@RequestParam(value = "direction", required = false) String direction, @RequestParam(value = "direction", required = false) String direction,
@RequestParam(value = "filter_hudi_table_type", required = false) List<String> selectHudiTableType,
@RequestParam(value = "filter_run_mode", required = false) List<String> selectedRunMode, @RequestParam(value = "filter_run_mode", required = false) List<String> selectedRunMode,
@RequestParam(value = "filter_compaction_status", required = false) List<String> selectedCompactionStatus @RequestParam(value = "filter_compaction_status", required = false) List<String> selectedCompactionStatus
) { ) {
@@ -55,6 +56,7 @@ public class InfoController {
alias, alias,
order, order,
direction, direction,
Lists.immutable.ofAll(selectHudiTableType),
Lists.immutable.ofAll(selectedRunMode), Lists.immutable.ofAll(selectedRunMode),
Lists.immutable.ofAll(selectedCompactionStatus) Lists.immutable.ofAll(selectedCompactionStatus)
); );

View File

@@ -59,6 +59,33 @@ public class InfoService {
return databaseService.getSyncState(flinkJobId, alias); return databaseService.getSyncState(flinkJobId, alias);
} }
private SqlBuilder generateJobIdAndAliasCriteria(
SelectSqlBuilder builder,
Integer page,
Integer count,
Long flinkJobId,
String alias,
String order,
String direction,
ImmutableList<String> selectHudiTableType,
ImmutableList<String> selectedRunMode,
ImmutableList<String> selectedCompactionStatus,
boolean limited
) {
int limit = Math.max(count, 1);
int offset = limit * Math.max(page - 1, 0);
return builder.from(TABLE_FLINK_JOB, TABLE_INFO, TABLE_SYNC_STATE)
.whereEq(TABLE_FLINK_JOB_ID, Column.as(TABLE_INFO_FLINK_JOB_ID))
.andEq(TABLE_SYNC_STATE_ID, Column.as(StrUtil.format("concat({}, '-', {})", TABLE_FLINK_JOB_ID, TABLE_INFO_ALIAS)))
.andLike(ObjectUtil.isNotNull(flinkJobId), TABLE_FLINK_JOB_ID, flinkJobId)
.andLike(ObjectUtil.isNotNull(alias), TABLE_INFO_ALIAS, alias)
.andIn(ObjectUtil.isNotEmpty(selectHudiTableType), TABLE_INFO_TARGET_TABLE_TYPE, selectHudiTableType)
.andIn(ObjectUtil.isNotEmpty(selectedRunMode), TABLE_FLINK_JOB_RUN_MODE, selectedRunMode)
.andIn(ObjectUtil.isNotEmpty(selectedCompactionStatus), TABLE_SYNC_STATE_COMPACTION_STATE, selectedCompactionStatus)
.orderBy(StrUtil.isNotBlank(order) && StrUtil.isNotBlank(direction), () -> StrUtil.format("{} {}", order, direction))
.limit(limited, offset, limit);
}
@Cacheable(value = "job-id-alias", sync = true) @Cacheable(value = "job-id-alias", sync = true)
@Retryable(Throwable.class) @Retryable(Throwable.class)
public PageResponse<JobIdAndAlias> findAllJobIdAndAlias( public PageResponse<JobIdAndAlias> findAllJobIdAndAlias(
@@ -68,50 +95,40 @@ public class InfoService {
String alias, String alias,
String order, String order,
String direction, String direction,
ImmutableList<String> selectHudiTableType,
ImmutableList<String> selectedRunMode, ImmutableList<String> selectedRunMode,
ImmutableList<String> selectedCompactionStatus ImmutableList<String> selectedCompactionStatus
) { ) {
return mysqlTransactionTemplate.execute(status -> { return mysqlTransactionTemplate.execute(status -> {
int limit = Math.max(count, 1); String orderField = null;
int offset = limit * Math.max(page - 1, 0); String orderDirection = null;
String searchFlinkJobId = flinkJobId == null ? "" : "and job.id like '%" + flinkJobId + "%'\n";
String searchAlias = alias == null ? "" : "and info.alias like '%" + alias + "%'\n";
String filterRunMode = selectedRunMode == null || selectedRunMode.isEmpty()
? ""
: "and job.run_mode in ('" + selectedRunMode.makeString("','") + "')\n";
String filterCompactionStatus = selectedCompactionStatus == null || selectedCompactionStatus.isEmpty()
? ""
: "and state.compaction_status in ('" + selectedCompactionStatus.makeString("','") + "')\n";
String orderBy = "";
if (order != null && direction != null) { if (order != null && direction != null) {
String orderField = null;
switch (order) { switch (order) {
case "source_start_time": case "source_start_time":
case "sourceStartTime": case "sourceStartTime":
orderField = "state.source_start_time"; orderField = "tahss.source_start_time";
break; break;
case "source_checkpoint_time": case "source_checkpoint_time":
case "sourceCheckpointTime": case "sourceCheckpointTime":
orderField = "state.source_checkpoint_time"; orderField = "tahss.source_checkpoint_time";
break; break;
case "source_publish_time": case "source_publish_time":
case "sourcePublishTime": case "sourcePublishTime":
orderField = "state.source_publish_time"; orderField = "tahss.source_publish_time";
break; break;
case "source_operation_time": case "source_operation_time":
case "sourceOperationTime": case "sourceOperationTime":
orderField = "state.source_op_time"; orderField = "tahss.source_op_time";
break; break;
case "compaction_start_time": case "compaction_start_time":
case "compactionStartTime": case "compactionStartTime":
orderField = "state.compaction_start_time"; orderField = "tahss.compaction_start_time";
break; break;
case "compaction_finish_time": case "compaction_finish_time":
case "compactionFinishTime": case "compactionFinishTime":
orderField = "state.compaction_finish_time"; orderField = "tahss.compaction_finish_time";
break; break;
} }
String orderDirection = null;
switch (direction) { switch (direction) {
case "desc": case "desc":
case "DESC": case "DESC":
@@ -126,37 +143,37 @@ public class InfoService {
orderDirection = "asc"; orderDirection = "asc";
break; break;
} }
if (ObjectUtil.isNotNull(orderField) && ObjectUtil.isNotNull(direction)) {
orderBy = StrUtil.format("order by {} {}\n", orderField, orderDirection);
}
} }
Long total = mysqlJdbcTemplate.queryForObject( Long total = mysqlJdbcTemplate.queryForObject(
"select count(*)\n" + generateJobIdAndAliasCriteria(
"from " + DATABASE_NAME + ".tb_app_flink_job_config job,\n" + SqlBuilder.select(COUNT),
" " + DATABASE_NAME + ".tb_app_collect_table_info info,\n" + page,
" " + DATABASE_NAME + ".tb_app_hudi_sync_state state\n" + count,
"where job.id = info.flink_job_id\n" + flinkJobId,
"and state.id = concat(job.id, '-', info.alias)\n" + alias,
searchFlinkJobId + orderField,
searchAlias + orderDirection,
filterRunMode + selectHudiTableType,
filterCompactionStatus + selectedRunMode,
orderBy, selectedCompactionStatus,
false
).build(),
Long.class Long.class
); );
List<JobIdAndAlias> list = mysqlJdbcTemplate.query( List<JobIdAndAlias> list = mysqlJdbcTemplate.query(
"select job.id, info.alias\n" + generateJobIdAndAliasCriteria(
"from " + DATABASE_NAME + ".tb_app_flink_job_config job,\n" + SqlBuilder.select(TABLE_FLINK_JOB_ID, TABLE_INFO_ALIAS),
" " + DATABASE_NAME + ".tb_app_collect_table_info info,\n" + page,
" " + DATABASE_NAME + ".tb_app_hudi_sync_state state\n" + count,
"where job.id = info.flink_job_id\n" + flinkJobId,
"and state.id = concat(job.id, '-', info.alias)\n" + alias,
searchFlinkJobId + orderField,
searchAlias + orderDirection,
filterRunMode + selectHudiTableType,
filterCompactionStatus + selectedRunMode,
orderBy + selectedCompactionStatus,
"limit " + limit + " offset " + offset, true
).build(),
(rs, row) -> new JobIdAndAlias(rs.getLong(1), rs.getString(2)) (rs, row) -> new JobIdAndAlias(rs.getLong(1), rs.getString(2))
); );
return new PageResponse<>(list, total); return new PageResponse<>(list, total);
@@ -243,9 +260,11 @@ public class InfoService {
private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority"); private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority");
private static final String TABLE_INFO_STATUS = column(TABLE_INFO, "status"); private static final String TABLE_INFO_STATUS = column(TABLE_INFO, "status");
private static final String TABLE_INFO_TARGET_HDFS = column(TABLE_INFO, "tgt_hdfs_path"); private static final String TABLE_INFO_TARGET_HDFS = column(TABLE_INFO, "tgt_hdfs_path");
private static final String TABLE_INFO_TARGET_TABLE_TYPE = column(TABLE_INFO, "tgt_table_type");
private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss"); private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss");
private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id"); private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id");
private static final String TABLE_SYNC_STATE_COMPACTION_STATE = column(TABLE_SYNC_STATE, "compaction_status");
private SqlBuilder generateVersionTableCriteria( private SqlBuilder generateVersionTableCriteria(
SelectSqlBuilder builder, SelectSqlBuilder builder,
@@ -730,6 +749,7 @@ public class InfoService {
private static final Alias TABLE_FLINK_JOB = Alias.of(StrUtil.format("{}.tb_app_flink_job_config", DATABASE_NAME), "tafjc"); private static final Alias TABLE_FLINK_JOB = Alias.of(StrUtil.format("{}.tb_app_flink_job_config", DATABASE_NAME), "tafjc");
private static final String TABLE_FLINK_JOB_ID = column(TABLE_FLINK_JOB, "id"); private static final String TABLE_FLINK_JOB_ID = column(TABLE_FLINK_JOB, "id");
private static final String TABLE_FLINK_JOB_STATUS = column(TABLE_FLINK_JOB, "status"); private static final String TABLE_FLINK_JOB_STATUS = column(TABLE_FLINK_JOB, "status");
private static final String TABLE_FLINK_JOB_RUN_MODE = column(TABLE_FLINK_JOB, "run_mode");
@Cacheable(value = "exists-table", sync = true) @Cacheable(value = "exists-table", sync = true)
@Retryable(Throwable.class) @Retryable(Throwable.class)

View File

@@ -1,8 +1,12 @@
import club.kingon.sql.builder.SelectSqlBuilder;
import club.kingon.sql.builder.SqlBuilder; import club.kingon.sql.builder.SqlBuilder;
import club.kingon.sql.builder.entry.Alias; import club.kingon.sql.builder.entry.Alias;
import club.kingon.sql.builder.entry.Column; import club.kingon.sql.builder.entry.Column;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil; import cn.hutool.core.util.StrUtil;
import cn.hutool.db.sql.SqlFormatter; import cn.hutool.db.sql.SqlFormatter;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME; import static com.eshore.odcp.hudi.connector.Constants.DATABASE_NAME;
@@ -24,9 +28,11 @@ public class SqlBuilderTests {
private static final String TABLE_INFO_ALIAS = column(TABLE_INFO, "alias"); private static final String TABLE_INFO_ALIAS = column(TABLE_INFO, "alias");
private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority"); private static final String TABLE_INFO_PRIORITY = column(TABLE_INFO, "priority");
private static final String TABLE_INFO_STATUS = column(TABLE_INFO, "status"); private static final String TABLE_INFO_STATUS = column(TABLE_INFO, "status");
private static final String TABLE_INFO_TARGET_TABLE_TYPE = column(TABLE_INFO, "tgt_table_type");
private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss"); private static final Alias TABLE_SYNC_STATE = Alias.of(StrUtil.format("{}.tb_app_hudi_sync_state", DATABASE_NAME), "tahss");
private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id"); private static final String TABLE_SYNC_STATE_ID = column(TABLE_SYNC_STATE, "id");
private static final String TABLE_SYNC_STATE_COMPACTION_STATE = column(TABLE_SYNC_STATE, "compaction_status");
private static String column(Alias table, String column) { private static String column(Alias table, String column) {
return StrUtil.format("{}.{}", table.getAlias(), column); return StrUtil.format("{}.{}", table.getAlias(), column);
@@ -40,15 +46,62 @@ public class SqlBuilderTests {
private static final Alias TABLE_FLINK_JOB = Alias.of(StrUtil.format("{}.tb_app_flink_job_config", DATABASE_NAME), "tafjc"); private static final Alias TABLE_FLINK_JOB = Alias.of(StrUtil.format("{}.tb_app_flink_job_config", DATABASE_NAME), "tafjc");
private static final String TABLE_FLINK_JOB_ID = column(TABLE_FLINK_JOB, "id"); private static final String TABLE_FLINK_JOB_ID = column(TABLE_FLINK_JOB, "id");
private static final String TABLE_FLINK_JOB_STATUS = column(TABLE_FLINK_JOB, "status"); private static final String TABLE_FLINK_JOB_STATUS = column(TABLE_FLINK_JOB, "status");
private static final String TABLE_FLINK_JOB_RUN_MODE = column(TABLE_FLINK_JOB, "run_mode");
private static SqlBuilder generateJobIdAndAliasCriteria(
SelectSqlBuilder builder,
Integer page,
Integer count,
Long flinkJobId,
String alias,
String order,
String direction,
ImmutableList<String> selectHudiTableType,
ImmutableList<String> selectedRunMode,
ImmutableList<String> selectedCompactionStatus,
boolean limited
) {
int limit = Math.max(count, 1);
int offset = limit * Math.max(page - 1, 0);
return builder.from(TABLE_FLINK_JOB, TABLE_INFO, TABLE_SYNC_STATE)
.whereEq(TABLE_FLINK_JOB_ID, Column.as(TABLE_INFO_FLINK_JOB_ID))
.andEq(TABLE_SYNC_STATE_ID, Column.as(StrUtil.format("concat({}, '-', {})", TABLE_FLINK_JOB_ID, TABLE_INFO_ALIAS)))
.andLike(ObjectUtil.isNotNull(flinkJobId), TABLE_FLINK_JOB_ID, flinkJobId)
.andLike(ObjectUtil.isNotNull(alias), TABLE_INFO_ALIAS, alias)
.andIn(ObjectUtil.isNotEmpty(selectHudiTableType), TABLE_INFO_TARGET_TABLE_TYPE, selectHudiTableType)
.andIn(ObjectUtil.isNotEmpty(selectedRunMode), TABLE_FLINK_JOB_RUN_MODE, selectedRunMode)
.andIn(ObjectUtil.isNotEmpty(selectedCompactionStatus), TABLE_SYNC_STATE_COMPACTION_STATE, selectedCompactionStatus)
.orderBy(StrUtil.isNotBlank(order) && StrUtil.isNotBlank(direction), () -> StrUtil.format("{} {}", order, direction))
.limit(limited, offset, limit);
}
public static void main(String[] args) { public static void main(String[] args) {
System.out.println(SqlFormatter.format( System.out.println(SqlFormatter.format(
SqlBuilder.select(StrUtil.format("distinct {}", TABLE_INFO_ALIAS)) /*SqlBuilder.select(StrUtil.format("distinct {}", TABLE_INFO_ALIAS))
.from(TABLE_INFO, TABLE_FLINK_JOB) .from(TABLE_FLINK_JOB, TABLE_INFO, TABLE_SYNC_STATE)
.whereEq(TABLE_INFO_FLINK_JOB_ID, Column.as(TABLE_FLINK_JOB_ID)) .whereEq(TABLE_FLINK_JOB_ID, Column.as(TABLE_INFO_FLINK_JOB_ID))
.andEq(TABLE_FLINK_JOB_STATUS, "y") .andEq(TABLE_SYNC_STATE_ID, StrUtil.format("concat({}, '-', {})", TABLE_FLINK_JOB_ID, TABLE_INFO_ALIAS))
.andEq(TABLE_INFO_STATUS, "y") .andLike(ObjectUtil.isNotNull(flinkJobId), TABLE_FLINK_JOB_ID, flinkJobId)
.build() .andLike(ObjectUtil.isNotNull(alias), TABLE_INFO_ALIAS, alias)
.andIn(ObjectUtil.isNotEmpty(selectHudiTableType), TABLE_INFO_TARGET_TABLE_TYPE, selectHudiTableType)
.andIn(ObjectUtil.isNotEmpty(selectedRunMode), TABLE_FLINK_JOB_RUN_MODE, selectedRunMode)
.andIn(ObjectUtil.isNotEmpty(selectedCompactionStatus), TABLE_SYNC_STATE_COMPACTION_STATE, selectedCompactionStatus)
.orderBy(StrUtil.isNotBlank(order) && StrUtil.isNotBlank(direction), () -> StrUtil.format("{} {}", order, direction))
.limit(limited, offset, limit)
.build()*/
generateJobIdAndAliasCriteria(
SqlBuilder.selectAll(),
1,
10,
1000L,
"hello",
"dog",
"asc",
Lists.immutable.of("MOR"),
Lists.immutable.of("ONE_IN_ONE", "ALL_IN_ONE"),
Lists.immutable.of("COMPLETE"),
true
).build()
)); ));
} }
} }

View File

@@ -27,6 +27,7 @@ import com.lanyuanxiaoyao.service.web.entity.TableVO;
import java.time.Duration; import java.time.Duration;
import java.util.List; import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Optional;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutionException;
import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Lists;
@@ -83,10 +84,14 @@ public class TableController extends BaseController {
@RequestParam(value = "direction", required = false) String direction, @RequestParam(value = "direction", required = false) String direction,
@RequestParam(value = "search_flink_job_id", required = false) String searchFlinkJobId, @RequestParam(value = "search_flink_job_id", required = false) String searchFlinkJobId,
@RequestParam(value = "search_alias", required = false) String searchAlias, @RequestParam(value = "search_alias", required = false) String searchAlias,
@RequestParam(value = "filter_hudi_table_type", required = false) List<String> hudiTableType,
@RequestParam(value = "filter_run_mode", required = false) List<String> runMode, @RequestParam(value = "filter_run_mode", required = false) List<String> runMode,
@RequestParam(value = "filter_compaction_status", required = false) List<String> compactionStatus @RequestParam(value = "filter_compaction_status", required = false) List<String> compactionStatus
) { ) {
MutableMap<String, Object> queryMap = buildQueryMap(page, count, order, direction, searchFlinkJobId, searchAlias); MutableMap<String, Object> queryMap = buildQueryMap(page, count, order, direction, searchFlinkJobId, searchAlias);
if (ObjectUtil.isNotEmpty(hudiTableType)) {
queryMap.put("filter_hudi_table_type", hudiTableType);
}
if (ObjectUtil.isNotEmpty(runMode)) { if (ObjectUtil.isNotEmpty(runMode)) {
queryMap.put("filter_run_mode", runMode); queryMap.put("filter_run_mode", runMode);
} }
@@ -144,7 +149,7 @@ public class TableController extends BaseController {
return new TableVO( return new TableVO(
flinkJobFuture.get(), flinkJobFuture.get(),
tableMetaFuture.get(), tableMetaFuture.get(),
new SyncStateVO(syncStateFuture.get()), Optional.of(syncStateFuture.get()).map(SyncStateVO::new).orElse(null),
ObjectUtil.isNotNull(syncRunMeta), ObjectUtil.isNotNull(syncRunMeta),
syncRunMeta, syncRunMeta,
ObjectUtil.isNotNull(compactionRunMeta), ObjectUtil.isNotNull(compactionRunMeta),

View File

@@ -19,6 +19,7 @@ function tableTab() {
direction: '${orderDir|default:undefined}', direction: '${orderDir|default:undefined}',
search_flink_job_id: '${flinkJobId|default:undefined}', search_flink_job_id: '${flinkJobId|default:undefined}',
search_alias: '${alias|default:undefined}', search_alias: '${alias|default:undefined}',
filter_hudi_table_type: '${tableMeta\\.hudi\\.targetTableType|default:undefined}',
filter_run_mode: '${flinkJob\\.runMode|default:undefined}', filter_run_mode: '${flinkJob\\.runMode|default:undefined}',
filter_compaction_status: '${syncState\\.compactionStatus|default:undefined}' filter_compaction_status: '${syncState\\.compactionStatus|default:undefined}'
} }

View File

@@ -153,6 +153,38 @@ function toolTab() {
} }
] ]
}, },
{
type: 'panel',
title: 'SQL日志',
body: [
{
type: 'action',
label: '查看',
actionType: 'dialog',
dialog: {
title: '日志',
...readOnlyDialogOptions(),
size: 'lg',
body: {
type: 'crud',
api: '${base}/log/query_sql_log',
...crudCommonOptions(),
loadDataOnce: true,
perPage: 50,
headerToolbar: [
"reload",
paginationCommonOptions(undefined, 10),
],
footerToolbar: [
paginationCommonOptions(undefined, 10),
],
columns: [
],
}
}
}
]
}
] ]
} }
} }