package com.lanyuanxiaoyao.service.info.service;

import club.kingon.sql.builder.ConditionSqlBuilder;
import club.kingon.sql.builder.SelectSqlBuilder;
import club.kingon.sql.builder.SqlBuilder;
import club.kingon.sql.builder.WhereSqlBuilder;
import club.kingon.sql.builder.entry.Alias;
import club.kingon.sql.builder.entry.Column;
import cn.hutool.core.collection.IterUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import com.lanyuanxiaoyao.service.common.exception.ConfigException;
import com.lanyuanxiaoyao.service.common.exception.TableMetaNotFoundException;
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
import com.lanyuanxiaoyao.service.configuration.entity.info.SimpleTableMeta;
import com.lanyuanxiaoyao.service.configuration.entity.info.TableInfoSearchCache;
import com.lanyuanxiaoyao.service.configuration.entity.info.TableMetaAdd;
import java.util.List;
import java.util.Locale;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.CacheConfig;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.retry.annotation.Retryable;
import org.springframework.stereotype.Service;

import static com.lanyuanxiaoyao.service.common.SQLConstants.HudiCollectBuild.*;
import static com.lanyuanxiaoyao.service.common.SQLConstants.IapDatahub.*;

/**
 * Table Meta
 *
 * @author lanyuanxiaoyao
 * @date 2024-01-03
 */
@SuppressWarnings("SqlSourceToSinkFlow")
@CacheConfig(cacheManager = "normal-cache")
@Service
public class TableMetaService extends BaseService {
    private static final Logger logger = LoggerFactory.getLogger(TableMetaService.class);
    private final JdbcTemplate mysqlJdbcTemplate;
    private final FlinkJobService flinkJobService;

    public TableMetaService(JdbcTemplate mysqlJdbcTemplate, FlinkJobService flinkJobService) {
        this.mysqlJdbcTemplate = mysqlJdbcTemplate;
        this.flinkJobService = flinkJobService;
    }

    private static void checkMoreThanOne(String fieldName, Iterable<?> iterable) throws ConfigException {
        ConfigException.check(fieldName + " cannot be more than 1", () -> IterUtil.size(iterable) > 1);
    }

    private static void checkEmpty(String fieldName, Iterable<?> iterable) throws ConfigException {
        ConfigException.check(fieldName + " cannot be empty", () -> IterUtil.isEmpty(iterable));
    }

    private static void checkEmptyOrMoreThanOne(String fieldName, Iterable<?> iterable) throws ConfigException {
        checkEmpty(fieldName, iterable);
        checkMoreThanOne(fieldName, iterable);
    }

    public ImmutableList<TableMeta> tableMetaList(Long flinkJobId) {
        return tableMetaList(flinkJobId, null);
    }

    public static ConditionSqlBuilder<WhereSqlBuilder> generateSimpleTableMetaList(SelectSqlBuilder builder) {
        return generateSimpleTableMetaList(builder, null, null, null, null);
    }

    public static ConditionSqlBuilder<WhereSqlBuilder> generateSimpleTableMetaList(SelectSqlBuilder builder, Integer priority) {
        return generateSimpleTableMetaList(builder, null, null, priority, null);
    }

    public static ConditionSqlBuilder<WhereSqlBuilder> generateSimpleTableMetaList(SelectSqlBuilder builder, Long flinkJobId, String aliasText) {
        return generateSimpleTableMetaList(builder, flinkJobId, aliasText, null, null);
    }

    public static ConditionSqlBuilder<WhereSqlBuilder> generateSimpleTableMetaList(SelectSqlBuilder builder, String hdfs) {
        return generateSimpleTableMetaList(builder, null, null, null, hdfs);
    }

    // 相比正式的table meta查询少了表字段的匹配，大幅提高了查询速度
    public static ConditionSqlBuilder<WhereSqlBuilder> generateSimpleTableMetaList(SelectSqlBuilder builder, Long flinkJobId, String aliasText, Integer priority, String hdfs) {
        return builder
                .from(
                        DataSource._alias_,
                        DataSourceTable._alias_,
                        TbAppFlinkJobConfig._alias_,
                        TbAppHudiJobConfig._alias_,
                        Alias.of(TbAppYarnJobConfig._origin_, "tayjc_sync"),
                        Alias.of(TbAppYarnJobConfig._origin_, "tayjc_compaction"),
                        TbAppGlobalConfig._alias_,
                        TbAppCollectTableInfo._alias_,
                        TbAppHudiSyncState._alias_
                )
                .whereEq(DataSource.DS_ROLE_A, "src")
                .andEq(DataSource.DS_STATE_A, STATUS_Y)
                .andEq(DataSource.RECORD_STATE_A, STATUS_Y)
                .andEq(DataSourceTable.DS_ID_A, Column.as(DataSource.DS_ID_A))
                .andEq(DataSourceTable.RECORD_STATE_A, STATUS_Y)
                .andIn(DataSource.DS_TYPE_A, "udal", "telepg")
                .andEq(DataSource.DS_NAME_A, Column.as(TbAppCollectTableInfo.SRC_DB_A))
                .andEq(DataSource.SCHEMA_NAME_A, Column.as(TbAppCollectTableInfo.SRC_SCHEMA_A))
                .andEq(DataSourceTable.TABLE_NAME_A, Column.as(TbAppCollectTableInfo.SRC_TABLE_A))
                .andEq(TbAppCollectTableInfo.FLINK_JOB_ID_A, Column.as(TbAppFlinkJobConfig.ID_A))
                .andEq(TbAppCollectTableInfo.HUDI_JOB_ID_A, Column.as(TbAppHudiJobConfig.ID_A))
                .andEq(TbAppCollectTableInfo.SYNC_YARN_JOB_ID_A, Column.as("tayjc_sync.id"))
                .andEq(TbAppCollectTableInfo.COMPACTION_YARN_JOB_ID_A, Column.as("tayjc_compaction.id"))
                .andEq(TbAppCollectTableInfo.CONFIG_ID_A, Column.as(TbAppGlobalConfig.ID_A))
                .andEq(TbAppHudiSyncState.ID_A, Column.as(StrUtil.format("concat({}, '-', {})", TbAppFlinkJobConfig.ID_A, TbAppCollectTableInfo.ALIAS_A)))
                .andEq(ObjectUtil.isNotNull(flinkJobId), TbAppFlinkJobConfig.ID_A, flinkJobId)
                .andEq(StrUtil.isNotBlank(aliasText), TbAppCollectTableInfo.ALIAS_A, aliasText)
                .andEq(ObjectUtil.isNotNull(priority), TbAppCollectTableInfo.PRIORITY_A, priority)
                .andEq(StrUtil.isNotBlank(hdfs), TbAppCollectTableInfo.TGT_HDFS_PATH_A, hdfs)
                .andEq(TbAppCollectTableInfo.STATUS_A, STATUS_Y)
                .andEq(TbAppFlinkJobConfig.STATUS_A, STATUS_Y)
                .andEq(TbAppHudiJobConfig.STATUS_A, STATUS_Y)
                .andEq("tayjc_sync.status", STATUS_Y)
                .andEq("tayjc_compaction.status", STATUS_Y);
    }

    private ImmutableList<TableMeta> tableMetaList(Long flinkJobId, String aliasText) {
        return Lists.immutable.ofAll(
                        mysqlJdbcTemplate.query(
                                SqlBuilder.select(
                                                DataSource.DS_NAME_A,
                                                DataSource.SCHEMA_NAME_A,
                                                DataSourceTable.TABLE_NAME_A,
                                                DataSourceTable.TABLE_TYPE_A,
                                                DataSourceTableField.FIELD_NAME_A,
                                                DataSourceTableField.FIELD_SEQ_A,
                                                DataSourceTableField.FIELD_TYPE_A,
                                                DataSourceTableField.PRIMARY_KEY_A,
                                                DataSourceTableField.PARTITION_KEY_A,
                                                DataSourceTableField.LENGTH_A,
                                                TbAppCollectTableInfo.TGT_DB_A,
                                                TbAppCollectTableInfo.TGT_TABLE_A,
                                                TbAppCollectTableInfo.TGT_TABLE_TYPE_A,
                                                TbAppCollectTableInfo.TGT_HDFS_PATH_A,
                                                TbAppHudiJobConfig.WRITE_TASKS_A,
                                                TbAppHudiJobConfig.WRITE_OPERATION_A,
                                                TbAppHudiJobConfig.WRITE_TASK_MAX_MEMORY_A,
                                                TbAppHudiJobConfig.WRITE_BATCH_SIZE_A,
                                                TbAppHudiJobConfig.WRITE_RATE_LIMIT_A,
                                                TbAppCollectTableInfo.BUCKET_NUMBER_A,
                                                TbAppHudiJobConfig.COMPACTION_STRATEGY_A,
                                                TbAppHudiJobConfig.COMPACTION_TASKS_A,
                                                TbAppHudiJobConfig.COMPACTION_DELTA_COMMITS_A,
                                                TbAppHudiJobConfig.COMPACTION_DELTA_SECONDS_A,
                                                TbAppHudiJobConfig.COMPACTION_ASYNC_ENABLED_A,
                                                TbAppHudiJobConfig.COMPACTION_MAX_MEMORY_A,
                                                TbAppHudiJobConfig.CONFIGS_A,
                                                TbAppCollectTableInfo.FILTER_FIELD_A,
                                                TbAppCollectTableInfo.FILTER_VALUES_A,
                                                TbAppCollectTableInfo.FILTER_TYPE_A,
                                                TbAppCollectTableInfo.SRC_TOPIC_A,
                                                TbAppCollectTableInfo.SRC_PULSAR_ADDR_A,
                                                Alias.of("tayjc_sync.job_manager_memory", "sync_job_manager_memory"),
                                                Alias.of("tayjc_sync.task_manager_memory", "sync_task_manager_memory"),
                                                Alias.of("tayjc_compaction.job_manager_memory", "compaction_job_manager_memory"),
                                                Alias.of("tayjc_compaction.task_manager_memory", "compaction_task_manger_momory"),
                                                TbAppCollectTableInfo.PARTITION_FIELD_A,
                                                TbAppHudiSyncState.MESSAGE_ID_A,
                                                TbAppGlobalConfig.METRIC_PUBLISH_URL_A,
                                                TbAppGlobalConfig.METRIC_PROMETHEUS_URL_A,
                                                TbAppGlobalConfig.METRIC_API_URL_A,
                                                TbAppGlobalConfig.METRIC_PUBLISH_DELAY_A,
                                                TbAppGlobalConfig.METRIC_PUBLISH_PERIOD_A,
                                                TbAppGlobalConfig.METRIC_PUBLISH_TIMEOUT_A,
                                                TbAppGlobalConfig.METRIC_PUBLISH_BATCH_A,
                                                Alias.of(TbAppFlinkJobConfig.ID_A, "job_id"),
                                                Alias.of(TbAppFlinkJobConfig.NAME_A, "job_name"),
                                                TbAppGlobalConfig.CHECKPOINT_ROOT_PATH_A,
                                                TbAppHudiJobConfig.SOURCE_TASKS_A,
                                                TbAppCollectTableInfo.ALIAS_A,
                                                DataSource.CONNECTION_A,
                                                TbAppCollectTableInfo.PRIORITY_A,
                                                DataSource.DS_TYPE_A,
                                                TbAppHudiJobConfig.KEEP_FILE_VERSION_A,
                                                TbAppHudiJobConfig.KEEP_COMMIT_VERSION_A,
                                                TbAppCollectTableInfo.TAGS_A,
                                                TbAppGlobalConfig.ZK_URL_A,
                                                TbAppCollectTableInfo.VERSION_A,
                                                DataSourceTableField.SCALE_A
                                        )
                                        .from(
                                                DataSource._alias_,
                                                DataSourceTable._alias_,
                                                DataSourceTableField._alias_,
                                                TbAppFlinkJobConfig._alias_,
                                                TbAppHudiJobConfig._alias_,
                                                Alias.of(TbAppYarnJobConfig._origin_, "tayjc_sync"),
                                                Alias.of(TbAppYarnJobConfig._origin_, "tayjc_compaction"),
                                                TbAppGlobalConfig._alias_,
                                                TbAppCollectTableInfo._alias_,
                                                TbAppHudiSyncState._alias_
                                        )
                                        .whereEq(DataSource.DS_ROLE_A, "src")
                                        .andEq(DataSource.DS_STATE_A, STATUS_Y)
                                        .andEq(DataSource.RECORD_STATE_A, STATUS_Y)
                                        .andEq(DataSourceTable.DS_ID_A, Column.as(DataSource.DS_ID_A))
                                        .andEq(DataSourceTable.RECORD_STATE_A, STATUS_Y)
                                        .andEq(DataSourceTableField.TABLE_ID_A, Column.as(DataSourceTable.TABLE_ID_A))
                                        .andEq(DataSourceTableField.RECORD_STATE_A, STATUS_Y)
                                        .andIn(DataSource.DS_TYPE_A, "udal", "telepg")
                                        .andEq(DataSource.DS_NAME_A, Column.as(TbAppCollectTableInfo.SRC_DB_A))
                                        .andEq(DataSource.SCHEMA_NAME_A, Column.as(TbAppCollectTableInfo.SRC_SCHEMA_A))
                                        .andEq(DataSourceTable.TABLE_NAME_A, Column.as(TbAppCollectTableInfo.SRC_TABLE_A))
                                        .andEq(TbAppCollectTableInfo.FLINK_JOB_ID_A, Column.as(TbAppFlinkJobConfig.ID_A))
                                        .andEq(TbAppCollectTableInfo.HUDI_JOB_ID_A, Column.as(TbAppHudiJobConfig.ID_A))
                                        .andEq(TbAppCollectTableInfo.SYNC_YARN_JOB_ID_A, Column.as("tayjc_sync.id"))
                                        .andEq(TbAppCollectTableInfo.COMPACTION_YARN_JOB_ID_A, Column.as("tayjc_compaction.id"))
                                        .andEq(TbAppCollectTableInfo.CONFIG_ID_A, Column.as(TbAppGlobalConfig.ID_A))
                                        .andEq(TbAppHudiSyncState.ID_A, Column.as(StrUtil.format("concat({}, '-', {})", TbAppFlinkJobConfig.ID_A, TbAppCollectTableInfo.ALIAS_A)))
                                        .andEq(ObjectUtil.isNotNull(flinkJobId), TbAppFlinkJobConfig.ID_A, flinkJobId)
                                        .andEq(StrUtil.isNotBlank(aliasText), TbAppCollectTableInfo.ALIAS_A, aliasText)
                                        .andEq(TbAppCollectTableInfo.STATUS_A, STATUS_Y)
                                        .andEq(TbAppFlinkJobConfig.STATUS_A, STATUS_Y)
                                        .andEq(TbAppHudiJobConfig.STATUS_A, STATUS_Y)
                                        .andEq("tayjc_sync.status", STATUS_Y)
                                        .andEq("tayjc_compaction.status", STATUS_Y)
                                        .orderBy(DataSourceTableField.FIELD_SEQ_A)
                                        .build(),
                                (rs, row) -> TableMeta.RowMeta.builder()
                                        .dsName(rs.getString(1))
                                        .schemaName(rs.getString(2))
                                        .tableName(rs.getString(3))
                                        .tableType(rs.getString(4))
                                        .fieldName(rs.getString(5))
                                        .fieldSeq(rs.getInt(6))
                                        .fieldType(rs.getString(7))
                                        .primaryKey(rs.getString(8))
                                        .partitionKey(rs.getString(9))
                                        .length(rs.getLong(10))
                                        .tgtDb(rs.getString(11))
                                        .tgtTable(rs.getString(12))
                                        .tgtTableType(rs.getString(13))
                                        .tgtHdfsPath(rs.getString(14))
                                        .writeTasks(rs.getInt(15))
                                        .writeOperation(rs.getString(16))
                                        .writeTaskMaxMemory(rs.getInt(17))
                                        .writeBatchSize(rs.getInt(18))
                                        .writeRateLimit(rs.getInt(19))
                                        .bucketIndexNumber(rs.getInt(20))
                                        .compactionStrategy(rs.getString(21))
                                        .compactionTasks(rs.getInt(22))
                                        .compactionDeltaCommits(rs.getInt(23))
                                        .compactionDeltaSeconds(rs.getInt(24))
                                        .compactionAsyncEnabled(rs.getString(25))
                                        .compactionMaxMemory(rs.getInt(26))
                                        .configs(rs.getString(27))
                                        .filterField(rs.getString(28))
                                        .filterValues(rs.getString(29))
                                        .filterType(rs.getString(30))
                                        .topic(rs.getString(31))
                                        .pulsarAddress(rs.getString(32))
                                        .syncJobManagerMemory(rs.getInt(33))
                                        .syncTaskManagerMemory(rs.getInt(34))
                                        .compactionJobManagerMemory(rs.getInt(35))
                                        .compactionTaskManagerMemory(rs.getInt(36))
                                        .partitionField(rs.getString(37))
                                        .messageId(rs.getString(38))
                                        .metricPublishUrl(rs.getString(39))
                                        .metricPrometheusUrl(rs.getString(40))
                                        .metricApiUrl(rs.getString(41))
                                        .metricPublishDelay(rs.getInt(42))
                                        .metricPublishPeriod(rs.getInt(43))
                                        .metricPublishTimeout(rs.getInt(44))
                                        .metricPublishBatch(rs.getInt(45))
                                        .jobId(rs.getLong(46))
                                        .jobName(rs.getString(47))
                                        .checkpointRootPath(rs.getString(48))
                                        .sourceTasks(rs.getInt(49))
                                        .alias(rs.getString(50))
                                        .connection(rs.getString(51))
                                        .priority(rs.getInt(52))
                                        .sourceType(rs.getString(53))
                                        .keepFileVersion(rs.getInt(54))
                                        .keepCommitVersion(rs.getInt(55))
                                        .tags(rs.getString(56))
                                        .zookeeperUrl(rs.getString(57))
                                        .version(rs.getInt(58))
                                        .scala(rs.getInt(59))
                                        .build()
                        )
                )
                .asParallel(ExecutorProvider.EXECUTORS, 5)
                .groupBy(TableMeta.RowMeta::getAlias)
                .multiValuesView()
                .collect(aliasRowMetas -> {
                    try {
                        ImmutableList<TableMeta.RowMeta> rows = aliasRowMetas
                                .toSortedListBy(TableMeta.RowMeta::getFieldSeq)
                                .toImmutable();

                        ImmutableList<String> aliasList = rows.collect(TableMeta.RowMeta::getAlias).distinct();
                        checkEmptyOrMoreThanOne("alias", aliasList);
                        String alias = aliasList.get(0);

                        ImmutableList<String> sourceTypeList = rows.collect(TableMeta.RowMeta::getSourceType).distinct();
                        checkEmptyOrMoreThanOne("source_type", sourceTypeList);
                        String sourceTypeText = sourceTypeList.get(0).toUpperCase();
                        TableMeta.SourceType sourceType;
                        try {
                            sourceType = TableMeta.SourceType.valueOf(sourceTypeText);
                        } catch (IllegalArgumentException e) {
                            throw new Exception("Cannot parse source type " + sourceTypeText);
                        }

                        ImmutableList<String> dsNames = rows.collect(TableMeta.RowMeta::getDsName).distinct();
                        checkEmptyOrMoreThanOne("ds_name", dsNames);
                        String dataSource = dsNames.get(0);

                        ImmutableList<String> schemaNames = rows.collect(TableMeta.RowMeta::getSchemaName).distinct();
                        checkEmptyOrMoreThanOne("schema_name", schemaNames);
                        String schema = schemaNames.get(0);

                        ImmutableList<String> tableNames = rows.collect(TableMeta.RowMeta::getTableName).distinct();
                        // 每次只能获取 1 张表的元信息
                        checkEmptyOrMoreThanOne("table_name", tableNames);
                        String table = tableNames.get(0);

                        ImmutableList<String> tableTypes = rows.collect(TableMeta.RowMeta::getTableType).distinct();
                        checkEmptyOrMoreThanOne("table_type", tableTypes);
                        String type = tableTypes.get(0);

                        ImmutableList<String> filterFields = rows.collect(TableMeta.RowMeta::getFilterField).distinct();
                        checkEmptyOrMoreThanOne("filter_field", filterFields);
                        String filterField = filterFields.get(0);

                        ImmutableList<String> filterValueList = rows.collect(TableMeta.RowMeta::getFilterValues).distinct();
                        checkEmptyOrMoreThanOne("filter_values", filterValueList);
                        String filterValuesText = filterValueList.get(0);
                        ImmutableList<String> filterValues = StrUtil.isBlank(filterValuesText)
                                ? Lists.immutable.empty()
                                : Lists.immutable.of(filterValuesText.split(","));
                        ImmutableList<String> filterTypes = rows.collect(TableMeta.RowMeta::getFilterType).distinct();
                        checkEmptyOrMoreThanOne("filter_field", filterFields);
                        TableMeta.FilterType filterType;
                        try {
                            filterType = TableMeta.FilterType.valueOf(filterTypes.get(0));
                        } catch (IllegalArgumentException e) {
                            filterType = TableMeta.FilterType.NONE;
                        }

                        ImmutableList<String> topics = rows.collect(TableMeta.RowMeta::getTopic).distinct();
                        checkEmptyOrMoreThanOne("topic", topics);
                        String topic = topics.get(0);

                        ImmutableList<String> pulsarAddresses = rows.collect(TableMeta.RowMeta::getPulsarAddress).distinct();
                        checkEmptyOrMoreThanOne("pulsar address", pulsarAddresses);
                        String pulsarAddress = pulsarAddresses.get(0);

                        ImmutableList<Integer> priorities = rows.collect(TableMeta.RowMeta::getPriority).distinct();
                        checkEmptyOrMoreThanOne("priority", priorities);
                        Integer priority = priorities.get(0);

                        ImmutableList<String> tagTexts = rows.collect(TableMeta.RowMeta::getTags).distinct();
                        checkEmptyOrMoreThanOne("tags", tagTexts);
                        String tagText = ObjectUtil.isNull(tagTexts.get(0)) ? "" : tagTexts.get(0);
                        ImmutableList<String> tags = Lists.immutable.of(tagText.split(","));

                        ImmutableList<Integer> versions = rows.collect(TableMeta.RowMeta::getVersion).distinct();
                        checkEmptyOrMoreThanOne("version", versions);
                        Integer version = versions.get(0);

                        // 获取 Hudi 配置, 因为查出来同一张表的配置都相同, 所以直接取第一条即可
                        TableMeta.RowMeta example = rows.get(0);
                        TableMeta.HudiMeta hudiMeta = TableMeta.HudiMeta.builder()
                                .targetDataSource(example.getTgtDb())
                                .targetTable(example.getTgtTable())
                                .targetTableType(example.getTgtTableType())
                                .targetHdfsPath(example.getTgtHdfsPath())
                                .sourceTasks(example.getSourceTasks())
                                .writeTasks(example.getWriteTasks())
                                .writeOperation(example.getWriteOperation())
                                .writeTaskMaxMemory(example.getWriteTaskMaxMemory())
                                .writeBatchSize(example.getWriteBatchSize())
                                .writeRateLimit(example.getWriteRateLimit())
                                .bucketIndexNumber(example.getBucketIndexNumber())
                                .compactionStrategy(example.getCompactionStrategy())
                                .compactionTasks(example.getCompactionTasks())
                                .compactionDeltaCommits(example.getCompactionDeltaCommits())
                                .compactionDeltaSeconds(example.getCompactionDeltaSeconds())
                                .compactionAsyncEnabled(example.getCompactionAsyncEnabled())
                                .compactionMaxMemory(example.getCompactionMaxMemory())
                                .configs(example.getConfigs())
                                .keepFileVersion(example.getKeepFileVersion())
                                .keepCommitVersion(example.getKeepCommitVersion())
                                .build();
                        TableMeta.YarnMeta syncYarnMeta = TableMeta.YarnMeta.builder()
                                .jobManagerMemory(example.getSyncJobManagerMemory())
                                .taskManagerMemory(example.getSyncTaskManagerMemory())
                                .build();
                        TableMeta.YarnMeta compactionYarnMeta = TableMeta.YarnMeta.builder()
                                .jobManagerMemory(example.getCompactionJobManagerMemory())
                                .taskManagerMemory(example.getCompactionTaskManagerMemory())
                                .build();
                        TableMeta.ConfigMeta configMeta = TableMeta.ConfigMeta.builder()
                                .messageId(example.getMessageId())
                                .metricPublishUrl(example.getMetricPublishUrl())
                                .metricPrometheusUrl(example.getMetricPrometheusUrl())
                                .metricApiUrl(example.getMetricApiUrl())
                                .metricPublishDelay(example.getMetricPublishDelay())
                                .metricPublishPeriod(example.getMetricPublishPeriod())
                                .metricPublishTimeout(example.getMetricPublishTimeout())
                                .metricPublishBatch(example.getMetricPublishBatch())
                                .checkpointRootPath(example.getCheckpointRootPath())
                                .zookeeperUrl(example.getZookeeperUrl())
                                .build();
                        TableMeta.JobMeta jobMeta = TableMeta.JobMeta.builder()
                                .id(example.getJobId())
                                .name(example.getJobName())
                                .build();

                        TableMeta.ConnectionMeta connectionMeta = null;
                        String connectionText = example.getConnection();
                        if (StrUtil.isNotBlank(connectionText)) {
                            JSONObject connectionObj = JSONUtil.parseObj(connectionText);
                            connectionMeta = TableMeta.ConnectionMeta.builder()
                                    .url(connectionObj.getStr("jdbc_url"))
                                    .user(connectionObj.getStr("jdbc_user"))
                                    .password(connectionObj.getStr("jdbc_password"))
                                    .driver(connectionObj.getStr("jdbc_driver"))
                                    .build();
                        }

                        ImmutableList<String> partitionFields = rows.collect(TableMeta.RowMeta::getPartitionField).distinct();
                        checkEmptyOrMoreThanOne("partition_field", filterFields);
                        String partitionField = partitionFields.get(0);

                        List<TableMeta.FieldMeta> primaryKeys = Lists.mutable.empty(),
                                partitionKeys = Lists.mutable.empty(),
                                fieldMetaList = Lists.mutable.empty();
                        for (TableMeta.RowMeta rowMeta : rows) {
                            boolean isPrimaryKey = StrUtil.equals(STATUS_Y, rowMeta.getPrimaryKey());
                            boolean isPartitionKey = StrUtil.equals(STATUS_Y, rowMeta.getPartitionKey());
                            TableMeta.FieldMeta fieldMeta = TableMeta.FieldMeta.builder()
                                    .name(rowMeta.getFieldName().toUpperCase(Locale.ROOT))
                                    .sequence(rowMeta.getFieldSeq())
                                    .type(rowMeta.getFieldType())
                                    .isPrimaryKey(isPrimaryKey)
                                    .partitionKey(isPartitionKey)
                                    .length(rowMeta.getLength())
                                    .scala(rowMeta.getScala())
                                    .build();
                            if (isPrimaryKey) {
                                primaryKeys.add(fieldMeta);
                            }
                            if (isPartitionKey) {
                                partitionKeys.add(fieldMeta);
                            }
                            fieldMetaList.add(fieldMeta);
                        }
                        return TableMeta.builder()
                                .alias(alias)
                                .source(dataSource)
                                .schema(schema)
                                .table(table)
                                .type(type)
                                .primaryKeys(primaryKeys)
                                .partitionKeys(partitionKeys)
                                .hudi(hudiMeta)
                                .fields(fieldMetaList)
                                .filterField(filterField)
                                .filterValues(filterValues.toList())
                                .filterType(filterType)
                                .topic(topic)
                                .pulsarAddress(pulsarAddress)
                                .syncYarn(syncYarnMeta)
                                .compactionYarn(compactionYarnMeta)
                                .partitionField(partitionField)
                                .config(configMeta)
                                .job(jobMeta)
                                .connection(connectionMeta)
                                .priority(priority)
                                .sourceType(sourceType)
                                .tags(tags.toList())
                                .version(version)
                                .build();
                    } catch (Throwable throwable) {
                        throw new RuntimeException(throwable);
                    }
                })
                .toList()
                .toImmutable();
    }

    @Cacheable(value = "table-metas", sync = true)
    @Retryable(Throwable.class)
    public ImmutableList<TableMeta> tableMetas() {
        return flinkJobService.flinkJobs().flatCollect(job -> tableMetaList(job.getId()));
    }

    @Cacheable(value = "table-metas", sync = true, key = "#flinkJobId")
    @Retryable(Throwable.class)
    public ImmutableList<TableMeta> tableMetas(Long flinkJobId) {
        return tableMetaList(flinkJobId);
    }

    @Cacheable(value = "table-metas", sync = true, key = "#flinkJobId.toString()+#alias")
    @Retryable(Throwable.class)
    public TableMeta tableMeta(Long flinkJobId, String alias) {
        return tableMetaList(flinkJobId, alias)
                .getFirstOptional()
                .orElseThrow(TableMetaNotFoundException::new);
    }

    @Cacheable(value = "table-count", sync = true)
    @Retryable(Throwable.class)
    public Long tableCount() {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select(StrUtil.format("count(distinct concat({}, {}))", TbAppCollectTableInfo.SRC_SCHEMA_A, TbAppCollectTableInfo.SRC_TABLE_A))
                ).build(),
                Long.class
        );
    }

    @Cacheable(value = "table-focus-count", sync = true)
    @Retryable(Throwable.class)
    public Long tableFocusCount() {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select(StrUtil.format("count(distinct concat({}, {}))", TbAppCollectTableInfo.SRC_SCHEMA_A, TbAppCollectTableInfo.SRC_TABLE_A)),
                        10000
                ).build(),
                Long.class
        );
    }

    @Cacheable(value = "hudi-count", sync = true)
    @Retryable(Throwable.class)
    public Long hudiCount() {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select(StrUtil.format("count(distinct {}) as count", TbAppCollectTableInfo.TGT_HDFS_PATH_A))
                ).build(),
                Long.class
        );
    }

    @Cacheable(value = "hudi-focus-count", sync = true)
    @Retryable(Throwable.class)
    public Long hudiFocusCount() {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select(StrUtil.format("count(distinct {}) as count", TbAppCollectTableInfo.TGT_HDFS_PATH_A)),
                        10000
                ).build(),
                Long.class
        );
    }

    @Cacheable(value = "hive-count", sync = true)
    @Retryable(Throwable.class)
    public Long hiveCount() {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select(StrUtil.format("count(distinct concat({}, {})) as count", TbAppCollectTableInfo.HIVE_DB_A, TbAppCollectTableInfo.HIVE_TABLE_A))
                ).build(),
                Long.class
        );
    }

    @Cacheable(value = "hive-focus-count", sync = true)
    @Retryable(Throwable.class)
    public Long hiveFocusCount() {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select(StrUtil.format("count(distinct concat({}, {})) as count", TbAppCollectTableInfo.HIVE_DB_A, TbAppCollectTableInfo.HIVE_TABLE_A)),
                        10000
                ).build(),
                Long.class
        );
    }

    @Cacheable(value = "simple-table-metas", sync = true, cacheManager = "long-cache")
    @Retryable(Throwable.class)
    public ImmutableList<SimpleTableMeta> simpleTableMetas(Long flinkJobId, String alias) {
        return Lists.immutable.ofAll(
                mysqlJdbcTemplate.query(
                        generateSimpleTableMetaList(
                                SqlBuilder.select(
                                        TbAppFlinkJobConfig.ID_A,
                                        TbAppFlinkJobConfig.NAME_A,
                                        TbAppCollectTableInfo.ALIAS_A,
                                        TbAppCollectTableInfo.SRC_SCHEMA_A,
                                        TbAppCollectTableInfo.SRC_TABLE_A,
                                        TbAppCollectTableInfo.TGT_DB_A,
                                        TbAppCollectTableInfo.TGT_TABLE_A,
                                        TbAppCollectTableInfo.TGT_HDFS_PATH_A
                                ),
                                flinkJobId,
                                alias
                        ).build(),
                        (rs, row) -> new SimpleTableMeta(
                                rs.getLong(1),
                                rs.getString(2),
                                rs.getString(3),
                                rs.getString(4),
                                rs.getString(5),
                                rs.getString(6),
                                rs.getString(7),
                                rs.getString(8)
                        )
                )
        );
    }

    @Cacheable(value = "exists-table", sync = true)
    @Retryable(Throwable.class)
    public Boolean existsTable(Long flinkJobId, String alias) {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select("count(*) > 0"),
                        flinkJobId,
                        alias
                ).build(),
                Boolean.class
        );
    }

    @Cacheable(value = "exists-table", sync = true)
    @Retryable(Throwable.class)
    public Boolean existsTableByHdfs(String hdfs) {
        return mysqlJdbcTemplate.queryForObject(
                generateSimpleTableMetaList(
                        SqlBuilder.select("count(*) > 0"),
                        hdfs
                ).build(),
                Boolean.class
        );
    }

    @Retryable(Throwable.class)
    public void disableTable(Long flinkJobId, String alias) {
        mysqlJdbcTemplate.update(
                SqlBuilder.update(TbAppCollectTableInfo._origin_)
                        .set(TbAppCollectTableInfo.STATUS_O, Q)
                        .whereEq(TbAppCollectTableInfo.FLINK_JOB_ID_O, Q)
                        .andEq(TbAppCollectTableInfo.ALIAS_O, Q)
                        .precompileSql(),
                STATUS_N,
                flinkJobId,
                alias
        );
    }

    @Retryable(Throwable.class)
    public void saveTableMeta(TableMetaAdd addon) {
        mysqlJdbcTemplate.update(
                SqlBuilder.insertInto(
                                TbAppCollectTableInfo._origin_,
                                TbAppCollectTableInfo.ID_O,
                                TbAppCollectTableInfo.ALIAS_O,
                                TbAppCollectTableInfo.FLINK_JOB_ID_O,
                                TbAppCollectTableInfo.HUDI_JOB_ID_O,
                                TbAppCollectTableInfo.SYNC_YARN_JOB_ID_O,
                                TbAppCollectTableInfo.COMPACTION_YARN_JOB_ID_O,
                                TbAppCollectTableInfo.CONFIG_ID_O,
                                TbAppCollectTableInfo.SRC_DB_O,
                                TbAppCollectTableInfo.SRC_TYPE_O,
                                TbAppCollectTableInfo.SRC_SCHEMA_O,
                                TbAppCollectTableInfo.SRC_TABLE_O,
                                TbAppCollectTableInfo.SRC_PULSAR_ADDR_O,
                                TbAppCollectTableInfo.SRC_TOPIC_O,
                                TbAppCollectTableInfo.TGT_DB_O,
                                TbAppCollectTableInfo.TGT_TABLE_O,
                                TbAppCollectTableInfo.TGT_HDFS_PATH_O,
                                TbAppCollectTableInfo.TGT_TABLE_TYPE_O,
                                TbAppCollectTableInfo.STATUS_O,
                                TbAppCollectTableInfo.FILTER_FIELD_O,
                                TbAppCollectTableInfo.FILTER_VALUES_O,
                                TbAppCollectTableInfo.FILTER_TYPE_O,
                                TbAppCollectTableInfo.BUCKET_NUMBER_O,
                                TbAppCollectTableInfo.PARTITION_FIELD_O
                        )
                        .values()
                        .addValue(Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q, Q)
                        .precompileSql(),
                addon.getId(),
                addon.getAlias(),
                addon.getFlinkJobId(),
                addon.getHudiJobId(),
                addon.getSyncYarnJobId(),
                addon.getCompactionYarnJobId(),
                1L,
                addon.getSourceDatabase(),
                addon.getSourceDatabaseType(),
                addon.getSourceSchema(),
                addon.getSourceTable(),
                addon.getSourcePulsarAddress(),
                addon.getSourceTopic(),
                addon.getTargetSchema(),
                addon.getTargetTable(),
                addon.getTargetHdfs(),
                "MERGE_ON_READ",
                STATUS_N,
                addon.getFilterField(),
                addon.getFilterValues(),
                addon.getFilterType(),
                addon.getBucketNumber(),
                "CITY_ID"
        );
    }

    @Cacheable(value = "all-table-info-search-cache", sync = true, cacheManager = "long-cache")
    @Retryable(Throwable.class)
    public ImmutableList<TableInfoSearchCache> allTableInfoSearchCache() {
        return Lists.immutable.ofAll(mysqlJdbcTemplate.query(
                generateSimpleTableMetaList(
                        SqlBuilder.select(
                                TbAppFlinkJobConfig.ID_A,
                                TbAppCollectTableInfo.ALIAS_A,
                                TbAppCollectTableInfo.TGT_HDFS_PATH_A,
                                TbAppCollectTableInfo.SRC_PULSAR_ADDR_A,
                                TbAppCollectTableInfo.SRC_TOPIC_A
                        )
                ).build(),
                (rs, row) -> new TableInfoSearchCache(
                        rs.getLong(1),
                        rs.getString(2),
                        rs.getString(3),
                        rs.getString(4),
                        rs.getString(5)
                )
        )).distinct();
    }
}
