package com.lanyuanxiaoyao.service.sync.utils;

import com.lanyuanxiaoyao.service.common.entity.FlinkJob;
import com.lanyuanxiaoyao.service.common.entity.Record;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import com.lanyuanxiaoyao.service.sync.configuration.GlobalConfiguration;
import com.lanyuanxiaoyao.service.sync.functions.OperationTypeFilter;
import com.lanyuanxiaoyao.service.sync.functions.Record2RowDataFunction;
import com.lanyuanxiaoyao.service.sync.functions.type.TypeConverter;
import java.util.List;
import java.util.Objects;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.EnvironmentSettings;
import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.types.logical.RowType;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.configuration.OptionsResolver;
import org.apache.hudi.org.apache.avro.Schema;
import org.apache.hudi.sink.utils.Pipelines;
import org.apache.hudi.util.AvroSchemaConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Flink 相关的工具
 *
 * @author lanyuanxiaoyao
 * @version 0.0.2
 * @date 2022-04-20
 */
public class SyncUtils {
    private static final Logger logger = LoggerFactory.getLogger(SyncUtils.class);

    private static final long K = 1024;
    private static final long M = 1024 * K;
    private static final long G = 1024 * M;

    public static Schema avroSchemaWithExtraFields(TableMeta meta) {
        return TypeConverter.getInstance(meta).convertToSchema(meta);
    }

    public static void sinkToHoodieByTable(GlobalConfiguration globalConfiguration, FlinkJob flinkJob, TableMeta tableMeta, StreamExecutionEnvironment environment, DataStream<Record> inputDataStream) {
        Schema schema = avroSchemaWithExtraFields(tableMeta);
        DataStream<RowData> dataStream = inputDataStream
                .filter(new OperationTypeFilter(globalConfiguration, flinkJob, tableMeta))
                .name("Count operation type")
                .map(new Record2RowDataFunction(globalConfiguration, flinkJob, tableMeta), TypeInformation.of(new TypeHint<List<RowData>>() {
                }))
                .name("Covert Row ( " + tableMeta.getSchema() + "-" + tableMeta.getTable() + " )")
                .flatMap((list, collector) -> list.forEach(collector::collect), TypeInformation.of(RowData.class))
                .filter(Objects::nonNull)
                .name("Filter not null");

        RowType rowType = (RowType) AvroSchemaConverter.convertToDataType(schema).getLogicalType();
        StreamTableEnvironment tableEnvironment = StreamTableEnvironment.create(
                environment,
                EnvironmentSettings.newInstance()
                        .inStreamingMode()
                        .useBlinkPlanner()
                        .build()
        );
        Configuration configuration = tableEnvironment.getConfig().getConfiguration();
        int parallelism = configuration.getInteger("parallelism", 1);
        configuration = ConfigurationUtils.getSyncFlinkConfiguration(globalConfiguration, configuration, flinkJob, tableMeta, schema, parallelism);

        DataStream<HoodieRecord> hoodieRecordDataStream = Pipelines.bootstrap(configuration, rowType, parallelism, dataStream);
        DataStream<Object> pipeline = Pipelines.hoodieStreamWrite(configuration, parallelism, hoodieRecordDataStream);
        if (OptionsResolver.needsAsyncCompaction(configuration)) {
            Pipelines.compact(configuration, pipeline);
        } else {
            Pipelines.clean(configuration, pipeline);
        }
    }
}
