package com.lanyuanxiaoyao.service.executor.task;

import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.common.Constants;
import com.lanyuanxiaoyao.service.executor.core.TaskContext;
import com.lanyuanxiaoyao.service.executor.task.functions.ReadHudiFile;
import com.lanyuanxiaoyao.service.executor.task.helper.ArgumentsHelper;
import com.lanyuanxiaoyao.service.executor.task.helper.FlinkHelper;
import com.lanyuanxiaoyao.service.executor.task.helper.HdfsHelper;
import org.apache.flink.api.common.functions.RichReduceFunction;
import org.apache.flink.api.common.typeinfo.TypeHint;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.CloseableIterator;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * 查找base文件最后的opts
 *
 * @author lanyuanxiaoyao
 * @date 2024-01-22
 */
public class LatestOperationTimeScan {
    private static final Logger logger = LoggerFactory.getLogger(LatestOperationTimeScan.class);

    public static void main(String[] args) throws Exception {
        TaskContext taskContext = ArgumentsHelper.getContext(args);
        logger.info("Context: {}", taskContext);

        ArgumentsHelper.checkMetadata(taskContext, "hdfs");
        String hdfs = (String) taskContext.getMetadata().get("hdfs");

        FileSystem fileSystem = FileSystem.get(new Configuration());
        HdfsHelper.checkHdfsPath(fileSystem, hdfs);
        ImmutableList<String> basePaths = HdfsHelper.latestBasePaths(fileSystem, hdfs);

        int parallelism = HdfsHelper.baseScanParallelismPredict(basePaths);

        StreamExecutionEnvironment environment = FlinkHelper.getBatchEnvironment();
        environment.setParallelism(parallelism);
        String maxValue = "0";
        try (CloseableIterator<String> iterator = environment
                .fromCollection(basePaths.toList())
                .name("Read base paths")
                .flatMap(new ReadHudiFile())
                .name("Read hudi file")
                .setParallelism(parallelism)
                .map(view -> (String) view.getAttributes().getOrDefault(Constants.LATEST_OPERATION_TIMESTAMP_KEY_NAME, "0"))
                .map(time -> new Tuple2<>(RandomUtil.randomInt(parallelism), time), TypeInformation.of(new TypeHint<Tuple2<Integer, String>>() {
                }))
                .keyBy(tuple -> tuple.f0)
                .reduce(new RichReduceFunction<Tuple2<Integer, String>>() {
                    @Override
                    public Tuple2<Integer, String> reduce(Tuple2<Integer, String> value1, Tuple2<Integer, String> value2) throws Exception {
                        return value1.f1.compareTo(value2.f1) > 0 ? value1 : value2;
                    }
                })
                .map(tuple -> tuple.f1)
                /*.sinkTo(FlinkHelper.createFileSink(taskContext))*/
                .executeAndCollect("Find latest opts")) {
            while (iterator.hasNext()) {
                String item = iterator.next();
                if (item.compareTo(maxValue) > 0) {
                    maxValue = item;
                }
            }
        }
        HdfsHelper.createResult(fileSystem, taskContext, StrUtil.trim(maxValue));
    }
}
