package com.lanyuanxiaoyao.service.executor.task.helper;

import cn.hutool.core.collection.IterUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
import com.lanyuanxiaoyao.service.executor.core.TaskContext;
import java.io.IOException;
import java.util.Optional;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * HDFS工具
 *
 * @author lanyuanxiaoyao
 * @date 2024-01-22
 */
public class HdfsHelper {
    private static final Logger logger = LoggerFactory.getLogger(HdfsHelper.class);

    public static void checkHdfsPath(FileSystem fileSystem, String path) throws IOException {
        checkHdfsPath(fileSystem, new Path(path));
    }

    public static void checkHdfsPath(FileSystem fileSystem, Path path) throws IOException {
        if (!fileSystem.exists(path)) {
            throw new RuntimeException(StrUtil.format("HDFS {} is not exists", path.toString()));
        }
    }

    public static Integer logScanParallelismPredict(Iterable<?> list) {
        return logScanParallelismPredict(IterUtil.size(list));
    }

    public static Integer logScanParallelismPredict(Integer pathNum) {
        return Math.max(1, Math.min(pathNum / 20, 100));
    }

    public static Integer baseScanParallelismPredict(Iterable<?> list) {
        return baseScanParallelismPredict(IterUtil.size(list));
    }

    public static Integer baseScanParallelismPredict(Integer pathNum) {
        return Math.max(1, Math.min(pathNum, 500));
    }

    public static ImmutableList<String> latestBasePaths(ImmutableList<FileStatus> paths) {
        return paths
                .asParallel(ExecutorProvider.EXECUTORS, 1)
                .reject(status -> status.getLen() < 1)
                .collect(FileStatus::getPath)
                .groupBy(FSUtils::getFileIdFromFilePath)
                .multiValuesView()
                .collect(pathList -> pathList
                        .toSortedListBy(path -> {
                            String commitTime = FSUtils.getCommitTime(path.getName());
                            try {
                                return Long.valueOf(commitTime);
                            } catch (Throwable throwable) {
                                return 0L;
                            }
                        })
                        .getLastOptional())
                .select(Optional::isPresent)
                .collect(Optional::get)
                .collect(Path::toString)
                .toList()
                .toImmutable();
    }

    public static ImmutableList<FileStatus> basePaths(ImmutableList<FileStatus> paths) throws IOException {
        return paths.select(status -> FSUtils.isBaseFile(status.getPath()));
    }

    public static ImmutableList<FileStatus> logPaths(ImmutableList<FileStatus> paths) {
        return paths.select(status -> FSUtils.isLogFile(status.getPath()));
    }

    public static ImmutableList<FileStatus> hdfsPaths(FileSystem fileSystem, String root) throws IOException {
        return hdfsPaths(fileSystem, new Path(root));
    }

    /**
     * 获取hdfs文件列表，这个方法是专门给hudi表查询base或log文件使用，所以只扫描一层
     */
    public static ImmutableList<FileStatus> hdfsPaths(FileSystem fileSystem, Path root) throws IOException {
        return Lists.immutable.of(fileSystem.listStatus(root))
                .reject(status -> StrUtil.equals(HoodieTableMetaClient.METAFOLDER_NAME, status.getPath().getName()))
                .flatCollect(status -> {
                    try {
                        if (status.isDirectory()) {
                            return Lists.immutable.of(fileSystem.listStatus(status.getPath()));
                        } else {
                            return Lists.immutable.of(status);
                        }
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                });
    }

    public static void createResult(FileSystem fileSystem, TaskContext context, String result) throws IOException {
        Path resultPath = new Path(context.getResultPath() + "/" + context.getTaskId() + "/task-result");
        try (FSDataOutputStream outputStream = fileSystem.create(resultPath)) {
            outputStream.writeBytes(result);
        }
    }
}
