package com.lanyuanxiaoyao.service.executor.task.helper;

import cn.hutool.core.collection.IterUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.configuration.ExecutorProvider;
import com.lanyuanxiaoyao.service.executor.core.TaskContext;
import java.io.IOException;
import java.util.Optional;
import java.util.function.Predicate;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.fs.FSUtils;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * HDFS工具
 *
 * @author lanyuanxiaoyao
 * @date 2024-01-22
 */
public class HdfsHelper {
    private static final Logger logger = LoggerFactory.getLogger(HdfsHelper.class);

    public static void checkHdfsPath(FileSystem fileSystem, String path) throws IOException {
        checkHdfsPath(fileSystem, new Path(path));
    }

    public static void checkHdfsPath(FileSystem fileSystem, Path path) throws IOException {
        if (!fileSystem.exists(path)) {
            throw new RuntimeException(StrUtil.format("HDFS {} is not exists", path.toString()));
        }
    }

    public static Integer logScanParallelismPredict(Iterable<?> list) {
        return logScanParallelismPredict(IterUtil.size(list));
    }

    public static Integer logScanParallelismPredict(Integer pathNum) {
        return Math.max(1, Math.min(pathNum / 20, 100));
    }

    public static Integer baseScanParallelismPredict(Iterable<?> list) {
        return baseScanParallelismPredict(IterUtil.size(list));
    }

    public static Integer baseScanParallelismPredict(Integer pathNum) {
        return Math.max(1, Math.min(pathNum / 2, 500));
    }

    public static ImmutableList<String> latestBasePaths(FileSystem fileSystem, String root) throws IOException {
        return latestBasePaths(fileSystem, new Path(root));
    }

    public static ImmutableList<String> latestBasePaths(FileSystem fileSystem, Path root) throws IOException {
        return basePaths(fileSystem, root)
                .asParallel(ExecutorProvider.EXECUTORS, 1)
                .collect(Path::new)
                .reject(path -> {
                    try {
                        return FSUtils.getFileSize(fileSystem, path) < 1;
                    } catch (IOException e) {
                        logger.error("Get file size error", e);
                    }
                    return true;
                })
                .groupBy(FSUtils::getFileIdFromFilePath)
                .multiValuesView()
                .collect(pathList -> pathList
                        .toSortedListBy(path -> {
                            String commitTime = FSUtils.getCommitTime(path.getName());
                            try {
                                return Long.valueOf(commitTime);
                            } catch (Throwable throwable) {
                                return 0L;
                            }
                        })
                        .getLastOptional())
                .select(Optional::isPresent)
                .collect(Optional::get)
                .collect(Path::toString)
                .toList()
                .toImmutable();
    }

    public static ImmutableList<String> basePaths(FileSystem fileSystem, String root) throws IOException {
        return basePaths(fileSystem, new Path(root));
    }

    public static ImmutableList<String> basePaths(FileSystem fileSystem, Path root) throws IOException {
        return hdfsPaths(fileSystem, root, FSUtils::isBaseFile);
    }

    public static ImmutableList<String> logPaths(FileSystem fileSystem, String root) throws IOException {
        return logPaths(fileSystem, new Path(root));
    }

    public static ImmutableList<String> logPaths(FileSystem fileSystem, Path root) throws IOException {
        return hdfsPaths(fileSystem, root, FSUtils::isLogFile);
    }

    public static ImmutableList<String> hdfsPaths(FileSystem fileSystem, Path root, Predicate<Path> check) throws IOException {
        return Lists.immutable.of(fileSystem.listStatus(root))
                .reject(status -> StrUtil.equals(".hoodie", status.getPath().getName()))
                .flatCollect(status -> {
                    try {
                        if (status.isDirectory()) {
                            return Lists.immutable.of(fileSystem.listStatus(status.getPath()));
                        } else {
                            return Lists.immutable.of(status);
                        }
                    } catch (IOException e) {
                        throw new RuntimeException(e);
                    }
                })
                .collect(FileStatus::getPath)
                .select(check::test)
                .collect(Path::toString);
    }

    public static void createResult(FileSystem fileSystem, TaskContext context, String result) throws IOException {
        Path resultPath = new Path(context.getResultPath() + "/" + context.getTaskId() + "/task-result");
        try(FSDataOutputStream outputStream = fileSystem.create(resultPath)) {
            outputStream.writeUTF(result);
        }
    }
}
