[MINOR] optimize FilePathUtils (#2931)
This commit is contained in:
@@ -131,21 +131,6 @@ public class FilePathUtils {
|
|||||||
return sb.toString();
|
return sb.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Generates partition values from path.
|
|
||||||
*
|
|
||||||
* @param currPath Partition file path
|
|
||||||
* @param hivePartition Whether the partition path is with Hive style
|
|
||||||
* @param partitionKeys Partition keys
|
|
||||||
* @return Sequential partition specs.
|
|
||||||
*/
|
|
||||||
public static List<String> extractPartitionValues(
|
|
||||||
Path currPath,
|
|
||||||
boolean hivePartition,
|
|
||||||
String[] partitionKeys) {
|
|
||||||
return new ArrayList<>(extractPartitionKeyValues(currPath, hivePartition, partitionKeys).values());
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generates partition key value mapping from path.
|
* Generates partition key value mapping from path.
|
||||||
*
|
*
|
||||||
@@ -265,7 +250,7 @@ public class FilePathUtils {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fileStatus.isDir() && !isHiddenFile(fileStatus)) {
|
if (fileStatus.isDirectory() && !isHiddenFile(fileStatus)) {
|
||||||
for (FileStatus stat : fs.listStatus(fileStatus.getPath())) {
|
for (FileStatus stat : fs.listStatus(fileStatus.getPath())) {
|
||||||
listStatusRecursively(fs, stat, level + 1, expectLevel, results);
|
listStatusRecursively(fs, stat, level + 1, expectLevel, results);
|
||||||
}
|
}
|
||||||
@@ -275,7 +260,7 @@ public class FilePathUtils {
|
|||||||
private static boolean isHiddenFile(FileStatus fileStatus) {
|
private static boolean isHiddenFile(FileStatus fileStatus) {
|
||||||
String name = fileStatus.getPath().getName();
|
String name = fileStatus.getPath().getName();
|
||||||
// the log files is hidden file
|
// the log files is hidden file
|
||||||
return name.startsWith("_") || name.startsWith(".") && !name.contains(".log.");
|
return name.startsWith("_") || (name.startsWith(".") && !name.contains(".log."));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -393,7 +378,7 @@ public class FilePathUtils {
|
|||||||
*/
|
*/
|
||||||
public static org.apache.flink.core.fs.Path[] toFlinkPaths(Path[] paths) {
|
public static org.apache.flink.core.fs.Path[] toFlinkPaths(Path[] paths) {
|
||||||
return Arrays.stream(paths)
|
return Arrays.stream(paths)
|
||||||
.map(p -> toFlinkPath(p))
|
.map(FilePathUtils::toFlinkPath)
|
||||||
.toArray(org.apache.flink.core.fs.Path[]::new);
|
.toArray(org.apache.flink.core.fs.Path[]::new);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user