lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2351] Extract common FS and IO utils for marker mechanism (#3529)

Browse Source
This commit is contained in:
Y Ethan Guo
2021-09-09 11:45:28 -07:00
committed by GitHub
parent 57c8113ee1
commit 56d08fbe70
8 changed files with 301 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

143
hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
View File

@@ -18,14 +18,19 @@
package org.apache.hudi.common.fs;
import org.apache.hudi.common.config.SerializableConfiguration;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.model.HoodieLogFile;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.Rule;
import org.junit.contrib.java.lang.system.EnvironmentVariables;
@@ -37,8 +42,10 @@ import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -47,6 +54,7 @@ import static org.apache.hudi.common.table.timeline.HoodieActiveTimeline.COMMIT_
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
/**
@@ -308,8 +316,139 @@ public class TestFSUtils extends HoodieCommonTestHarness {
Files.createFile(partitionPath.resolve(log3));
assertEquals(3, (int) FSUtils.getLatestLogVersion(FSUtils.getFs(basePath, new Configuration()),
new Path(partitionPath.toString()), fileId, LOG_EXTENTION, instantTime).get().getLeft());
new Path(partitionPath.toString()), fileId, LOG_EXTENTION, instantTime).get().getLeft());
assertEquals(4, FSUtils.computeNextLogVersion(FSUtils.getFs(basePath, new Configuration()),
new Path(partitionPath.toString()), fileId, LOG_EXTENTION, instantTime));
new Path(partitionPath.toString()), fileId, LOG_EXTENTION, instantTime));
}
private void prepareTestDirectory(FileSystem fileSystem, String rootDir) throws IOException {
// Directory structure
// .hoodie/.temp/
// - subdir1
// - file1.txt
// - subdir2
// - file2.txt
// - file3
Path dirPath = new Path(rootDir);
String subDir1 = rootDir + "/subdir1";
String file1 = subDir1 + "/file1.txt";
String subDir2 = rootDir + "/subdir2";
String file2 = subDir2 + "/file2.txt";
String file3 = rootDir + "/file3.txt";
String[] dirs = new String[]{rootDir, subDir1, subDir2};
String[] files = new String[]{file1, file2, file3};
// clean up first
cleanUpTestDirectory(fileSystem, rootDir);
for (String dir : dirs) {
fileSystem.mkdirs(new Path(dir));
}
for (String filename : files) {
fileSystem.create(new Path(filename));
}
}
private void cleanUpTestDirectory(FileSystem fileSystem, String rootDir) throws IOException {
fileSystem.delete(new Path(rootDir), true);
}
@Test
public void testDeleteExistingDir() throws IOException {
String rootDir = basePath + "/.hoodie/.temp";
FileSystem fileSystem = metaClient.getFs();
prepareTestDirectory(fileSystem, rootDir);
Path rootDirPath = new Path(rootDir);
assertTrue(fileSystem.exists(rootDirPath));
assertTrue(FSUtils.deleteDir(
new HoodieLocalEngineContext(metaClient.getHadoopConf()), fileSystem, rootDirPath, 2));
assertFalse(fileSystem.exists(rootDirPath));
}
@Test
public void testDeleteNonExistingDir() throws IOException {
String rootDir = basePath + "/.hoodie/.temp";
FileSystem fileSystem = metaClient.getFs();
cleanUpTestDirectory(fileSystem, rootDir);
assertFalse(FSUtils.deleteDir(
new HoodieLocalEngineContext(metaClient.getHadoopConf()), fileSystem, new Path(rootDir), 2));
}
@Test
public void testDeleteSubDirectoryRecursively() throws IOException {
String rootDir = basePath + "/.hoodie/.temp";
String subPathStr = rootDir + "/subdir1";
FileSystem fileSystem = metaClient.getFs();
prepareTestDirectory(fileSystem, rootDir);
assertTrue(FSUtils.deleteSubPath(
subPathStr, new SerializableConfiguration(fileSystem.getConf()), true));
}
@Test
public void testDeleteSubDirectoryNonRecursively() throws IOException {
String rootDir = basePath + "/.hoodie/.temp";
String subPathStr = rootDir + "/subdir1";
FileSystem fileSystem = metaClient.getFs();
prepareTestDirectory(fileSystem, rootDir);
assertThrows(
HoodieIOException.class,
() -> FSUtils.deleteSubPath(
subPathStr, new SerializableConfiguration(fileSystem.getConf()), false));
}
@Test
public void testDeleteSubPathAsFile() throws IOException {
String rootDir = basePath + "/.hoodie/.temp";
String subPathStr = rootDir + "/file3.txt";
FileSystem fileSystem = metaClient.getFs();
prepareTestDirectory(fileSystem, rootDir);
assertTrue(FSUtils.deleteSubPath(
subPathStr, new SerializableConfiguration(fileSystem.getConf()), false));
}
@Test
public void testDeleteNonExistingSubDirectory() throws IOException {
String rootDir = basePath + "/.hoodie/.temp";
String subPathStr = rootDir + "/subdir10";
FileSystem fileSystem = metaClient.getFs();
cleanUpTestDirectory(fileSystem, rootDir);
assertFalse(FSUtils.deleteSubPath(
subPathStr, new SerializableConfiguration(fileSystem.getConf()), true));
}
@Test
public void testParallelizeSubPathProcessWithExistingDir() throws IOException {
String rootDir = basePath + "/.hoodie/.temp";
FileSystem fileSystem = metaClient.getFs();
prepareTestDirectory(fileSystem, rootDir);
Map<String, List<String>> result = FSUtils.parallelizeSubPathProcess(
new HoodieLocalEngineContext(fileSystem.getConf()), fileSystem, new Path(rootDir), 2,
fileStatus -> !fileStatus.getPath().getName().contains("1"),
pairOfSubPathAndConf -> {
Path subPath = new Path(pairOfSubPathAndConf.getKey());
List<String> listFiles = new ArrayList<>();
try {
FileSystem fs = subPath.getFileSystem(pairOfSubPathAndConf.getValue().get());
FileStatus[] fileStatuses = fs.listStatus(subPath);
listFiles = Arrays.stream(fileStatuses)
.map(fileStatus -> fileStatus.getPath().getName()).collect(Collectors.toList());
} catch (IOException e) {
e.printStackTrace();
}
return listFiles;
}
);
assertEquals(2, result.size());
for (String subPath : result.keySet()) {
if (subPath.contains("subdir2")) {
assertEquals(Collections.singletonList("file2.txt"), result.get(subPath));
} else if (subPath.contains("file3")) {
assertEquals(Collections.singletonList("file3.txt"), result.get(subPath));
}
}
}
}

11
hudi-common/src/test/java/org/apache/hudi/common/util/TestFileIOUtils.java
View File

@@ -26,6 +26,9 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -63,4 +66,12 @@ public class TestFileIOUtils extends HoodieCommonTestHarness {
inputStream = new ByteArrayInputStream(msg.getBytes(StandardCharsets.UTF_8));
assertEquals(msg.length(), FileIOUtils.readAsByteArray(inputStream).length);
}
@Test
public void testReadAsUTFStringLines() {
String content = "a\nb\nc";
List<String> expectedLines = Arrays.stream(new String[]{"a", "b", "c"}).collect(Collectors.toList());
ByteArrayInputStream inputStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
assertEquals(expectedLines, FileIOUtils.readAsUTFStringLines(inputStream));
}
}