lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3984] Remove mandatory check of partiton path for cli command (#5458)

Browse Source
This commit is contained in:
miomiocat
2022-07-01 01:00:13 +08:00
committed by GitHub
parent 8547899a39
commit 397fd30142
5 changed files with 106 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java
View File

@@ -59,24 +59,73 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
@Tag("functional")
public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
private String nonpartitionedTablePath;
private String partitionedTablePath;
private String partitionPath;
private SyncableFileSystemView fsView;
private SyncableFileSystemView nonpartitionedFsView;
private SyncableFileSystemView partitionedFsView;
@BeforeEach
public void init() throws IOException {
createNonpartitionedTable();
createPartitionedTable();
}
private void createNonpartitionedTable() throws IOException {
HoodieCLI.conf = hadoopConf();
// Create table and connect
String tableName = tableName();
String tablePath = tablePath(tableName);
String nonpartitionedTableName = "nonpartitioned_" + tableName();
nonpartitionedTablePath = tablePath(nonpartitionedTableName);
new TableCommand().createTable(
tablePath, tableName,
"COPY_ON_WRITE", "", 1, "org.apache.hudi.common.model.HoodieAvroPayload");
nonpartitionedTablePath, nonpartitionedTableName,
"COPY_ON_WRITE", "", 1, "org.apache.hudi.common.model.HoodieAvroPayload");
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
Files.createDirectories(Paths.get(nonpartitionedTablePath));
// Generate 2 commits
String commitTime1 = "3";
String commitTime2 = "4";
String fileId1 = UUID.randomUUID().toString();
// Write date files and log file
String testWriteToken = "2-0-2";
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
.makeBaseFileName(commitTime1, testWriteToken, fileId1)));
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime1, 0, testWriteToken)));
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
.makeBaseFileName(commitTime2, testWriteToken, fileId1)));
Files.createFile(Paths.get(nonpartitionedTablePath, FSUtils
.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime2, 0, testWriteToken)));
// Write commit files
Files.createFile(Paths.get(nonpartitionedTablePath, ".hoodie", commitTime1 + ".commit"));
Files.createFile(Paths.get(nonpartitionedTablePath, ".hoodie", commitTime2 + ".commit"));
// Reload meta client and create fsView
metaClient = HoodieTableMetaClient.reload(metaClient);
nonpartitionedFsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline(), true);
}
private void createPartitionedTable() throws IOException {
HoodieCLI.conf = hadoopConf();
// Create table and connect
String partitionedTableName = "partitioned_" + tableName();
partitionedTablePath = tablePath(partitionedTableName);
new TableCommand().createTable(
partitionedTablePath, partitionedTableName,
"COPY_ON_WRITE", "", 1, "org.apache.hudi.common.model.HoodieAvroPayload");
HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient();
partitionPath = HoodieTestCommitMetadataGenerator.DEFAULT_FIRST_PARTITION_PATH;
String fullPartitionPath = Paths.get(tablePath, partitionPath).toString();
String fullPartitionPath = Paths.get(partitionedTablePath, partitionPath).toString();
Files.createDirectories(Paths.get(fullPartitionPath));
// Generate 2 commits
@@ -97,13 +146,13 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
.makeLogFileName(fileId1, HoodieLogFile.DELTA_EXTENSION, commitTime2, 0, testWriteToken)));
// Write commit files
Files.createFile(Paths.get(tablePath, ".hoodie", commitTime1 + ".commit"));
Files.createFile(Paths.get(tablePath, ".hoodie", commitTime2 + ".commit"));
Files.createFile(Paths.get(partitionedTablePath, ".hoodie", commitTime1 + ".commit"));
Files.createFile(Paths.get(partitionedTablePath, ".hoodie", commitTime2 + ".commit"));
// Reload meta client and create fsView
metaClient = HoodieTableMetaClient.reload(metaClient);
fsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline(), true);
partitionedFsView = new HoodieTableFileSystemView(metaClient, metaClient.getActiveTimeline(), true);
}
/**
@@ -116,7 +165,7 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
assertTrue(cr.isSuccess());
// Get all file groups
Stream<HoodieFileGroup> fileGroups = fsView.getAllFileGroups(partitionPath);
Stream<HoodieFileGroup> fileGroups = partitionedFsView.getAllFileGroups(partitionPath);
List<Comparable[]> rows = new ArrayList<>();
fileGroups.forEach(fg -> fg.getAllFileSlices().forEach(fs -> {
@@ -164,7 +213,7 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
assertTrue(cr.isSuccess());
List<Comparable[]> rows = new ArrayList<>();
Stream<HoodieFileGroup> fileGroups = fsView.getAllFileGroups(partitionPath);
Stream<HoodieFileGroup> fileGroups = partitionedFsView.getAllFileGroups(partitionPath);
// Only get instant 1, since maxInstant was specified 2
fileGroups.forEach(fg -> fg.getAllFileSlices().filter(fs -> fs.getBaseInstantTime().equals("1")).forEach(fs -> {
@@ -197,17 +246,7 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
assertEquals(expected, got);
}
/**
* Test case for command 'show fsview latest'.
*/
@Test
public void testShowLatestFileSlices() {
// Test show with partition path '2016/03/15'
CommandResult cr = shell().executeCommand("show fsview latest --partitionPath " + partitionPath);
assertTrue(cr.isSuccess());
Stream<FileSlice> fileSlice = fsView.getLatestFileSlices(partitionPath);
private List<Comparable[]> fileSlicesToCRList(Stream<FileSlice> fileSlice, String partitionPath) {
List<Comparable[]> rows = new ArrayList<>();
fileSlice.forEach(fs -> {
int idx = 0;
@@ -245,7 +284,14 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
.collect(Collectors.toList()).toString();
rows.add(row);
});
return rows;
}
/**(
* Test case for command 'show fsview latest'.
*/
@Test
public void testShowLatestFileSlices() throws IOException {
Function<Object, String> converterFunction =
entry -> NumericUtils.humanReadableByteCount((Double.parseDouble(entry.toString())));
Map<String, Function<Object, String>> fieldNameToConverterMap = new HashMap<>();
@@ -267,9 +313,32 @@ public class TestFileSystemViewCommand extends CLIFunctionalTestHarness {
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_BASE_UNSCHEDULED)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES_SCHEDULED)
.addTableHeaderField(HoodieTableHeaderFields.HEADER_DELTA_FILES_UNSCHEDULED);
String expected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, rows);
expected = removeNonWordAndStripSpace(expected);
String got = removeNonWordAndStripSpace(cr.getResult().toString());
assertEquals(expected, got);
// Test show with partition path '2016/03/15'
new TableCommand().connect(partitionedTablePath, null, false, 0, 0, 0);
CommandResult partitionedTableCR = shell().executeCommand("show fsview latest --partitionPath " + partitionPath);
assertTrue(partitionedTableCR.isSuccess());
Stream<FileSlice> partitionedFileSlice = partitionedFsView.getLatestFileSlices(partitionPath);
List<Comparable[]> partitionedRows = fileSlicesToCRList(partitionedFileSlice, partitionPath);
String partitionedExpected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, partitionedRows);
partitionedExpected = removeNonWordAndStripSpace(partitionedExpected);
String partitionedResults = removeNonWordAndStripSpace(partitionedTableCR.getResult().toString());
assertEquals(partitionedExpected, partitionedResults);
// Test show for non-partitioned table
new TableCommand().connect(nonpartitionedTablePath, null, false, 0, 0, 0);
CommandResult nonpartitionedTableCR = shell().executeCommand("show fsview latest");
assertTrue(nonpartitionedTableCR.isSuccess());
Stream<FileSlice> nonpartitionedFileSlice = nonpartitionedFsView.getLatestFileSlices("");
List<Comparable[]> nonpartitionedRows = fileSlicesToCRList(nonpartitionedFileSlice, "");
String nonpartitionedExpected = HoodiePrintHelper.print(header, fieldNameToConverterMap, "", false, -1, false, nonpartitionedRows);
nonpartitionedExpected = removeNonWordAndStripSpace(nonpartitionedExpected);
String nonpartitionedResults = removeNonWordAndStripSpace(nonpartitionedTableCR.getResult().toString());
assertEquals(nonpartitionedExpected, nonpartitionedResults);
}
}