[HUDI-509] Renaming code in sync with cWiki restructuring (#1212)
- Storage Type replaced with Table Type (remaining instances) - View types replaced with query types; - ReadOptimized view referred as Snapshot Query - TableFileSystemView sub interfaces renamed to BaseFileOnly and Slice Views - HoodieDataFile renamed to HoodieBaseFile - Hive Sync tool will register RO tables for MOR with a `_ro` suffix - Datasource/Deltastreamer options renamed accordingly - Support fallback to old config values as well, so migration is painless - Config for controlling _ro suffix addition - Renaming DataFile to BaseFile across DTOs, HoodieFileSlice and AbstractTableFileSystemView
This commit is contained in:
vinoth chandar
@@ -21,14 +21,14 @@ package org.apache.hudi.timeline.service;
|
||||
import org.apache.hudi.common.table.HoodieTimeline;
|
||||
import org.apache.hudi.common.table.SyncableFileSystemView;
|
||||
import org.apache.hudi.common.table.timeline.dto.CompactionOpDTO;
|
||||
import org.apache.hudi.common.table.timeline.dto.DataFileDTO;
|
||||
import org.apache.hudi.common.table.timeline.dto.BaseFileDTO;
|
||||
import org.apache.hudi.common.table.timeline.dto.FileGroupDTO;
|
||||
import org.apache.hudi.common.table.timeline.dto.FileSliceDTO;
|
||||
import org.apache.hudi.common.table.timeline.dto.InstantDTO;
|
||||
import org.apache.hudi.common.table.timeline.dto.TimelineDTO;
|
||||
import org.apache.hudi.common.table.view.FileSystemViewManager;
|
||||
import org.apache.hudi.common.table.view.RemoteHoodieTableFileSystemView;
|
||||
import org.apache.hudi.timeline.service.handlers.DataFileHandler;
|
||||
import org.apache.hudi.timeline.service.handlers.BaseFileHandler;
|
||||
import org.apache.hudi.timeline.service.handlers.FileSliceHandler;
|
||||
import org.apache.hudi.timeline.service.handlers.TimelineHandler;
|
||||
|
||||
@@ -60,14 +60,14 @@ public class FileSystemViewHandler {
|
||||
private final Javalin app;
|
||||
private final TimelineHandler instantHandler;
|
||||
private final FileSliceHandler sliceHandler;
|
||||
private final DataFileHandler dataFileHandler;
|
||||
private final BaseFileHandler dataFileHandler;
|
||||
|
||||
public FileSystemViewHandler(Javalin app, Configuration conf, FileSystemViewManager viewManager) throws IOException {
|
||||
this.viewManager = viewManager;
|
||||
this.app = app;
|
||||
this.instantHandler = new TimelineHandler(conf, viewManager);
|
||||
this.sliceHandler = new FileSliceHandler(conf, viewManager);
|
||||
this.dataFileHandler = new DataFileHandler(conf, viewManager);
|
||||
this.dataFileHandler = new BaseFileHandler(conf, viewManager);
|
||||
}
|
||||
|
||||
public void register() {
|
||||
@@ -160,14 +160,14 @@ public class FileSystemViewHandler {
|
||||
*/
|
||||
private void registerDataFilesAPI() {
|
||||
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_DATA_FILES_URL, new ViewHandler(ctx -> {
|
||||
List<DataFileDTO> dtos = dataFileHandler.getLatestDataFiles(
|
||||
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFiles(
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(),
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow());
|
||||
writeValueAsString(ctx, dtos);
|
||||
}, true));
|
||||
|
||||
app.get(RemoteHoodieTableFileSystemView.LATEST_PARTITION_DATA_FILE_URL, new ViewHandler(ctx -> {
|
||||
List<DataFileDTO> dtos = dataFileHandler.getLatestDataFile(
|
||||
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFile(
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(),
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(),
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.FILEID_PARAM).getOrThrow());
|
||||
@@ -175,13 +175,13 @@ public class FileSystemViewHandler {
|
||||
}, true));
|
||||
|
||||
app.get(RemoteHoodieTableFileSystemView.LATEST_ALL_DATA_FILES, new ViewHandler(ctx -> {
|
||||
List<DataFileDTO> dtos = dataFileHandler
|
||||
List<BaseFileDTO> dtos = dataFileHandler
|
||||
.getLatestDataFiles(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow());
|
||||
writeValueAsString(ctx, dtos);
|
||||
}, true));
|
||||
|
||||
app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILES_BEFORE_ON_INSTANT_URL, new ViewHandler(ctx -> {
|
||||
List<DataFileDTO> dtos = dataFileHandler.getLatestDataFilesBeforeOrOn(
|
||||
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFilesBeforeOrOn(
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(),
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(),
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.MAX_INSTANT_PARAM).getOrThrow());
|
||||
@@ -189,7 +189,7 @@ public class FileSystemViewHandler {
|
||||
}, true));
|
||||
|
||||
app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILE_ON_INSTANT_URL, new ViewHandler(ctx -> {
|
||||
List<DataFileDTO> dtos = dataFileHandler.getLatestDataFileOn(
|
||||
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFileOn(
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(),
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow(),
|
||||
ctx.queryParam(RemoteHoodieTableFileSystemView.INSTANT_PARAM),
|
||||
@@ -198,14 +198,14 @@ public class FileSystemViewHandler {
|
||||
}, true));
|
||||
|
||||
app.get(RemoteHoodieTableFileSystemView.ALL_DATA_FILES, new ViewHandler(ctx -> {
|
||||
List<DataFileDTO> dtos = dataFileHandler.getAllDataFiles(
|
||||
List<BaseFileDTO> dtos = dataFileHandler.getAllDataFiles(
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(),
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.PARTITION_PARAM).getOrThrow());
|
||||
writeValueAsString(ctx, dtos);
|
||||
}, true));
|
||||
|
||||
app.get(RemoteHoodieTableFileSystemView.LATEST_DATA_FILES_RANGE_INSTANT_URL, new ViewHandler(ctx -> {
|
||||
List<DataFileDTO> dtos = dataFileHandler.getLatestDataFilesInRange(
|
||||
List<BaseFileDTO> dtos = dataFileHandler.getLatestDataFilesInRange(
|
||||
ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.BASEPATH_PARAM).getOrThrow(), Arrays
|
||||
.asList(ctx.validatedQueryParam(RemoteHoodieTableFileSystemView.INSTANTS_PARAM).getOrThrow().split(",")));
|
||||
writeValueAsString(ctx, dtos);
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
|
||||
package org.apache.hudi.timeline.service.handlers;
|
||||
|
||||
import org.apache.hudi.common.table.timeline.dto.DataFileDTO;
|
||||
import org.apache.hudi.common.table.timeline.dto.BaseFileDTO;
|
||||
import org.apache.hudi.common.table.view.FileSystemViewManager;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
@@ -30,49 +30,49 @@ import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* REST Handler servicing data-file requests.
|
||||
* REST Handler servicing base-file requests.
|
||||
*/
|
||||
public class DataFileHandler extends Handler {
|
||||
public class BaseFileHandler extends Handler {
|
||||
|
||||
public DataFileHandler(Configuration conf, FileSystemViewManager viewManager) throws IOException {
|
||||
public BaseFileHandler(Configuration conf, FileSystemViewManager viewManager) throws IOException {
|
||||
super(conf, viewManager);
|
||||
}
|
||||
|
||||
public List<DataFileDTO> getLatestDataFiles(String basePath, String partitionPath) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestDataFiles(partitionPath)
|
||||
.map(DataFileDTO::fromHoodieDataFile).collect(Collectors.toList());
|
||||
public List<BaseFileDTO> getLatestDataFiles(String basePath, String partitionPath) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestBaseFiles(partitionPath)
|
||||
.map(BaseFileDTO::fromHoodieBaseFile).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<DataFileDTO> getLatestDataFile(String basePath, String partitionPath, String fileId) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestDataFile(partitionPath, fileId)
|
||||
.map(DataFileDTO::fromHoodieDataFile).map(Arrays::asList).orElse(new ArrayList<>());
|
||||
public List<BaseFileDTO> getLatestDataFile(String basePath, String partitionPath, String fileId) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestBaseFile(partitionPath, fileId)
|
||||
.map(BaseFileDTO::fromHoodieBaseFile).map(Arrays::asList).orElse(new ArrayList<>());
|
||||
}
|
||||
|
||||
public List<DataFileDTO> getLatestDataFiles(String basePath) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestDataFiles().map(DataFileDTO::fromHoodieDataFile)
|
||||
public List<BaseFileDTO> getLatestDataFiles(String basePath) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestBaseFiles().map(BaseFileDTO::fromHoodieBaseFile)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<DataFileDTO> getLatestDataFilesBeforeOrOn(String basePath, String partitionPath, String maxInstantTime) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestDataFilesBeforeOrOn(partitionPath, maxInstantTime)
|
||||
.map(DataFileDTO::fromHoodieDataFile).collect(Collectors.toList());
|
||||
public List<BaseFileDTO> getLatestDataFilesBeforeOrOn(String basePath, String partitionPath, String maxInstantTime) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestBaseFilesBeforeOrOn(partitionPath, maxInstantTime)
|
||||
.map(BaseFileDTO::fromHoodieBaseFile).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<DataFileDTO> getLatestDataFileOn(String basePath, String partitionPath, String instantTime,
|
||||
String fileId) {
|
||||
List<DataFileDTO> result = new ArrayList<>();
|
||||
viewManager.getFileSystemView(basePath).getDataFileOn(partitionPath, instantTime, fileId)
|
||||
.map(DataFileDTO::fromHoodieDataFile).ifPresent(result::add);
|
||||
public List<BaseFileDTO> getLatestDataFileOn(String basePath, String partitionPath, String instantTime,
|
||||
String fileId) {
|
||||
List<BaseFileDTO> result = new ArrayList<>();
|
||||
viewManager.getFileSystemView(basePath).getBaseFileOn(partitionPath, instantTime, fileId)
|
||||
.map(BaseFileDTO::fromHoodieBaseFile).ifPresent(result::add);
|
||||
return result;
|
||||
}
|
||||
|
||||
public List<DataFileDTO> getLatestDataFilesInRange(String basePath, List<String> instants) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestDataFilesInRange(instants)
|
||||
.map(DataFileDTO::fromHoodieDataFile).collect(Collectors.toList());
|
||||
public List<BaseFileDTO> getLatestDataFilesInRange(String basePath, List<String> instants) {
|
||||
return viewManager.getFileSystemView(basePath).getLatestBaseFilesInRange(instants)
|
||||
.map(BaseFileDTO::fromHoodieBaseFile).collect(Collectors.toList());
|
||||
}
|
||||
|
||||
public List<DataFileDTO> getAllDataFiles(String basePath, String partitionPath) {
|
||||
return viewManager.getFileSystemView(basePath).getAllDataFiles(partitionPath).map(DataFileDTO::fromHoodieDataFile)
|
||||
public List<BaseFileDTO> getAllDataFiles(String basePath, String partitionPath) {
|
||||
return viewManager.getFileSystemView(basePath).getAllBaseFiles(partitionPath).map(BaseFileDTO::fromHoodieBaseFile)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user