diff --git a/.idea/httpRequests/http-requests-log.http b/.idea/httpRequests/http-requests-log.http index ecb8171..067f43a 100644 --- a/.idea/httpRequests/http-requests-log.http +++ b/.idea/httpRequests/http-requests-log.http @@ -1,3 +1,41 @@ +GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr +Connection: Keep-Alive +User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10) +Cookie: JSESSIONID=C23877E9843F4E9C87FC2787EC5EA701 +Accept-Encoding: br,deflate,gzip,x-gzip + +<> 2024-04-26T172547.200.json + +### + +GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@http:/b12s8.hdp.dc:33681/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr +Connection: Keep-Alive +User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10) +Accept-Encoding: br,deflate,gzip,x-gzip + +<> 2024-04-26T172511.503.html + +### + +GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:30943/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr/.hoodie +Connection: Keep-Alive +User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10) +Cookie: JSESSIONID=C23877E9843F4E9C87FC2787EC5EA701 +Accept-Encoding: br,deflate,gzip,x-gzip + +<> 2024-04-26T162856.200.json + +### + +GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:30943/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr +Connection: Keep-Alive +User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10) +Accept-Encoding: br,deflate,gzip,x-gzip + +<> 2024-04-26T162825.200.json + +### + GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425 Connection: Keep-Alive User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10) @@ -375,35 +413,3 @@ Accept-Encoding: br,deflate,gzip,x-gzip ### -GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre -Connection: Keep-Alive -User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9) -Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877 -Accept-Encoding: br,deflate,gzip,x-gzip - -### - -GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre -Connection: Keep-Alive -User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9) -Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877 -Accept-Encoding: br,deflate,gzip,x-gzip - -### - -GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre -Connection: Keep-Alive -User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9) -Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877 -Accept-Encoding: br,deflate,gzip,x-gzip - -### - -GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre -Connection: Keep-Alive -User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9) -Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877 -Accept-Encoding: br,deflate,gzip,x-gzip - -### - diff --git a/service-cli/service-cli-runner/src/main/resources/application-b12.yml b/service-cli/service-cli-runner/src/main/resources/application-b12.yml index 82c2389..6bc2ff2 100644 --- a/service-cli/service-cli-runner/src/main/resources/application-b12.yml +++ b/service-cli/service-cli-runner/src/main/resources/application-b12.yml @@ -63,7 +63,7 @@ deploy: service-launcher-a4: replicas: 6 service-launcher-b12: - replicas: 6 + replicas: 15 service-info-query: replicas: 10 service-yarn-query: diff --git a/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/hudi/HPath.java b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/hudi/HPath.java new file mode 100644 index 0000000..535a65e --- /dev/null +++ b/service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/hudi/HPath.java @@ -0,0 +1,92 @@ +package com.lanyuanxiaoyao.service.configuration.entity.hudi; + +/** + * HDFS文件 + * + * @author lanyuanxiaoyao + * @date 2024-04-26 + */ +public class HPath { + private String name; + private String path; + private Boolean file; + private Boolean folder; + private Long size; + private String group; + private String owner; + private String permission; + private Integer replication; + private Long modifyTime; + + public HPath() { + } + + public HPath(String name, String path, Boolean file, Boolean folder, Long size, String group, String owner, String permission, Integer replication, Long modifyTime) { + this.name = name; + this.path = path; + this.file = file; + this.folder = folder; + this.size = size; + this.group = group; + this.owner = owner; + this.permission = permission; + this.replication = replication; + this.modifyTime = modifyTime; + } + + public String getName() { + return name; + } + + public String getPath() { + return path; + } + + public Boolean getFile() { + return file; + } + + public Boolean getFolder() { + return folder; + } + + public Long getSize() { + return size; + } + + public String getGroup() { + return group; + } + + public String getOwner() { + return owner; + } + + public String getPermission() { + return permission; + } + + public Integer getReplication() { + return replication; + } + + public Long getModifyTime() { + return modifyTime; + } + + @Override + public String toString() { + return "HPath{" + + "name='" + name + '\'' + + ", path='" + path + '\'' + + ", file=" + file + + ", folder=" + folder + + ", size=" + size + + ", group='" + group + '\'' + + ", owner='" + owner + '\'' + + ", permission='" + permission + '\'' + + ", replication=" + replication + + ", modifyTime=" + modifyTime + + '}'; + } +} diff --git a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/HudiService.java b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/HudiService.java index c8d8240..9e2bb6e 100644 --- a/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/HudiService.java +++ b/service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/HudiService.java @@ -4,10 +4,12 @@ import com.dtflys.forest.annotation.BaseRequest; import com.dtflys.forest.annotation.Get; import com.dtflys.forest.annotation.Query; import com.lanyuanxiaoyao.service.configuration.entity.PageResponse; +import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath; import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan; import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan; import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant; import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiRollbackPlan; +import java.io.InputStream; import java.util.Map; import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.map.ImmutableMap; @@ -64,4 +66,16 @@ public interface HudiService { @Get("/hdfs/exists_path") Boolean existsPath(@Query("hdfs") String hdfs); + + @Get("/hdfs/get") + HPath get(@Query("root") String root); + + @Get("/hdfs/list") + ImmutableList list(@Query("root") String root); + + @Get("/hdfs/read") + String read(@Query("root") String root); + + @Get("/hdfs/download") + InputStream download(@Query("root") String root); } diff --git a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/HdfsController.java b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/HdfsController.java index 402f4e6..cf327a9 100644 --- a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/HdfsController.java +++ b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/HdfsController.java @@ -1,6 +1,10 @@ package com.lanyuanxiaoyao.service.hudi.controller; +import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath; import com.lanyuanxiaoyao.service.hudi.service.HdfsService; +import java.io.IOException; +import javax.servlet.http.HttpServletResponse; +import org.eclipse.collections.api.list.ImmutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.web.bind.annotation.GetMapping; @@ -42,4 +46,24 @@ public class HdfsController { public Boolean existsPath(@RequestParam("hdfs") String hdfs) { return hdfsService.existsPath(hdfs); } + + @GetMapping("get") + public HPath get(@RequestParam("root")String root) throws IOException { + return hdfsService.get(root); + } + + @GetMapping("list") + public ImmutableList list(@RequestParam("root")String root) throws IOException { + return hdfsService.list(root); + } + + @GetMapping("read") + public String read(@RequestParam("root")String root) throws IOException { + return hdfsService.read(root); + } + + @GetMapping("download") + public void download(@RequestParam("root")String root, HttpServletResponse response) throws IOException { + hdfsService.download(root, response.getOutputStream()); + } } diff --git a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/HdfsService.java b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/HdfsService.java index b4fa5ab..eb86370 100644 --- a/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/HdfsService.java +++ b/service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/HdfsService.java @@ -1,13 +1,21 @@ package com.lanyuanxiaoyao.service.hudi.service; +import cn.hutool.core.io.IoUtil; import cn.hutool.core.util.ObjectUtil; import com.lanyuanxiaoyao.service.common.entity.TableMeta; +import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath; import com.lanyuanxiaoyao.service.forest.service.InfoService; import java.io.IOException; +import java.io.OutputStream; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.list.MutableList; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.annotation.Cacheable; @@ -58,10 +66,81 @@ public class HdfsService { @Cacheable(value = "exists-path", sync = true) public Boolean existsPath(String hdfs) { - try(FileSystem fileSystem = FileSystem.get(new Configuration())) { + try (FileSystem fileSystem = FileSystem.get(new Configuration())) { return fileSystem.exists(new Path(hdfs)); } catch (IOException ignored) { } return false; } + + @Cacheable(value = "get-hpath", sync = true) + public HPath get(String root) throws IOException { + if (!existsPath(root)) { + throw new RuntimeException("File not found"); + } + try (FileSystem fileSystem = FileSystem.get(new Configuration())) { + FileStatus status = fileSystem.getFileStatus(new Path(root)); + return new HPath( + status.getPath().getName(), + status.getPath().toString(), + status.isFile(), + status.isDirectory(), + status.getLen(), + status.getGroup(), + status.getOwner(), + status.getPermission().toString(), + (int) status.getReplication(), + status.getModificationTime() + ); + } + } + + @Cacheable(value = "list-hpath", sync = true) + public ImmutableList list(String root) throws IOException { + if (!existsPath(root)) { + return Lists.immutable.empty(); + } + try (FileSystem fileSystem = FileSystem.get(new Configuration())) { + MutableList files = Lists.mutable.empty(); + for (FileStatus status : fileSystem.listStatus(new Path(root))) { + files.add(new HPath( + status.getPath().getName(), + status.getPath().toString(), + status.isFile(), + status.isDirectory(), + status.getLen(), + status.getGroup(), + status.getOwner(), + status.getPermission().toString(), + (int) status.getReplication(), + status.getModificationTime() + )); + } + return files.toImmutable(); + } + } + + @Cacheable(value = "read-hpath", sync = true) + public String read(String root) throws IOException { + if (!existsPath(root)) { + return ""; + } + try (FileSystem fileSystem = FileSystem.get(new Configuration())) { + try (FSDataInputStream stream = fileSystem.open(new Path(root))) { + return IoUtil.readUtf8(stream); + } + } + } + + @SuppressWarnings("SpringCacheableMethodCallsInspection") + public void download(String root, OutputStream outputStream) throws IOException { + if (!existsPath(root)) { + return; + } + try (FileSystem fileSystem = FileSystem.get(new Configuration())) { + try (FSDataInputStream stream = fileSystem.open(new Path(root))) { + IoUtil.copy(stream, outputStream); + } + } + } } diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java index aa2967c..c028765 100644 --- a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java @@ -1,8 +1,10 @@ package com.lanyuanxiaoyao.service.web.controller; +import cn.hutool.core.io.IoUtil; import cn.hutool.core.util.ObjectUtil; import cn.hutool.core.util.StrUtil; import com.lanyuanxiaoyao.service.configuration.entity.PageResponse; +import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath; import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan; import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan; import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant; @@ -10,15 +12,21 @@ import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiRollbackPlan; import com.lanyuanxiaoyao.service.forest.service.HudiService; import com.lanyuanxiaoyao.service.web.controller.base.AmisCrudResponse; import com.lanyuanxiaoyao.service.web.controller.base.AmisDetailResponse; +import com.lanyuanxiaoyao.service.web.controller.base.AmisMapResponse; import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse; import com.lanyuanxiaoyao.service.web.controller.base.BaseController; +import com.lanyuanxiaoyao.service.web.entity.HPathChildrenVO; +import com.lanyuanxiaoyao.service.web.entity.HPathVO; +import java.io.InputStream; import java.util.List; import java.util.function.Function; +import javax.servlet.http.HttpServletResponse; import org.eclipse.collections.api.factory.Maps; import org.eclipse.collections.api.map.ImmutableMap; import org.eclipse.collections.api.map.MutableMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.MediaType; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; @@ -163,4 +171,46 @@ public class HudiController extends BaseController { } throw new Exception("Flink job id and alias or hdfs cannot be blank"); } + + @GetMapping("hdfs_list") + public AmisCrudResponse hdfsList(@RequestParam("root") String root) throws Exception { + if (StrUtil.isBlank(root)) { + throw new Exception("Root path cannot be blank"); + } + return AmisCrudResponse.responseCrudData(hudiService.list(root).collect(HPathVO::new)); + } + + @GetMapping("hdfs_list_children") + public AmisResponse hdfsListChildren(@RequestParam("root") String root) throws Exception { + if (StrUtil.isBlank(root)) { + throw new Exception("Root path cannot be blank"); + } + return AmisResponse.responseSuccess(new HPathChildrenVO(hudiService.list(root).collect(HPathVO::new))); + } + + @GetMapping("hdfs_read") + public AmisMapResponse hdfsRead(@RequestParam("root") String root) throws Exception { + if (StrUtil.isBlank(root)) { + throw new Exception("Root path cannot be blank"); + } + return AmisResponse.responseMapData() + .setData("text", hudiService.read(root)); + } + + @GetMapping(value = "hdfs_download") + public void hdfsDownload(@RequestParam("root") String root, HttpServletResponse response) throws Exception { + if (StrUtil.isBlank(root)) { + throw new Exception("Root path cannot be blank"); + } + HPath hPath = hudiService.get(root); + if (hPath.getFolder()) { + throw new Exception("Folder cannot be downloaded"); + } + try (InputStream inputStream = hudiService.download(root)) { + response.setHeader("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE); + response.setHeader("Content-Disposition", StrUtil.format("attachment;filename={}", hPath.getName())); + response.setHeader("Content-Length", hPath.getSize().toString()); + IoUtil.copy(inputStream, response.getOutputStream()); + } + } } diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathChildrenVO.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathChildrenVO.java new file mode 100644 index 0000000..41cfe41 --- /dev/null +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathChildrenVO.java @@ -0,0 +1,29 @@ +package com.lanyuanxiaoyao.service.web.entity; + +import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath; +import org.eclipse.collections.api.list.ImmutableList; + +/** + * HPath对象的有children版本 + * + * @author lanyuanxiaoyao + * @date 2024-04-26 + */ +public class HPathChildrenVO { + private final ImmutableList children; + + public HPathChildrenVO(ImmutableList children) { + this.children = children; + } + + public ImmutableList getChildren() { + return children; + } + + @Override + public String toString() { + return "AmisCrudChildrenVO{" + + "children=" + children + + '}'; + } +} diff --git a/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathVO.java b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathVO.java new file mode 100644 index 0000000..9728fee --- /dev/null +++ b/service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathVO.java @@ -0,0 +1,80 @@ +package com.lanyuanxiaoyao.service.web.entity; + +import cn.hutool.core.io.unit.DataSizeUtil; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath; + +/** + * @author lanyuanxiaoyao + * @date 2024-04-26 + */ +public class HPathVO { + @JsonIgnore + private final HPath hPath; + + private final Boolean defer; + private final String sizeText; + + public HPathVO(HPath hPath) { + this.hPath = hPath; + this.defer = hPath.getFolder(); + this.sizeText = DataSizeUtil.format(hPath.getSize()); + } + + public String getName() { + return hPath.getName(); + } + + public String getPath() { + return hPath.getPath(); + } + + public Boolean getFile() { + return hPath.getFile(); + } + + public Boolean getFolder() { + return hPath.getFolder(); + } + + public Long getSize() { + return hPath.getSize(); + } + + public String getGroup() { + return hPath.getGroup(); + } + + public String getOwner() { + return hPath.getOwner(); + } + + public String getPermission() { + return hPath.getPermission(); + } + + public Integer getReplication() { + return hPath.getReplication(); + } + + public Long getModifyTime() { + return hPath.getModifyTime(); + } + + public Boolean getDefer() { + return defer; + } + + public String getSizeText() { + return sizeText; + } + + @Override + public String toString() { + return "HPathVO{" + + "hPath=" + hPath + + ", defer=" + defer + + ", sizeText='" + sizeText + '\'' + + '}'; + } +} diff --git a/service-web/src/main/resources/static/components/tool-tab.js b/service-web/src/main/resources/static/components/tool-tab.js index 3bbe827..b8c7f8c 100644 --- a/service-web/src/main/resources/static/components/tool-tab.js +++ b/service-web/src/main/resources/static/components/tool-tab.js @@ -55,6 +55,176 @@ function toolTab() { } ] }, + { + type: 'form', + title: 'HDFS文件管理器', + actions: [ + { + label: '直接下载', + type: 'action', + onEvent: { + click: { + actions: [ + { + actionType: 'custom', + script: (context, action, event) => { + let downloadUrl = `${event.data.base}/hudi/hdfs_download?root=${encodeURI(event.data.hdfs)}` + window.open(downloadUrl, '_blank') + }, + } + ] + } + } + }, + { + type: 'submit', + label: '查看', + actionType: 'dialog', + dialog: { + title: { + type: 'tpl', + tpl: '${hdfs}' + }, + ...readOnlyDialogOptions(), + size: 'xl', + body: { + type: 'crud', + api: { + method: 'get', + url: '${base}/hudi/hdfs_list', + data: { + root: '${hdfs|default:undefined}', + }, + }, + deferApi: { + method: 'get', + url: '${base}/hudi/hdfs_list_children', + data: { + root: '${path|default:undefined}', + }, + }, + ...crudCommonOptions(), + perPage: 10, + headerToolbar: [ + "reload", + paginationCommonOptions(undefined, 10), + ], + footerToolbar: [ + paginationCommonOptions(undefined, 10), + ], + columns: [ + { + name: 'name', + label: '文件名', + className: 'font-mono', + }, + { + name: 'group', + label: '组', + width: 70, + }, + { + name: 'owner', + label: '用户', + width: 70, + }, + { + label: '大小', + width: 90, + type: 'tpl', + tpl: "${sizeText}", + }, + { + label: '修改时间', + width: 140, + type: 'tpl', + tpl: "${DATETOSTR(DATE(modifyTime), 'YYYY-MM-DD HH:mm:ss')}", + }, + { + type: 'operation', + label: '操作', + width: 160, + fixed: 'right', + buttons: [ + { + label: '完整路径', + type: 'action', + level: 'link', + size: 'xs', + actionType: 'copy', + content: '${path}', + tooltip: '复制 ${path}', + }, + { + disabledOn: 'folder || size > 1048576', + label: '查看', + type: 'action', + level: 'link', + size: 'xs', + actionType: 'dialog', + dialog: { + title: { + type: 'tpl', + tpl: '文件内容：${path}' + }, + size: 'md', + ...readOnlyDialogOptions(), + body: { + type: 'service', + api: { + method: 'get', + url: '${base}/hudi/hdfs_read', + data: { + root: '${path|default:undefined}' + } + }, + body: { + type: 'textarea', + name: 'text', + readOnly: true, + } + } + } + }, + { + disabledOn: 'folder', + label: '下载', + type: 'action', + level: 'link', + size: 'xs', + onEvent: { + click: { + actions: [ + { + actionType: 'custom', + script: (context, action, event) => { + let downloadUrl = `${event.data.base}/hudi/hdfs_download?root=${encodeURI(event.data.path)}` + window.open(downloadUrl, '_blank') + }, + } + ] + } + } + }, + ] + } + ] + } + } + } + ], + body: [ + { + type: 'input-text', + name: 'hdfs', + label: 'HDFS根路经', + required: true, + clearable: true, + description: '输入表HDFS路径', + autoComplete: '${base}/table/all_hdfs?key=$term', + }, + ] + }, { type: 'form', title: '查询时间线', diff --git a/test/test.http b/test/test.http index da78a9e..fd44d78 100644 --- a/test/test.http +++ b/test/test.http @@ -96,3 +96,6 @@ GET {{exporter-url}}/exporter/un_running_flink_job ### Pulsar backlog GET http://{{username}}:{{password}}@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425 + +### Test HDFS list +GET http://{{username}}:{{password}}@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr