lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(web): 增加HDFS文件管理器

Browse Source 
在页面直接查看HDFS文件和目录，还可以查看和下载
This commit is contained in:
v-zhangjc9
2024-04-28 16:28:48 +08:00
parent 9913943a27
commit 572a1dab9f
11 changed files with 581 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
.idea/httpRequests/http-requests-log.http generated
View File

@@ -1,3 +1,41 @@
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
Cookie: JSESSIONID=C23877E9843F4E9C87FC2787EC5EA701
Accept-Encoding: br,deflate,gzip,x-gzip
<> 2024-04-26T172547.200.json
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@http:/b12s8.hdp.dc:33681/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
Accept-Encoding: br,deflate,gzip,x-gzip
<> 2024-04-26T172511.503.html
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:30943/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr/.hoodie
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
Cookie: JSESSIONID=C23877E9843F4E9C87FC2787EC5EA701
Accept-Encoding: br,deflate,gzip,x-gzip
<> 2024-04-26T162856.200.json
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:30943/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
Accept-Encoding: br,deflate,gzip,x-gzip
<> 2024-04-26T162825.200.json
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
@@ -375,35 +413,3 @@ Accept-Encoding: br,deflate,gzip,x-gzip
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
Accept-Encoding: br,deflate,gzip,x-gzip
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
Accept-Encoding: br,deflate,gzip,x-gzip
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
Accept-Encoding: br,deflate,gzip,x-gzip
###
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
Connection: Keep-Alive
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
Accept-Encoding: br,deflate,gzip,x-gzip
###

2
service-cli/service-cli-runner/src/main/resources/application-b12.yml
View File

@@ -63,7 +63,7 @@ deploy:
service-launcher-a4:
replicas: 6
service-launcher-b12:
replicas: 6
replicas: 15
service-info-query:
replicas: 10
service-yarn-query:

92
service-configuration/src/main/java/com/lanyuanxiaoyao/service/configuration/entity/hudi/HPath.java Normal file
View File

@@ -0,0 +1,92 @@
package com.lanyuanxiaoyao.service.configuration.entity.hudi;
/**
* HDFS文件
*
* @author lanyuanxiaoyao
* @date 2024-04-26
*/
public class HPath {
private String name;
private String path;
private Boolean file;
private Boolean folder;
private Long size;
private String group;
private String owner;
private String permission;
private Integer replication;
private Long modifyTime;
public HPath() {
}
public HPath(String name, String path, Boolean file, Boolean folder, Long size, String group, String owner, String permission, Integer replication, Long modifyTime) {
this.name = name;
this.path = path;
this.file = file;
this.folder = folder;
this.size = size;
this.group = group;
this.owner = owner;
this.permission = permission;
this.replication = replication;
this.modifyTime = modifyTime;
}
public String getName() {
return name;
}
public String getPath() {
return path;
}
public Boolean getFile() {
return file;
}
public Boolean getFolder() {
return folder;
}
public Long getSize() {
return size;
}
public String getGroup() {
return group;
}
public String getOwner() {
return owner;
}
public String getPermission() {
return permission;
}
public Integer getReplication() {
return replication;
}
public Long getModifyTime() {
return modifyTime;
}
@Override
public String toString() {
return "HPath{" +
"name='" + name + '\'' +
", path='" + path + '\'' +
", file=" + file +
", folder=" + folder +
", size=" + size +
", group='" + group + '\'' +
", owner='" + owner + '\'' +
", permission='" + permission + '\'' +
", replication=" + replication +
", modifyTime=" + modifyTime +
'}';
}
}

14
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/HudiService.java
View File

@@ -4,10 +4,12 @@ import com.dtflys.forest.annotation.BaseRequest;
import com.dtflys.forest.annotation.Get;
import com.dtflys.forest.annotation.Query;
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiRollbackPlan;
import java.io.InputStream;
import java.util.Map;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.map.ImmutableMap;
@@ -64,4 +66,16 @@ public interface HudiService {
@Get("/hdfs/exists_path")
Boolean existsPath(@Query("hdfs") String hdfs);
@Get("/hdfs/get")
HPath get(@Query("root") String root);
@Get("/hdfs/list")
ImmutableList<HPath> list(@Query("root") String root);
@Get("/hdfs/read")
String read(@Query("root") String root);
@Get("/hdfs/download")
InputStream download(@Query("root") String root);
}

24
service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/controller/HdfsController.java
View File

@@ -1,6 +1,10 @@
package com.lanyuanxiaoyao.service.hudi.controller;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
import com.lanyuanxiaoyao.service.hudi.service.HdfsService;
import java.io.IOException;
import javax.servlet.http.HttpServletResponse;
import org.eclipse.collections.api.list.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.GetMapping;
@@ -42,4 +46,24 @@ public class HdfsController {
public Boolean existsPath(@RequestParam("hdfs") String hdfs) {
return hdfsService.existsPath(hdfs);
}
@GetMapping("get")
public HPath get(@RequestParam("root")String root) throws IOException {
return hdfsService.get(root);
}
@GetMapping("list")
public ImmutableList<HPath> list(@RequestParam("root")String root) throws IOException {
return hdfsService.list(root);
}
@GetMapping("read")
public String read(@RequestParam("root")String root) throws IOException {
return hdfsService.read(root);
}
@GetMapping("download")
public void download(@RequestParam("root")String root, HttpServletResponse response) throws IOException {
hdfsService.download(root, response.getOutputStream());
}
}

81
service-hudi-query/src/main/java/com/lanyuanxiaoyao/service/hudi/service/HdfsService.java
View File

@@ -1,13 +1,21 @@
package com.lanyuanxiaoyao.service.hudi.service;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.ObjectUtil;
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
import com.lanyuanxiaoyao.service.forest.service.InfoService;
import java.io.IOException;
import java.io.OutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.list.ImmutableList;
import org.eclipse.collections.api.list.MutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.annotation.Cacheable;
@@ -58,10 +66,81 @@ public class HdfsService {
@Cacheable(value = "exists-path", sync = true)
public Boolean existsPath(String hdfs) {
try(FileSystem fileSystem = FileSystem.get(new Configuration())) {
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
return fileSystem.exists(new Path(hdfs));
} catch (IOException ignored) {
}
return false;
}
@Cacheable(value = "get-hpath", sync = true)
public HPath get(String root) throws IOException {
if (!existsPath(root)) {
throw new RuntimeException("File not found");
}
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
FileStatus status = fileSystem.getFileStatus(new Path(root));
return new HPath(
status.getPath().getName(),
status.getPath().toString(),
status.isFile(),
status.isDirectory(),
status.getLen(),
status.getGroup(),
status.getOwner(),
status.getPermission().toString(),
(int) status.getReplication(),
status.getModificationTime()
);
}
}
@Cacheable(value = "list-hpath", sync = true)
public ImmutableList<HPath> list(String root) throws IOException {
if (!existsPath(root)) {
return Lists.immutable.empty();
}
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
MutableList<HPath> files = Lists.mutable.empty();
for (FileStatus status : fileSystem.listStatus(new Path(root))) {
files.add(new HPath(
status.getPath().getName(),
status.getPath().toString(),
status.isFile(),
status.isDirectory(),
status.getLen(),
status.getGroup(),
status.getOwner(),
status.getPermission().toString(),
(int) status.getReplication(),
status.getModificationTime()
));
}
return files.toImmutable();
}
}
@Cacheable(value = "read-hpath", sync = true)
public String read(String root) throws IOException {
if (!existsPath(root)) {
return "";
}
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
try (FSDataInputStream stream = fileSystem.open(new Path(root))) {
return IoUtil.readUtf8(stream);
}
}
}
@SuppressWarnings("SpringCacheableMethodCallsInspection")
public void download(String root, OutputStream outputStream) throws IOException {
if (!existsPath(root)) {
return;
}
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
try (FSDataInputStream stream = fileSystem.open(new Path(root))) {
IoUtil.copy(stream, outputStream);
}
}
}
}

50
service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/HudiController.java
View File

@@ -1,8 +1,10 @@
package com.lanyuanxiaoyao.service.web.controller;
import cn.hutool.core.io.IoUtil;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
@@ -10,15 +12,21 @@ import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiRollbackPlan;
import com.lanyuanxiaoyao.service.forest.service.HudiService;
import com.lanyuanxiaoyao.service.web.controller.base.AmisCrudResponse;
import com.lanyuanxiaoyao.service.web.controller.base.AmisDetailResponse;
import com.lanyuanxiaoyao.service.web.controller.base.AmisMapResponse;
import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse;
import com.lanyuanxiaoyao.service.web.controller.base.BaseController;
import com.lanyuanxiaoyao.service.web.entity.HPathChildrenVO;
import com.lanyuanxiaoyao.service.web.entity.HPathVO;
import java.io.InputStream;
import java.util.List;
import java.util.function.Function;
import javax.servlet.http.HttpServletResponse;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.map.ImmutableMap;
import org.eclipse.collections.api.map.MutableMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
@@ -163,4 +171,46 @@ public class HudiController extends BaseController {
}
throw new Exception("Flink job id and alias or hdfs cannot be blank");
}
@GetMapping("hdfs_list")
public AmisCrudResponse hdfsList(@RequestParam("root") String root) throws Exception {
if (StrUtil.isBlank(root)) {
throw new Exception("Root path cannot be blank");
}
return AmisCrudResponse.responseCrudData(hudiService.list(root).collect(HPathVO::new));
}
@GetMapping("hdfs_list_children")
public AmisResponse<HPathChildrenVO> hdfsListChildren(@RequestParam("root") String root) throws Exception {
if (StrUtil.isBlank(root)) {
throw new Exception("Root path cannot be blank");
}
return AmisResponse.responseSuccess(new HPathChildrenVO(hudiService.list(root).collect(HPathVO::new)));
}
@GetMapping("hdfs_read")
public AmisMapResponse hdfsRead(@RequestParam("root") String root) throws Exception {
if (StrUtil.isBlank(root)) {
throw new Exception("Root path cannot be blank");
}
return AmisResponse.responseMapData()
.setData("text", hudiService.read(root));
}
@GetMapping(value = "hdfs_download")
public void hdfsDownload(@RequestParam("root") String root, HttpServletResponse response) throws Exception {
if (StrUtil.isBlank(root)) {
throw new Exception("Root path cannot be blank");
}
HPath hPath = hudiService.get(root);
if (hPath.getFolder()) {
throw new Exception("Folder cannot be downloaded");
}
try (InputStream inputStream = hudiService.download(root)) {
response.setHeader("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE);
response.setHeader("Content-Disposition", StrUtil.format("attachment;filename={}", hPath.getName()));
response.setHeader("Content-Length", hPath.getSize().toString());
IoUtil.copy(inputStream, response.getOutputStream());
}
}
}

29
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathChildrenVO.java Normal file
View File

@@ -0,0 +1,29 @@
package com.lanyuanxiaoyao.service.web.entity;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
import org.eclipse.collections.api.list.ImmutableList;
/**
* HPath对象的有children版本
*
* @author lanyuanxiaoyao
* @date 2024-04-26
*/
public class HPathChildrenVO {
private final ImmutableList<HPathVO> children;
public HPathChildrenVO(ImmutableList<HPathVO> children) {
this.children = children;
}
public ImmutableList<HPathVO> getChildren() {
return children;
}
@Override
public String toString() {
return "AmisCrudChildrenVO{" +
"children=" + children +
'}';
}
}

80
service-web/src/main/java/com/lanyuanxiaoyao/service/web/entity/HPathVO.java Normal file
View File

@@ -0,0 +1,80 @@
package com.lanyuanxiaoyao.service.web.entity;
import cn.hutool.core.io.unit.DataSizeUtil;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
/**
* @author lanyuanxiaoyao
* @date 2024-04-26
*/
public class HPathVO {
@JsonIgnore
private final HPath hPath;
private final Boolean defer;
private final String sizeText;
public HPathVO(HPath hPath) {
this.hPath = hPath;
this.defer = hPath.getFolder();
this.sizeText = DataSizeUtil.format(hPath.getSize());
}
public String getName() {
return hPath.getName();
}
public String getPath() {
return hPath.getPath();
}
public Boolean getFile() {
return hPath.getFile();
}
public Boolean getFolder() {
return hPath.getFolder();
}
public Long getSize() {
return hPath.getSize();
}
public String getGroup() {
return hPath.getGroup();
}
public String getOwner() {
return hPath.getOwner();
}
public String getPermission() {
return hPath.getPermission();
}
public Integer getReplication() {
return hPath.getReplication();
}
public Long getModifyTime() {
return hPath.getModifyTime();
}
public Boolean getDefer() {
return defer;
}
public String getSizeText() {
return sizeText;
}
@Override
public String toString() {
return "HPathVO{" +
"hPath=" + hPath +
", defer=" + defer +
", sizeText='" + sizeText + '\'' +
'}';
}
}

170
service-web/src/main/resources/static/components/tool-tab.js
View File

@@ -55,6 +55,176 @@ function toolTab() {
}
]
},
{
type: 'form',
title: 'HDFS文件管理器',
actions: [
{
label: '直接下载',
type: 'action',
onEvent: {
click: {
actions: [
{
actionType: 'custom',
script: (context, action, event) => {
let downloadUrl = `${event.data.base}/hudi/hdfs_download?root=${encodeURI(event.data.hdfs)}`
window.open(downloadUrl, '_blank')
},
}
]
}
}
},
{
type: 'submit',
label: '查看',
actionType: 'dialog',
dialog: {
title: {
type: 'tpl',
tpl: '${hdfs}'
},
...readOnlyDialogOptions(),
size: 'xl',
body: {
type: 'crud',
api: {
method: 'get',
url: '${base}/hudi/hdfs_list',
data: {
root: '${hdfs|default:undefined}',
},
},
deferApi: {
method: 'get',
url: '${base}/hudi/hdfs_list_children',
data: {
root: '${path|default:undefined}',
},
},
...crudCommonOptions(),
perPage: 10,
headerToolbar: [
"reload",
paginationCommonOptions(undefined, 10),
],
footerToolbar: [
paginationCommonOptions(undefined, 10),
],
columns: [
{
name: 'name',
label: '文件名',
className: 'font-mono',
},
{
name: 'group',
label: '组',
width: 70,
},
{
name: 'owner',
label: '用户',
width: 70,
},
{
label: '大小',
width: 90,
type: 'tpl',
tpl: "${sizeText}",
},
{
label: '修改时间',
width: 140,
type: 'tpl',
tpl: "${DATETOSTR(DATE(modifyTime), 'YYYY-MM-DD HH:mm:ss')}",
},
{
type: 'operation',
label: '操作',
width: 160,
fixed: 'right',
buttons: [
{
label: '完整路径',
type: 'action',
level: 'link',
size: 'xs',
actionType: 'copy',
content: '${path}',
tooltip: '复制 ${path}',
},
{
disabledOn: 'folder || size > 1048576',
label: '查看',
type: 'action',
level: 'link',
size: 'xs',
actionType: 'dialog',
dialog: {
title: {
type: 'tpl',
tpl: '文件内容：${path}'
},
size: 'md',
...readOnlyDialogOptions(),
body: {
type: 'service',
api: {
method: 'get',
url: '${base}/hudi/hdfs_read',
data: {
root: '${path|default:undefined}'
}
},
body: {
type: 'textarea',
name: 'text',
readOnly: true,
}
}
}
},
{
disabledOn: 'folder',
label: '下载',
type: 'action',
level: 'link',
size: 'xs',
onEvent: {
click: {
actions: [
{
actionType: 'custom',
script: (context, action, event) => {
let downloadUrl = `${event.data.base}/hudi/hdfs_download?root=${encodeURI(event.data.path)}`
window.open(downloadUrl, '_blank')
},
}
]
}
}
},
]
}
]
}
}
}
],
body: [
{
type: 'input-text',
name: 'hdfs',
label: 'HDFS根路经',
required: true,
clearable: true,
description: '输入表HDFS路径',
autoComplete: '${base}/table/all_hdfs?key=$term',
},
]
},
{
type: 'form',
title: '查询时间线',

3
test/test.http
View File

@@ -96,3 +96,6 @@ GET {{exporter-url}}/exporter/un_running_flink_job
### Pulsar backlog
GET http://{{username}}:{{password}}@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
### Test HDFS list
GET http://{{username}}:{{password}}@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr