feat(web): 增加HDFS文件管理器
在页面直接查看HDFS文件和目录,还可以查看和下载
This commit is contained in:
70
.idea/httpRequests/http-requests-log.http
generated
70
.idea/httpRequests/http-requests-log.http
generated
@@ -1,3 +1,41 @@
|
|||||||
|
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
|
||||||
|
Connection: Keep-Alive
|
||||||
|
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
|
||||||
|
Cookie: JSESSIONID=C23877E9843F4E9C87FC2787EC5EA701
|
||||||
|
Accept-Encoding: br,deflate,gzip,x-gzip
|
||||||
|
|
||||||
|
<> 2024-04-26T172547.200.json
|
||||||
|
|
||||||
|
###
|
||||||
|
|
||||||
|
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@http:/b12s8.hdp.dc:33681/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
|
||||||
|
Connection: Keep-Alive
|
||||||
|
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
|
||||||
|
Accept-Encoding: br,deflate,gzip,x-gzip
|
||||||
|
|
||||||
|
<> 2024-04-26T172511.503.html
|
||||||
|
|
||||||
|
###
|
||||||
|
|
||||||
|
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:30943/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr/.hoodie
|
||||||
|
Connection: Keep-Alive
|
||||||
|
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
|
||||||
|
Cookie: JSESSIONID=C23877E9843F4E9C87FC2787EC5EA701
|
||||||
|
Accept-Encoding: br,deflate,gzip,x-gzip
|
||||||
|
|
||||||
|
<> 2024-04-26T162856.200.json
|
||||||
|
|
||||||
|
###
|
||||||
|
|
||||||
|
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s10.hdp.dc:30943/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
|
||||||
|
Connection: Keep-Alive
|
||||||
|
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
|
||||||
|
Accept-Encoding: br,deflate,gzip,x-gzip
|
||||||
|
|
||||||
|
<> 2024-04-26T162825.200.json
|
||||||
|
|
||||||
|
###
|
||||||
|
|
||||||
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
|
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
|
||||||
Connection: Keep-Alive
|
Connection: Keep-Alive
|
||||||
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
|
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.10)
|
||||||
@@ -375,35 +413,3 @@ Accept-Encoding: br,deflate,gzip,x-gzip
|
|||||||
|
|
||||||
###
|
###
|
||||||
|
|
||||||
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
|
|
||||||
Connection: Keep-Alive
|
|
||||||
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
|
|
||||||
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
|
|
||||||
Accept-Encoding: br,deflate,gzip,x-gzip
|
|
||||||
|
|
||||||
###
|
|
||||||
|
|
||||||
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
|
|
||||||
Connection: Keep-Alive
|
|
||||||
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
|
|
||||||
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
|
|
||||||
Accept-Encoding: br,deflate,gzip,x-gzip
|
|
||||||
|
|
||||||
###
|
|
||||||
|
|
||||||
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
|
|
||||||
Connection: Keep-Alive
|
|
||||||
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
|
|
||||||
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
|
|
||||||
Accept-Encoding: br,deflate,gzip,x-gzip
|
|
||||||
|
|
||||||
###
|
|
||||||
|
|
||||||
GET http://AxhEbscwsJDbYMH2:cYxg3b4PtWoVD5SjFayWxtnSVsjzRsg4@132.126.207.130:35690/hudi_services/queue/queue/clear?name=compaction-queue-pre
|
|
||||||
Connection: Keep-Alive
|
|
||||||
User-Agent: Apache-HttpClient/4.5.14 (Java/17.0.9)
|
|
||||||
Cookie: JSESSIONID=9AB8D98C10FACE15EA1CB758D79F8877
|
|
||||||
Accept-Encoding: br,deflate,gzip,x-gzip
|
|
||||||
|
|
||||||
###
|
|
||||||
|
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ deploy:
|
|||||||
service-launcher-a4:
|
service-launcher-a4:
|
||||||
replicas: 6
|
replicas: 6
|
||||||
service-launcher-b12:
|
service-launcher-b12:
|
||||||
replicas: 6
|
replicas: 15
|
||||||
service-info-query:
|
service-info-query:
|
||||||
replicas: 10
|
replicas: 10
|
||||||
service-yarn-query:
|
service-yarn-query:
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package com.lanyuanxiaoyao.service.configuration.entity.hudi;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* HDFS文件
|
||||||
|
*
|
||||||
|
* @author lanyuanxiaoyao
|
||||||
|
* @date 2024-04-26
|
||||||
|
*/
|
||||||
|
public class HPath {
|
||||||
|
private String name;
|
||||||
|
private String path;
|
||||||
|
private Boolean file;
|
||||||
|
private Boolean folder;
|
||||||
|
private Long size;
|
||||||
|
private String group;
|
||||||
|
private String owner;
|
||||||
|
private String permission;
|
||||||
|
private Integer replication;
|
||||||
|
private Long modifyTime;
|
||||||
|
|
||||||
|
public HPath() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public HPath(String name, String path, Boolean file, Boolean folder, Long size, String group, String owner, String permission, Integer replication, Long modifyTime) {
|
||||||
|
this.name = name;
|
||||||
|
this.path = path;
|
||||||
|
this.file = file;
|
||||||
|
this.folder = folder;
|
||||||
|
this.size = size;
|
||||||
|
this.group = group;
|
||||||
|
this.owner = owner;
|
||||||
|
this.permission = permission;
|
||||||
|
this.replication = replication;
|
||||||
|
this.modifyTime = modifyTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPath() {
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Boolean getFile() {
|
||||||
|
return file;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Boolean getFolder() {
|
||||||
|
return folder;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getSize() {
|
||||||
|
return size;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getGroup() {
|
||||||
|
return group;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getOwner() {
|
||||||
|
return owner;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPermission() {
|
||||||
|
return permission;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getReplication() {
|
||||||
|
return replication;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getModifyTime() {
|
||||||
|
return modifyTime;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "HPath{" +
|
||||||
|
"name='" + name + '\'' +
|
||||||
|
", path='" + path + '\'' +
|
||||||
|
", file=" + file +
|
||||||
|
", folder=" + folder +
|
||||||
|
", size=" + size +
|
||||||
|
", group='" + group + '\'' +
|
||||||
|
", owner='" + owner + '\'' +
|
||||||
|
", permission='" + permission + '\'' +
|
||||||
|
", replication=" + replication +
|
||||||
|
", modifyTime=" + modifyTime +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,10 +4,12 @@ import com.dtflys.forest.annotation.BaseRequest;
|
|||||||
import com.dtflys.forest.annotation.Get;
|
import com.dtflys.forest.annotation.Get;
|
||||||
import com.dtflys.forest.annotation.Query;
|
import com.dtflys.forest.annotation.Query;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
|
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
|
||||||
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan;
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan;
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiRollbackPlan;
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiRollbackPlan;
|
||||||
|
import java.io.InputStream;
|
||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
import org.eclipse.collections.api.list.ImmutableList;
|
import org.eclipse.collections.api.list.ImmutableList;
|
||||||
import org.eclipse.collections.api.map.ImmutableMap;
|
import org.eclipse.collections.api.map.ImmutableMap;
|
||||||
@@ -64,4 +66,16 @@ public interface HudiService {
|
|||||||
|
|
||||||
@Get("/hdfs/exists_path")
|
@Get("/hdfs/exists_path")
|
||||||
Boolean existsPath(@Query("hdfs") String hdfs);
|
Boolean existsPath(@Query("hdfs") String hdfs);
|
||||||
|
|
||||||
|
@Get("/hdfs/get")
|
||||||
|
HPath get(@Query("root") String root);
|
||||||
|
|
||||||
|
@Get("/hdfs/list")
|
||||||
|
ImmutableList<HPath> list(@Query("root") String root);
|
||||||
|
|
||||||
|
@Get("/hdfs/read")
|
||||||
|
String read(@Query("root") String root);
|
||||||
|
|
||||||
|
@Get("/hdfs/download")
|
||||||
|
InputStream download(@Query("root") String root);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package com.lanyuanxiaoyao.service.hudi.controller;
|
package com.lanyuanxiaoyao.service.hudi.controller;
|
||||||
|
|
||||||
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
|
||||||
import com.lanyuanxiaoyao.service.hudi.service.HdfsService;
|
import com.lanyuanxiaoyao.service.hudi.service.HdfsService;
|
||||||
|
import java.io.IOException;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
|
import org.eclipse.collections.api.list.ImmutableList;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
@@ -42,4 +46,24 @@ public class HdfsController {
|
|||||||
public Boolean existsPath(@RequestParam("hdfs") String hdfs) {
|
public Boolean existsPath(@RequestParam("hdfs") String hdfs) {
|
||||||
return hdfsService.existsPath(hdfs);
|
return hdfsService.existsPath(hdfs);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping("get")
|
||||||
|
public HPath get(@RequestParam("root")String root) throws IOException {
|
||||||
|
return hdfsService.get(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("list")
|
||||||
|
public ImmutableList<HPath> list(@RequestParam("root")String root) throws IOException {
|
||||||
|
return hdfsService.list(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("read")
|
||||||
|
public String read(@RequestParam("root")String root) throws IOException {
|
||||||
|
return hdfsService.read(root);
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("download")
|
||||||
|
public void download(@RequestParam("root")String root, HttpServletResponse response) throws IOException {
|
||||||
|
hdfsService.download(root, response.getOutputStream());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,13 +1,21 @@
|
|||||||
package com.lanyuanxiaoyao.service.hudi.service;
|
package com.lanyuanxiaoyao.service.hudi.service;
|
||||||
|
|
||||||
|
import cn.hutool.core.io.IoUtil;
|
||||||
import cn.hutool.core.util.ObjectUtil;
|
import cn.hutool.core.util.ObjectUtil;
|
||||||
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
|
import com.lanyuanxiaoyao.service.common.entity.TableMeta;
|
||||||
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
|
||||||
import com.lanyuanxiaoyao.service.forest.service.InfoService;
|
import com.lanyuanxiaoyao.service.forest.service.InfoService;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.io.OutputStream;
|
||||||
import org.apache.hadoop.conf.Configuration;
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FSDataInputStream;
|
||||||
|
import org.apache.hadoop.fs.FileStatus;
|
||||||
import org.apache.hadoop.fs.FileSystem;
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
import org.apache.hadoop.fs.Path;
|
import org.apache.hadoop.fs.Path;
|
||||||
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
import org.apache.hudi.common.table.HoodieTableMetaClient;
|
||||||
|
import org.eclipse.collections.api.factory.Lists;
|
||||||
|
import org.eclipse.collections.api.list.ImmutableList;
|
||||||
|
import org.eclipse.collections.api.list.MutableList;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.cache.annotation.Cacheable;
|
import org.springframework.cache.annotation.Cacheable;
|
||||||
@@ -58,10 +66,81 @@ public class HdfsService {
|
|||||||
|
|
||||||
@Cacheable(value = "exists-path", sync = true)
|
@Cacheable(value = "exists-path", sync = true)
|
||||||
public Boolean existsPath(String hdfs) {
|
public Boolean existsPath(String hdfs) {
|
||||||
try(FileSystem fileSystem = FileSystem.get(new Configuration())) {
|
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
|
||||||
return fileSystem.exists(new Path(hdfs));
|
return fileSystem.exists(new Path(hdfs));
|
||||||
} catch (IOException ignored) {
|
} catch (IOException ignored) {
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Cacheable(value = "get-hpath", sync = true)
|
||||||
|
public HPath get(String root) throws IOException {
|
||||||
|
if (!existsPath(root)) {
|
||||||
|
throw new RuntimeException("File not found");
|
||||||
|
}
|
||||||
|
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
|
||||||
|
FileStatus status = fileSystem.getFileStatus(new Path(root));
|
||||||
|
return new HPath(
|
||||||
|
status.getPath().getName(),
|
||||||
|
status.getPath().toString(),
|
||||||
|
status.isFile(),
|
||||||
|
status.isDirectory(),
|
||||||
|
status.getLen(),
|
||||||
|
status.getGroup(),
|
||||||
|
status.getOwner(),
|
||||||
|
status.getPermission().toString(),
|
||||||
|
(int) status.getReplication(),
|
||||||
|
status.getModificationTime()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Cacheable(value = "list-hpath", sync = true)
|
||||||
|
public ImmutableList<HPath> list(String root) throws IOException {
|
||||||
|
if (!existsPath(root)) {
|
||||||
|
return Lists.immutable.empty();
|
||||||
|
}
|
||||||
|
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
|
||||||
|
MutableList<HPath> files = Lists.mutable.empty();
|
||||||
|
for (FileStatus status : fileSystem.listStatus(new Path(root))) {
|
||||||
|
files.add(new HPath(
|
||||||
|
status.getPath().getName(),
|
||||||
|
status.getPath().toString(),
|
||||||
|
status.isFile(),
|
||||||
|
status.isDirectory(),
|
||||||
|
status.getLen(),
|
||||||
|
status.getGroup(),
|
||||||
|
status.getOwner(),
|
||||||
|
status.getPermission().toString(),
|
||||||
|
(int) status.getReplication(),
|
||||||
|
status.getModificationTime()
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return files.toImmutable();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Cacheable(value = "read-hpath", sync = true)
|
||||||
|
public String read(String root) throws IOException {
|
||||||
|
if (!existsPath(root)) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
|
||||||
|
try (FSDataInputStream stream = fileSystem.open(new Path(root))) {
|
||||||
|
return IoUtil.readUtf8(stream);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("SpringCacheableMethodCallsInspection")
|
||||||
|
public void download(String root, OutputStream outputStream) throws IOException {
|
||||||
|
if (!existsPath(root)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
try (FileSystem fileSystem = FileSystem.get(new Configuration())) {
|
||||||
|
try (FSDataInputStream stream = fileSystem.open(new Path(root))) {
|
||||||
|
IoUtil.copy(stream, outputStream);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,10 @@
|
|||||||
package com.lanyuanxiaoyao.service.web.controller;
|
package com.lanyuanxiaoyao.service.web.controller;
|
||||||
|
|
||||||
|
import cn.hutool.core.io.IoUtil;
|
||||||
import cn.hutool.core.util.ObjectUtil;
|
import cn.hutool.core.util.ObjectUtil;
|
||||||
import cn.hutool.core.util.StrUtil;
|
import cn.hutool.core.util.StrUtil;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
|
import com.lanyuanxiaoyao.service.configuration.entity.PageResponse;
|
||||||
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan;
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCleanerPlan;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan;
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiCompactionPlan;
|
||||||
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiInstant;
|
||||||
@@ -10,15 +12,21 @@ import com.lanyuanxiaoyao.service.configuration.entity.hudi.HudiRollbackPlan;
|
|||||||
import com.lanyuanxiaoyao.service.forest.service.HudiService;
|
import com.lanyuanxiaoyao.service.forest.service.HudiService;
|
||||||
import com.lanyuanxiaoyao.service.web.controller.base.AmisCrudResponse;
|
import com.lanyuanxiaoyao.service.web.controller.base.AmisCrudResponse;
|
||||||
import com.lanyuanxiaoyao.service.web.controller.base.AmisDetailResponse;
|
import com.lanyuanxiaoyao.service.web.controller.base.AmisDetailResponse;
|
||||||
|
import com.lanyuanxiaoyao.service.web.controller.base.AmisMapResponse;
|
||||||
import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse;
|
import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse;
|
||||||
import com.lanyuanxiaoyao.service.web.controller.base.BaseController;
|
import com.lanyuanxiaoyao.service.web.controller.base.BaseController;
|
||||||
|
import com.lanyuanxiaoyao.service.web.entity.HPathChildrenVO;
|
||||||
|
import com.lanyuanxiaoyao.service.web.entity.HPathVO;
|
||||||
|
import java.io.InputStream;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.function.Function;
|
import java.util.function.Function;
|
||||||
|
import javax.servlet.http.HttpServletResponse;
|
||||||
import org.eclipse.collections.api.factory.Maps;
|
import org.eclipse.collections.api.factory.Maps;
|
||||||
import org.eclipse.collections.api.map.ImmutableMap;
|
import org.eclipse.collections.api.map.ImmutableMap;
|
||||||
import org.eclipse.collections.api.map.MutableMap;
|
import org.eclipse.collections.api.map.MutableMap;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.http.MediaType;
|
||||||
import org.springframework.web.bind.annotation.GetMapping;
|
import org.springframework.web.bind.annotation.GetMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestMapping;
|
import org.springframework.web.bind.annotation.RequestMapping;
|
||||||
import org.springframework.web.bind.annotation.RequestParam;
|
import org.springframework.web.bind.annotation.RequestParam;
|
||||||
@@ -163,4 +171,46 @@ public class HudiController extends BaseController {
|
|||||||
}
|
}
|
||||||
throw new Exception("Flink job id and alias or hdfs cannot be blank");
|
throw new Exception("Flink job id and alias or hdfs cannot be blank");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GetMapping("hdfs_list")
|
||||||
|
public AmisCrudResponse hdfsList(@RequestParam("root") String root) throws Exception {
|
||||||
|
if (StrUtil.isBlank(root)) {
|
||||||
|
throw new Exception("Root path cannot be blank");
|
||||||
|
}
|
||||||
|
return AmisCrudResponse.responseCrudData(hudiService.list(root).collect(HPathVO::new));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("hdfs_list_children")
|
||||||
|
public AmisResponse<HPathChildrenVO> hdfsListChildren(@RequestParam("root") String root) throws Exception {
|
||||||
|
if (StrUtil.isBlank(root)) {
|
||||||
|
throw new Exception("Root path cannot be blank");
|
||||||
|
}
|
||||||
|
return AmisResponse.responseSuccess(new HPathChildrenVO(hudiService.list(root).collect(HPathVO::new)));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping("hdfs_read")
|
||||||
|
public AmisMapResponse hdfsRead(@RequestParam("root") String root) throws Exception {
|
||||||
|
if (StrUtil.isBlank(root)) {
|
||||||
|
throw new Exception("Root path cannot be blank");
|
||||||
|
}
|
||||||
|
return AmisResponse.responseMapData()
|
||||||
|
.setData("text", hudiService.read(root));
|
||||||
|
}
|
||||||
|
|
||||||
|
@GetMapping(value = "hdfs_download")
|
||||||
|
public void hdfsDownload(@RequestParam("root") String root, HttpServletResponse response) throws Exception {
|
||||||
|
if (StrUtil.isBlank(root)) {
|
||||||
|
throw new Exception("Root path cannot be blank");
|
||||||
|
}
|
||||||
|
HPath hPath = hudiService.get(root);
|
||||||
|
if (hPath.getFolder()) {
|
||||||
|
throw new Exception("Folder cannot be downloaded");
|
||||||
|
}
|
||||||
|
try (InputStream inputStream = hudiService.download(root)) {
|
||||||
|
response.setHeader("Content-Type", MediaType.APPLICATION_OCTET_STREAM_VALUE);
|
||||||
|
response.setHeader("Content-Disposition", StrUtil.format("attachment;filename={}", hPath.getName()));
|
||||||
|
response.setHeader("Content-Length", hPath.getSize().toString());
|
||||||
|
IoUtil.copy(inputStream, response.getOutputStream());
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package com.lanyuanxiaoyao.service.web.entity;
|
||||||
|
|
||||||
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
|
||||||
|
import org.eclipse.collections.api.list.ImmutableList;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* HPath对象的有children版本
|
||||||
|
*
|
||||||
|
* @author lanyuanxiaoyao
|
||||||
|
* @date 2024-04-26
|
||||||
|
*/
|
||||||
|
public class HPathChildrenVO {
|
||||||
|
private final ImmutableList<HPathVO> children;
|
||||||
|
|
||||||
|
public HPathChildrenVO(ImmutableList<HPathVO> children) {
|
||||||
|
this.children = children;
|
||||||
|
}
|
||||||
|
|
||||||
|
public ImmutableList<HPathVO> getChildren() {
|
||||||
|
return children;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "AmisCrudChildrenVO{" +
|
||||||
|
"children=" + children +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package com.lanyuanxiaoyao.service.web.entity;
|
||||||
|
|
||||||
|
import cn.hutool.core.io.unit.DataSizeUtil;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
||||||
|
import com.lanyuanxiaoyao.service.configuration.entity.hudi.HPath;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @author lanyuanxiaoyao
|
||||||
|
* @date 2024-04-26
|
||||||
|
*/
|
||||||
|
public class HPathVO {
|
||||||
|
@JsonIgnore
|
||||||
|
private final HPath hPath;
|
||||||
|
|
||||||
|
private final Boolean defer;
|
||||||
|
private final String sizeText;
|
||||||
|
|
||||||
|
public HPathVO(HPath hPath) {
|
||||||
|
this.hPath = hPath;
|
||||||
|
this.defer = hPath.getFolder();
|
||||||
|
this.sizeText = DataSizeUtil.format(hPath.getSize());
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return hPath.getName();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPath() {
|
||||||
|
return hPath.getPath();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Boolean getFile() {
|
||||||
|
return hPath.getFile();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Boolean getFolder() {
|
||||||
|
return hPath.getFolder();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getSize() {
|
||||||
|
return hPath.getSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getGroup() {
|
||||||
|
return hPath.getGroup();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getOwner() {
|
||||||
|
return hPath.getOwner();
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPermission() {
|
||||||
|
return hPath.getPermission();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Integer getReplication() {
|
||||||
|
return hPath.getReplication();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Long getModifyTime() {
|
||||||
|
return hPath.getModifyTime();
|
||||||
|
}
|
||||||
|
|
||||||
|
public Boolean getDefer() {
|
||||||
|
return defer;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSizeText() {
|
||||||
|
return sizeText;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "HPathVO{" +
|
||||||
|
"hPath=" + hPath +
|
||||||
|
", defer=" + defer +
|
||||||
|
", sizeText='" + sizeText + '\'' +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -55,6 +55,176 @@ function toolTab() {
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
type: 'form',
|
||||||
|
title: 'HDFS文件管理器',
|
||||||
|
actions: [
|
||||||
|
{
|
||||||
|
label: '直接下载',
|
||||||
|
type: 'action',
|
||||||
|
onEvent: {
|
||||||
|
click: {
|
||||||
|
actions: [
|
||||||
|
{
|
||||||
|
actionType: 'custom',
|
||||||
|
script: (context, action, event) => {
|
||||||
|
let downloadUrl = `${event.data.base}/hudi/hdfs_download?root=${encodeURI(event.data.hdfs)}`
|
||||||
|
window.open(downloadUrl, '_blank')
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'submit',
|
||||||
|
label: '查看',
|
||||||
|
actionType: 'dialog',
|
||||||
|
dialog: {
|
||||||
|
title: {
|
||||||
|
type: 'tpl',
|
||||||
|
tpl: '${hdfs}'
|
||||||
|
},
|
||||||
|
...readOnlyDialogOptions(),
|
||||||
|
size: 'xl',
|
||||||
|
body: {
|
||||||
|
type: 'crud',
|
||||||
|
api: {
|
||||||
|
method: 'get',
|
||||||
|
url: '${base}/hudi/hdfs_list',
|
||||||
|
data: {
|
||||||
|
root: '${hdfs|default:undefined}',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
deferApi: {
|
||||||
|
method: 'get',
|
||||||
|
url: '${base}/hudi/hdfs_list_children',
|
||||||
|
data: {
|
||||||
|
root: '${path|default:undefined}',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
...crudCommonOptions(),
|
||||||
|
perPage: 10,
|
||||||
|
headerToolbar: [
|
||||||
|
"reload",
|
||||||
|
paginationCommonOptions(undefined, 10),
|
||||||
|
],
|
||||||
|
footerToolbar: [
|
||||||
|
paginationCommonOptions(undefined, 10),
|
||||||
|
],
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
name: 'name',
|
||||||
|
label: '文件名',
|
||||||
|
className: 'font-mono',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'group',
|
||||||
|
label: '组',
|
||||||
|
width: 70,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'owner',
|
||||||
|
label: '用户',
|
||||||
|
width: 70,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: '大小',
|
||||||
|
width: 90,
|
||||||
|
type: 'tpl',
|
||||||
|
tpl: "${sizeText}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
label: '修改时间',
|
||||||
|
width: 140,
|
||||||
|
type: 'tpl',
|
||||||
|
tpl: "${DATETOSTR(DATE(modifyTime), 'YYYY-MM-DD HH:mm:ss')}",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'operation',
|
||||||
|
label: '操作',
|
||||||
|
width: 160,
|
||||||
|
fixed: 'right',
|
||||||
|
buttons: [
|
||||||
|
{
|
||||||
|
label: '完整路径',
|
||||||
|
type: 'action',
|
||||||
|
level: 'link',
|
||||||
|
size: 'xs',
|
||||||
|
actionType: 'copy',
|
||||||
|
content: '${path}',
|
||||||
|
tooltip: '复制 ${path}',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
disabledOn: 'folder || size > 1048576',
|
||||||
|
label: '查看',
|
||||||
|
type: 'action',
|
||||||
|
level: 'link',
|
||||||
|
size: 'xs',
|
||||||
|
actionType: 'dialog',
|
||||||
|
dialog: {
|
||||||
|
title: {
|
||||||
|
type: 'tpl',
|
||||||
|
tpl: '文件内容:${path}'
|
||||||
|
},
|
||||||
|
size: 'md',
|
||||||
|
...readOnlyDialogOptions(),
|
||||||
|
body: {
|
||||||
|
type: 'service',
|
||||||
|
api: {
|
||||||
|
method: 'get',
|
||||||
|
url: '${base}/hudi/hdfs_read',
|
||||||
|
data: {
|
||||||
|
root: '${path|default:undefined}'
|
||||||
|
}
|
||||||
|
},
|
||||||
|
body: {
|
||||||
|
type: 'textarea',
|
||||||
|
name: 'text',
|
||||||
|
readOnly: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
disabledOn: 'folder',
|
||||||
|
label: '下载',
|
||||||
|
type: 'action',
|
||||||
|
level: 'link',
|
||||||
|
size: 'xs',
|
||||||
|
onEvent: {
|
||||||
|
click: {
|
||||||
|
actions: [
|
||||||
|
{
|
||||||
|
actionType: 'custom',
|
||||||
|
script: (context, action, event) => {
|
||||||
|
let downloadUrl = `${event.data.base}/hudi/hdfs_download?root=${encodeURI(event.data.path)}`
|
||||||
|
window.open(downloadUrl, '_blank')
|
||||||
|
},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
body: [
|
||||||
|
{
|
||||||
|
type: 'input-text',
|
||||||
|
name: 'hdfs',
|
||||||
|
label: 'HDFS根路经',
|
||||||
|
required: true,
|
||||||
|
clearable: true,
|
||||||
|
description: '输入表HDFS路径',
|
||||||
|
autoComplete: '${base}/table/all_hdfs?key=$term',
|
||||||
|
},
|
||||||
|
]
|
||||||
|
},
|
||||||
{
|
{
|
||||||
type: 'form',
|
type: 'form',
|
||||||
title: '查询时间线',
|
title: '查询时间线',
|
||||||
|
|||||||
@@ -96,3 +96,6 @@ GET {{exporter-url}}/exporter/un_running_flink_job
|
|||||||
|
|
||||||
### Pulsar backlog
|
### Pulsar backlog
|
||||||
GET http://{{username}}:{{password}}@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
|
GET http://{{username}}:{{password}}@b12s15.hdp.dc:21685/pulsar/backlog?name=main&topic=persistent://odcp/grid/grid_serv_staff&subscription=Hudi_Sync_Pulsar_Reader_1552408245762723840_grid_grid_serv_staff_b_20230425
|
||||||
|
|
||||||
|
### Test HDFS list
|
||||||
|
GET http://{{username}}:{{password}}@b12s10.hdp.dc:16695/hdfs/list?root=hdfs://b2/apps/datalake/hive/dws_test/external_table_hudi/dws_ord_prod_inst_attr
|
||||||
|
|||||||
Reference in New Issue
Block a user