lanyuanxiaoyao/gringotts
1
0
Fork 0
Code Activity

feat(web): 增加数据资源接口

Browse Source
This commit is contained in:
兰缘小妖
2024-11-20 18:35:31 +08:00
parent 5ce66e1470
commit ba19775764
22 changed files with 662 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
gringotts-frontend/components/resource/dialog-resource.css Normal file
View File

@@ -0,0 +1,3 @@
textarea {
resize: none !important;
}

268
gringotts-frontend/components/resource/dialog-resource.js Normal file
View File

@@ -0,0 +1,268 @@
import './dialog-resource.css'
import {apiPost, horizontalFormOptions} from "../constants.js";
const clearable = {
clearable: true,
clearValueOnEmpty: true,
}
export function resourceAddDialog() {
return {
actionType: 'dialog',
dialog: {
title: '新增数据资源',
// size: 'lg',
actions: [
{
type: 'reset',
label: '重置',
},
{
type: 'submit',
label: '确定',
level: 'primary',
}
],
body: {
debug: true,
type: 'form',
api: apiPost('${base}/data_resource/create'),
...horizontalFormOptions(),
horizontal: {
left: 2,
},
body: [
{
type: 'input-text',
name: 'name',
label: '资源名称',
required: true,
...clearable,
},
{
type: 'textarea',
name: 'description',
label: '资源描述',
...clearable,
showCounter: true,
trimContents: true,
minRows: 2,
maxRows: 2,
},
{
type: 'fieldSet',
title: '资源类型定义',
body: [
{
name: 'resourceType',
type: 'select',
label: '资源类型',
selectFirst: true,
required: true,
options: [
{label: 'API', value: 'api'},
{label: '文件', value: 'file'},
{label: '数据库', value: 'database'},
{label: 'HDFS', value: 'hdfs'},
{label: 'FTP', value: 'ftp'},
]
},
{
visibleOn: "${resourceType === 'api'}",
type: 'fieldSet',
body: [
{
type: 'input-text',
label: 'API地址',
name: 'apiUrl',
required: true,
...clearable,
},
{
type: 'input-text',
label: '用户名',
name: 'apiUsername',
...clearable,
},
{
type: 'input-password',
label: '密码',
name: 'apiPassword',
...clearable,
},
]
},
{
visibleOn: "${resourceType === 'file'}",
type: 'fieldSet',
body: [
{
type: 'input-file',
name: 'filePath',
accept: '.zip',
description: '只适合小于2M的资源文件使用，大文件请使用其他资源类型',
},
]
},
{
visibleOn: "${resourceType === 'database'}",
type: 'fieldSet',
body: [
{
type: 'select',
label: '数据库类型',
name: 'databaseType',
required: true,
options: [
{label: 'MySQL', value: 'MYSQL'},
{label: 'Oracle', value: 'ORACLE'},
{label: 'PostgreSQL', value: 'POSTGRESQL'},
]
},
{
type: 'input-text',
label: 'JDBC',
name: 'databaseJdbc',
required: true,
...clearable,
},
{
type: 'input-text',
label: '用户名',
name: 'databaseUsername',
...clearable,
},
{
type: 'input-password',
label: '密码',
name: 'databasePassword',
...clearable,
},
]
},
{
visibleOn: "${resourceType === 'hdfs'}",
type: 'fieldSet',
body: [
{
type: 'input-file',
description: 'core-site.xml',
name: 'coreSiteFile',
accept: '.xml',
},
{
type: 'input-file',
description: 'hdfs-site.xml',
name: 'hdfsSiteFile',
accept: '.xml',
},
]
},
{
visibleOn: "${resourceType === 'ftp'}",
type: 'fieldSet',
body: [
{
type: 'input-text',
label: 'FTP地址',
name: 'ftpUrl',
required: true,
...clearable,
},
{
type: 'input-text',
label: 'FTP账号',
name: 'ftpUsername',
...clearable,
},
{
type: 'input-password',
label: 'FTP密码',
name: 'ftpPassword',
...clearable,
},
{
type: 'input-text',
label: '相对路径',
name: 'ftpPath',
description: '若为空，则使用用户根目录',
...clearable,
},
{
type: 'input-text',
label: '文件筛选',
name: 'ftpRegex',
description: '正则表达式，用于匹配文件的路径，只有符合筛选条件的文件才会被采集；若为空则默认采集全部文件',
...clearable,
},
]
},
]
},
{
type: 'fieldSet',
title: '资源格式定义',
className: 'mt-5',
body: [
{
name: 'formatType',
type: 'select',
label: '资源格式',
selectFirst: true,
required: true,
options: [
{label: '无', value: 'NONE'},
{label: 'Line', value: 'LINE'},
{label: 'JSON', value: 'JSON'},
{label: 'JSON Line', value: 'JSON_LINE'},
{label: 'CSV', value: 'CSV'},
]
},
{
visibleOn: "${formatType === 'JSON'}",
type: 'json-schema-editor',
name: 'jsonSchema',
label: 'JSON格式',
required: true,
enableAdvancedSetting: true,
mini: true,
},
{
visibleOn: "${formatType === 'JSON_LINE'}",
type: 'json-schema-editor',
name: 'jsonLineSchema',
label: 'JSON格式',
description: 'JSON Line类型请定义单行JSON数据格式',
required: true,
enableAdvancedSetting: true,
mini: true,
},
{
visibleOn: "${formatType === 'CSV'}",
type: 'json-schema-editor',
name: 'csvSchema',
label: 'CSV格式',
description: '请定义单行数据中各个字段的格式',
required: true,
enableAdvancedSetting: true,
mini: true,
disabledTypes: [
'object',
'array',
'null',
]
},
{
type: 'input-file',
label: '资源示例',
description: '可以上传用于作为格式示范的样例数据',
name: 'example',
accept: '*',
},
]
}
]
}
}
}
}

4
gringotts-frontend/pages/index/main.js
View File

@@ -57,9 +57,9 @@ useAmis(information => {
type: 'tabs',
tabsMode: 'vertical',
tabs: [
tabOverview(),
tabMarket(),
tabData(),
// tabOverview(),
tabMarket(),
tabUser(),
tabLog(),
]

11
gringotts-frontend/pages/index/tab-data.js
View File

@@ -1,9 +1,16 @@
import {resourceAddDialog} from "../../components/resource/dialog-resource.js";
export function tabData() {
return {
title: '数据管理',
title: '数据资源',
icon: 'fa fa-database',
body: [
'hello world'
{
type: 'action',
label: '',
icon: 'fa fa-plus',
...resourceAddDialog()
}
]
}
}

38
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/IdOnlyEntity.java Normal file
View File

@@ -0,0 +1,38 @@
package com.eshore.gringotts.web.domain.entity;
import com.eshore.gringotts.web.domain.user.entity.User;
import java.time.LocalDateTime;
import javax.persistence.ConstraintMode;
import javax.persistence.EntityListeners;
import javax.persistence.FetchType;
import javax.persistence.ForeignKey;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.MappedSuperclass;
import javax.persistence.OneToOne;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.GenericGenerator;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
/**
* 实体类公共字段
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public class IdOnlyEntity {
@Id
@GeneratedValue(generator = "snowflake")
@GenericGenerator(name = "snowflake", strategy = "com.eshore.gringotts.web.configuration.SnowflakeIdGenerator")
private Long id;
}

9
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/SuperEntity.java → gringotts-web/src/main/java/com/eshore/gringotts/web/domain/entity/SimpleEntity.java
View File

@@ -6,15 +6,12 @@ import javax.persistence.ConstraintMode;
import javax.persistence.EntityListeners;
import javax.persistence.FetchType;
import javax.persistence.ForeignKey;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.JoinColumn;
import javax.persistence.MappedSuperclass;
import javax.persistence.OneToOne;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.GenericGenerator;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@@ -30,11 +27,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@ToString
@MappedSuperclass
@EntityListeners(AuditingEntityListener.class)
public class SuperEntity {
@Id
@GeneratedValue(generator = "snowflake")
@GenericGenerator(name = "snowflake", strategy = "com.eshore.gringotts.web.configuration.SnowflakeIdGenerator")
private Long id;
public class SimpleEntity extends IdOnlyEntity {
@CreatedDate
private LocalDateTime createdTime;
@OneToOne(fetch = FetchType.LAZY)

54
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/controller/DataResourceController.java Normal file
View File

@@ -0,0 +1,54 @@
package com.eshore.gringotts.web.domain.resource.controller;
import java.util.Map;
import lombok.Data;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* 数据资源
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@RestController
@RequestMapping("/data_resource")
public class DataResourceController {
private static final Logger logger = LoggerFactory.getLogger(DataResourceController.class);
@PostMapping("/create")
public void create(@RequestBody CreateRequest request) {
logger.info("Create request: {}", request);
}
@Data
public static final class CreateRequest {
private String name;
private String description;
private String resourceType;
private String apiUrl;
private String apiUsername;
private String apiPassword;
private String filePath;
private String databaseType;
private String databaseJdbc;
private String databaseUsername;
private String databasePassword;
private String coreSiteFile;
private String hdfsSiteFile;
private String ftpUrl;
private String ftpUsername;
private String ftpPassword;
private String ftpPath;
private String ftpRegex;
private String formatType;
private Map<?, ?> jsonSchema;
private Map<?, ?> jsonLineSchema;
private Map<?, ?> csvSchema;
private String example;
}
}

19
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/ApiResource.java
View File

@@ -1,19 +0,0 @@
package com.eshore.gringotts.web.domain.resource.entity;
import com.eshore.gringotts.core.Constants;
import javax.persistence.Entity;
import javax.persistence.EntityListeners;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Getter
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "data_resource_api")
public class ApiResource extends DataResource {
private String url;
}

23
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/DataResource.java
View File

@@ -1,26 +1,20 @@
package com.eshore.gringotts.web.domain.resource.entity;
import com.eshore.gringotts.core.Constants;
import com.eshore.gringotts.web.domain.entity.SuperEntity;
import com.eshore.gringotts.web.domain.user.entity.User;
import java.time.LocalDateTime;
import com.eshore.gringotts.web.domain.entity.SimpleEntity;
import com.eshore.gringotts.web.domain.resource.entity.format.ResourceFormat;
import com.eshore.gringotts.web.domain.resource.entity.type.ResourceType;
import javax.persistence.Column;
import javax.persistence.ConstraintMode;
import javax.persistence.ForeignKey;
import javax.persistence.JoinColumn;
import javax.persistence.ManyToOne;
import javax.persistence.OneToOne;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.DynamicUpdate;
import org.hibernate.annotations.GenericGenerator;
import org.springframework.data.annotation.CreatedDate;
import org.springframework.data.annotation.LastModifiedDate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
import javax.persistence.Entity;
import javax.persistence.EntityListeners;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.Inheritance;
import javax.persistence.InheritanceType;
import javax.persistence.Table;
@@ -32,9 +26,14 @@ import javax.persistence.Table;
@EntityListeners(AuditingEntityListener.class)
@DynamicUpdate
@Table(name = Constants.TABLE_PREFIX + "data_resource")
@Inheritance(strategy = InheritanceType.JOINED)
public class DataResource extends SuperEntity {
public class DataResource extends SimpleEntity {
@Column(nullable = false)
private String name;
private String description;
@OneToOne
@JoinColumn(nullable = false)
private ResourceType type;
@OneToOne
@JoinColumn(nullable = false)
private ResourceFormat format;
}

22
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/format/CsvResourceFormat.java Normal file
View File

@@ -0,0 +1,22 @@
package com.eshore.gringotts.web.domain.resource.entity.format;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
/**
* CSV格式
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@Entity
@DiscriminatorValue("CSV")
public class CsvResourceFormat extends ResourceFormat{
private String schema;
}

22
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/format/JsonLineResourceFormat.java Normal file
View File

@@ -0,0 +1,22 @@
package com.eshore.gringotts.web.domain.resource.entity.format;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
/**
* JSON line格式
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@Entity
@DiscriminatorValue("JSON_LINE")
public class JsonLineResourceFormat extends ResourceFormat{
private String schema;
}

23
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/format/JsonResourceFormat.java Normal file
View File

@@ -0,0 +1,23 @@
package com.eshore.gringotts.web.domain.resource.entity.format;
import javax.persistence.Column;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
/**
* 一个文件就是一个JSON
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@Entity
@DiscriminatorValue("JSON")
public class JsonResourceFormat extends ResourceFormat{
private String schema;
}

21
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/format/LineResourceFormat.java Normal file
View File

@@ -0,0 +1,21 @@
package com.eshore.gringotts.web.domain.resource.entity.format;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
/**
* 无格式，文件每行就是一份数据
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@Entity
@DiscriminatorValue("LINE")
public class LineResourceFormat extends ResourceFormat{
}

21
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/format/NoneResourceFormat.java Normal file
View File

@@ -0,0 +1,21 @@
package com.eshore.gringotts.web.domain.resource.entity.format;
import javax.persistence.DiscriminatorValue;
import javax.persistence.Entity;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
/**
* 无格式，整个文件就是一份数据
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@Entity
@DiscriminatorValue("NONE")
public class NoneResourceFormat extends ResourceFormat{
}

38
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/format/ResourceFormat.java Normal file
View File

@@ -0,0 +1,38 @@
package com.eshore.gringotts.web.domain.resource.entity.format;
import com.eshore.gringotts.core.Constants;
import com.eshore.gringotts.web.domain.entity.IdOnlyEntity;
import javax.persistence.DiscriminatorColumn;
import javax.persistence.DiscriminatorType;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Inheritance;
import javax.persistence.InheritanceType;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
/**
* 资源格式
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "resource_format")
@Inheritance(strategy = InheritanceType.SINGLE_TABLE)
@DiscriminatorColumn(name = "type", discriminatorType = DiscriminatorType.STRING)
public class ResourceFormat extends IdOnlyEntity {
public enum Type {
NONE,
LINE,
JSON,
JSON_LINE,
CSV,
}
}

21
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/type/ApiResourceType.java Normal file
View File

@@ -0,0 +1,21 @@
package com.eshore.gringotts.web.domain.resource.entity.type;
import com.eshore.gringotts.core.Constants;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "resource_type_api")
public class ApiResourceType extends ResourceType {
@Column(nullable = false)
private String url;
private String username;
private String password;
}

32
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/type/DatabaseResourceType.java Normal file
View File

@@ -0,0 +1,32 @@
package com.eshore.gringotts.web.domain.resource.entity.type;
import com.eshore.gringotts.core.Constants;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "resource_type_database")
public class DatabaseResourceType extends ResourceType {
@Column(nullable = false)
private String jdbc;
private String username;
private String password;
@Column(nullable = false)
@Enumerated(EnumType.STRING)
private Type type;
public enum Type {
MYSQL,
POSTGRESQL,
ORACLE,
}
}

8
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/FileResource.java → gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/type/FileResourceType.java
View File

@@ -1,6 +1,7 @@
package com.eshore.gringotts.web.domain.resource.entity;
package com.eshore.gringotts.web.domain.resource.entity.type;
import com.eshore.gringotts.core.Constants;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import lombok.Getter;
@@ -11,7 +12,8 @@ import lombok.ToString;
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "data_resource_file")
public class FileResource extends DataResource{
@Table(name = Constants.TABLE_PREFIX + "resource_type_file")
public class FileResourceType extends ResourceType {
@Column(nullable = false)
private String path;
}

22
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/type/FtpResourceType.java Normal file
View File

@@ -0,0 +1,22 @@
package com.eshore.gringotts.web.domain.resource.entity.type;
import com.eshore.gringotts.core.Constants;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "resource_type_ftp")
public class FtpResourceType extends ResourceType {
@Column(nullable = false)
private String url;
private String username;
private String password;
private String path;
}

21
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/type/HDFSResourceType.java Normal file
View File

@@ -0,0 +1,21 @@
package com.eshore.gringotts.web.domain.resource.entity.type;
import com.eshore.gringotts.core.Constants;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
@Getter
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "resource_type_hdfs")
public class HDFSResourceType extends ResourceType {
@Column(nullable = false)
private String coreSite;
@Column(nullable = false)
private String hdfsSite;
}

26
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/resource/entity/type/ResourceType.java Normal file
View File

@@ -0,0 +1,26 @@
package com.eshore.gringotts.web.domain.resource.entity.type;
import com.eshore.gringotts.core.Constants;
import com.eshore.gringotts.web.domain.entity.IdOnlyEntity;
import javax.persistence.Entity;
import javax.persistence.Inheritance;
import javax.persistence.InheritanceType;
import javax.persistence.Table;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
/**
* 资源类型
*
* @author lanyuanxiaoyao
* @date 2024-11-20
*/
@Getter
@Setter
@ToString
@Entity
@Table(name = Constants.TABLE_PREFIX + "resource_type")
@Inheritance(strategy = InheritanceType.JOINED)
public class ResourceType extends IdOnlyEntity {
}

6
gringotts-web/src/main/java/com/eshore/gringotts/web/domain/user/entity/User.java
View File

@@ -1,12 +1,11 @@
package com.eshore.gringotts.web.domain.user.entity;
import com.eshore.gringotts.core.Constants;
import com.eshore.gringotts.web.domain.entity.SuperEntity;
import com.eshore.gringotts.web.domain.entity.SimpleEntity;
import java.time.LocalDateTime;
import javax.persistence.Column;
import javax.persistence.ConstraintMode;
import javax.persistence.Entity;
import javax.persistence.EntityListeners;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.FetchType;
@@ -18,7 +17,6 @@ import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.hibernate.annotations.DynamicUpdate;
import org.springframework.data.jpa.domain.support.AuditingEntityListener;
/**
* 用户
@@ -32,7 +30,7 @@ import org.springframework.data.jpa.domain.support.AuditingEntityListener;
@Entity
@DynamicUpdate
@Table(name = Constants.TABLE_PREFIX + "user")
public class User extends SuperEntity {
public class User extends SimpleEntity {
@Column(unique = true, nullable = false)
private String username;
@Column(nullable = false)