lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(all): 初始化项目

Browse Source 
迁移项目到独立的 service 进行集中开发
包含以下组件的查询 API 服务
flink
hudi
database(info)
pulsar
yarn
包含前端服务和 UI
web
包含公共代码
configuration
This commit is contained in:
兰缘小妖
2023-05-01 00:35:57 +08:00
parent 6b4374ffcc
commit 2f2c10e7b7
191 changed files with 789282 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

661
service-web/src/main/resources/static/components/common.js Normal file
View File

@@ -0,0 +1,661 @@
function timeAndFrom(field, fromNow, emptyText) {
if (!emptyText) {
emptyText = '未停止'
}
// language=TEXT
return "<span class='font-bold'>${" + fromNow + "}</span><br> ${IF(" + field + " === 0, '(" + emptyText + ")', CONCATENATE('(',DATETOSTR(DATE(" + field + ")),')'))}"
}
function yarnCrudColumns() {
return [
{
name: 'id',
label: 'ID',
width: 240,
fixed: 'left',
...copyField('id')
},
{
name: 'name',
label: '名称',
fixed: 'left',
...copyField('name')
},
{
label: '用户',
width: 80,
type: 'tooltip-wrapper',
body: '${TRUNCATE(user, 8)}',
content: '${user}',
},
{
name: 'startedTime',
label: '启动时间',
width: 140,
type: 'tpl',
tpl: timeAndFrom('startedTime', 'startTimeFromNow'),
sortable: true,
align: 'center',
canAccessSuperData: false,
},
{
name: 'finishedTime',
label: '停止时间',
width: 140,
type: 'tpl',
tpl: timeAndFrom('finishedTime', 'finishTimeFromNow'),
sortable: true,
align: 'center',
canAccessSuperData: false,
},
{
name: 'state',
label: '运行状态',
width: 70,
align: 'center',
type: 'mapping',
canAccessSuperData: false,
map: {
'NEW': '<span class=\'label bg-pink-300\'>创建中</span>',
'NEW_SAVING': '<span class=\'label bg-purple-300\'>已创建</span>',
'SUBMITTED': '<span class=\'label bg-indigo-300\'>已提交</span>',
'ACCEPTED': '<span class=\'label bg-cyan-300\'>调度中</span>',
'RUNNING': '<span class=\'label label-success\'>运行中</span>',
'FINISHED': '<span class=\'label label-default\'>已结束</span>',
'FAILED': '<span class=\'label label-danger\'>已失败</span>',
'KILLED': '<span class=\'label label-warning\'>被停止</span>',
'*': '<span class=\'label bg-gray-300\'>${state}</span>'
},
filterable: {
multiple: true,
options: [
{label: '创建中', value: 'NEW'},
{label: '已创建', value: 'NEW_SAVING'},
{label: '已提交', value: 'SUBMITTED'},
{label: '调度中', value: 'ACCEPTED'},
{label: '运行中', value: 'RUNNING'},
{label: '已结束', value: 'FINISHED'},
{label: '已失败', value: 'FAILED'},
{label: '被停止', value: 'KILLED'},
]
}
}, {
name: 'finalStatus',
label: '最终状态',
width: 70,
align: 'center',
type: 'mapping',
canAccessSuperData: false,
map: {
'UNDEFINED': '<span class=\'label label-info\'>运行</span>',
'SUCCEEDED': '<span class=\'label label-success\'>成功</span>',
'FAILED': '<span class=\'label label-danger\'>失败</span>',
'KILLED': '<span class=\'label label-warning\'>终止</span>',
'ENDED': '<span class=\'label label-default\'>结束</span>',
'*': '<span class=\'label bg-gray-300\'>${state}</span>'
},
filterable: {
multiple: true,
options: [
{label: '运行', value: 'UNDEFINED'},
{label: '成功', value: 'SUCCEEDED'},
{label: '失败', value: 'FAILED'},
{label: '终止', value: 'KILLED'},
{label: '结束', value: 'ENDED'},
]
}
},
{
type: "operation",
label: "操作",
width: 100,
fixed: 'right',
buttons: [
/*{
disabled: true,
label: "停止",
type: "button",
level: "link",
tooltip: '从队列中移除该任务',
actionType: 'dialog',
dialog: {
closeOnEsc: true,
closeOnOutside: true,
size: 'sm',
close: true,
title: '停止确认',
body: '确认停止任务: <br>${name}<br> ?',
actions: [
{
type: 'button',
label: '取消',
actionType: 'cancel'
},
{
type: 'button',
label: '确认移除',
level: 'danger',
actionType: 'ajax',
api: '',
primary: true,
message: {
success: '操作成功',
failed: '操作失败'
}
}
]
}
},*/
{
disabledOn: '${trackingUrl == null}',
disabledTip: '无应用页面',
label: "应用页面",
type: "action",
level: "link",
tooltip: '打开应用页面: ${trackingUrl}',
actionType: 'url',
url: '${trackingUrl}',
blank: true,
}
]
}
]
}
function simpleYarnDialog(mode, title) {
return {
title: title,
actions: [],
size: 'xl',
body: [
{
type: 'service',
body: [
{
type: 'tpl',
tpl: 'hello',
visibleOn: '${1 == 1}',
},
{
type: 'tpl',
tpl: 'hello',
visibleOn: '${1 == 2}',
},
],
},
{
type: 'crud',
api: {
method: 'get',
url: `\${base}/${mode}_yarn/job_list`,
data: {
page: '${page|default:undefined}',
count: '${perPage|default:undefined}',
order: '${orderBy|default:undefined}',
direction: '${orderDir|default:undefined}',
filter_state: '${state|default:undefined}',
filter_final_status: '${finalStatus|default:undefined}',
search_id: '${id|default:undefined}',
search_name: `\${${mode}JobName}`,
precise: true,
}
},
interval: 10000,
syncLocation: false,
silentPolling: true,
resizable: false,
perPage: 10,
headerToolbar: [
"reload",
{
type: 'pagination',
layout: ['pager', 'perPage'],
maxButtons: 8,
showPageInput: false,
}
],
footerToolbar: [],
columns: yarnCrudColumns(),
}
],
}
}
function copyField(field, tips = '复制', ignoreLength = 0) {
let tpl = ignoreLength === 0 ? `\${${field}}` : `\${TRUNCATE(${field}, ${ignoreLength})}`
return {
type: 'wrapper',
size: 'none',
body: [
{
type: 'tpl',
className: 'mr-1',
tpl: tpl,
},
{
type: 'action',
level: 'link',
label: '',
icon: 'fa fa-copy',
size: 'xs',
actionType: 'copy',
content: `\$${field}`,
tooltip: `${tips}`,
},
]
}
}
function flinkJobProperty(id, name, runMode) {
return {
type: 'property',
title: 'Flink Job 配置',
items: [
{label: 'ID', content: copyField(id, '复制 ID')},
{label: '任务名称', content: copyField(name, '复制任务名')},
{
label: '任务模式',
content: {
...runModeMapping(`${runMode}`)
}
},
],
}
}
function flinkJobDialog() {
return {
title: 'Flink job 详情',
actions: [],
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
size: 'md',
body: [
flinkJobProperty('flinkJobId', 'flinkJob.name', 'flinkJob.runMode'),
{type: 'divider'},
{
type: 'action',
label: '打开同步详情',
actionType: 'dialog',
dialog: simpleYarnDialog('sync', '同步详情')
},
{type: 'divider'},
{
type: 'service',
api: {
method: 'get',
url: '${base}/table/list_metas',
data: {
flink_job_id: '${flinkJobId}'
},
},
canAccessSuperData: true,
body: [
{
type: "table",
title: "包含 Hudi 同步表",
source: "${items}",
columns: [
{
label: '别名',
type: 'wrapper',
size: 'none',
body: [
{
type: 'action',
level: 'link',
label: '${tableMeta.alias}',
size: 'xs',
actionType: 'dialog',
tooltip: '查看详情',
dialog: tableMetaDialog(),
},
{
type: 'action',
level: 'link',
label: '',
icon: 'fa fa-copy',
size: 'xs',
actionType: 'copy',
content: '${tableMeta.alias}',
tooltip: '复制别名',
},
],
},
{
label: '并行度',
name: 'tableMeta.hudi.writeTasks',
align: 'center',
},
{
label: 'Bucket 数量',
name: 'tableMeta.hudi.bucketIndexNumber',
align: 'center',
},
{
label: '写并行度',
name: 'tableMeta.hudi.writeTasks',
align: 'center',
},
{
label: '压缩并行度',
name: 'tableMeta.hudi.compactionTasks',
align: 'center',
},
]
}
]
}
]
}
}
function tableMetaDialog() {
return {
title: 'Table 详情',
actions: [],
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
size: 'lg',
body: [
flinkJobProperty('flinkJobId', 'flinkJob.name', 'flinkJob.runMode'),
{type: 'divider'},
{
type: 'property',
title: 'Table 配置',
items: [
{label: '别名', content: copyField('tableMeta.alias', '复制别名')},
{
label: '分片表',
content: '${IF(tableMeta.type === \'sharding\', \'是\', \'否\')}'
},
{label: '分区字段', content: copyField('tableMeta.partitionField', '复制分区字段')},
{label: '源端', content: copyField('tableMeta.source', '复制源端')},
{label: '源库名', content: copyField('tableMeta.schema', '复制库名')},
{label: '源表名', content: copyField('tableMeta.table', '复制表名')},
{label: '源表类型', content: '${tableMeta.sourceType}'},
{label: '优先级', content: '${tableMeta.priority}', span: 2},
{
label: '标签',
content: {
type: 'each',
source: '${SPLIT(tableMeta.tags, ",")}',
items: tagsMapping('item')
},
span: 3,
},
{label: '订阅 Topic', content: copyField('tableMeta.topic', '复制 Topic'), span: 3},
{
label: 'Pulsar 地址',
content: copyField('tableMeta.pulsarAddress', '复制地址', 130),
span: 3
},
{label: '过滤模式', content: filterModeMapping('tableMeta.filterType')},
{label: '过滤字段', content: '${tableMeta.filterField}', span: 2},
{
label: '过滤内容',
content: {
type: 'each',
source: '${SPLIT(tableMeta.filterValues, ",")}',
items: {
type: 'tpl',
tpl: "<span class='label bg-info mr-1'>${item}</span>",
}
},
span: 3,
},
],
},
{type: 'divider'},
{
type: 'property',
title: 'Hudi 配置',
items: [
{label: '表类型', content: hudiTableTypeMapping('tableMeta.hudi.targetTableType')},
{label: '库名', content: copyField('tableMeta.hudi.targetDataSource', '复制库名')},
{label: '表名', content: copyField('tableMeta.hudi.targetTable', '复制表名')},
{
label: 'HDFS',
content: copyField('tableMeta.hudi.targetHdfsPath', '复制路径'),
span: 3
},
{
label: 'Bucket 数量',
content: '${tableMeta.hudi.bucketIndexNumber}'
},
{
label: '保留文件版本',
content: '${tableMeta.hudi.keepFileVersion}'
},
{
label: '保留提交个数',
content: '${tableMeta.hudi.keepCommitVersion}'
},
{label: '读并行度', content: '${tableMeta.hudi.sourceTasks}'},
{label: '写并行度', content: '${tableMeta.hudi.writeTasks}'},
{
label: '压缩并行度',
content: '${tableMeta.hudi.compactionTasks}'
},
{
label: '写任务最大内存',
content: '${tableMeta.hudi. writeTaskMaxMemory}M'
},
{
label: '写批次大小',
content: '${tableMeta.hudi.writeBatchSize}M'
},
{
label: '写限速',
content: '${tableMeta.hudi.writeRateLimit}条/秒'
},
{
label: '压缩策略',
content: '${tableMeta.hudi. compactionStrategy}'
},
{
label: '压缩增量个数',
content: '${tableMeta.hudi.compactionDeltaCommits}'
},
{
label: '压缩增量时间',
content: '${tableMeta.hudi.compactionDeltaSeconds}秒'
},
],
},
{type: 'divider'},
{
type: 'property',
title: 'Yarn 配置',
column: 2,
items: [
{
label: '同步 JM 内存',
content: '${tableMeta.syncYarn.jobManagerMemory}M'
},
{
label: '同步 TM 内存',
content: '${tableMeta.syncYarn.taskManagerMemory}M'
},
{
label: '压缩 JM 内存',
content: '${tableMeta.compactionYarn.jobManagerMemory}M'
},
{
label: '压缩 TM 内存',
content: '${tableMeta.compactionYarn.taskManagerMemory}M'
},
],
},
{type: 'divider'},
{
type: 'property',
title: '其他配置',
column: 4,
items: [
{
label: '指标发布地址',
content: copyField('tableMeta.config.metricPublishUrl'),
span: 4
},
{
label: '指标打点延迟',
content: '${tableMeta.config.metricPublishDelay}'
},
{
label: '指标打点间隔',
content: '${tableMeta.config.metricPublishPeriod}'
},
{
label: '指标发布超时',
content: '${tableMeta.config.metricPublishTimeout}'
},
{
label: '指标批次数量',
content: '${tableMeta.config.metricPublishBatch}'
},
{
label: 'Prometheus 地址',
content: copyField('tableMeta.config.metricPrometheusUrl'),
span: 4
},
{
label: '事件提交服务地址',
content: copyField('tableMeta.config.metricApiUrl', '复制 URL', 130),
span: 4
},
{
label: 'Checkpoint 存储',
content: copyField('tableMeta.config.checkpointRootPath'),
span: 4
},
],
},
{type: 'divider'},
{
type: 'table',
title: '表字段详情',
source: '${tableMeta.fields}',
resizable: false,
columns: [
{name: 'sequence', label: '排序', width: 50, align: 'center'},
{name: 'name', label: '字段名'},
{name: 'type', label: '字段类型', width: 60},
{name: 'length', label: '字段长度', width: 60, align: 'center'},
{
label: '是否主键',
align: 'center',
width: 50,
type: 'tpl',
tpl: "${primaryKey|isTrue:'<i class=\"fa fa-check-circle text-success text-md p-0\"></i>':'<i></i>'|raw}",
},
{
label: '是否分片键',
align: 'center',
width: 70,
type: 'tpl',
tpl: "${partitionKey|isTrue:'<i class=\"fa fa-check-circle text-success text-md p-0\"></i>':'<i></i>'|raw}",
}
],
}
],
}
}
function runModeMapping(field) {
return {
type: 'mapping',
value: `\${${field}}`,
map: {
'ONE_IN_ONE': "<span class='label bg-pink-300'>1对1</span>",
'ALL_IN_ONE': "<span class='label bg-purple-300'>1对多</span>",
'ALL_IN_ONE_BY_TABLE': "<span class='label bg-cyan-300'>按表1对多</span>",
'ALL_IN_ONE_BY_SCHEMA': "<span class='label bg-indigo-300'>按库1对多</span>",
'*': `<span class='label bg-gray-300'>\${${field}}</span>`
},
}
}
function compactionStatusMapping(field) {
return {
type: 'mapping',
value: `\${${field}}`,
map: {
'SCHEDULE': "<span class='label bg-info'>调度</span>",
'START': "<span class='label'>开始</span>",
'FINISH': "<span class='label bg-success'>完成</span>",
'FAIL': "<span class='label bg-danger'>失败</span>",
'*': `<span class='label bg-gray-300'>\${${field}}</span>`
},
}
}
function tagsMapping(field) {
return {
type: 'mapping',
value: `\${${field}}`,
map: {
'NO_COMPACT': "<span class='label bg-info'>不压缩</span>",
'PULSAR_BACKUP': "<span class='label bg-info'>备份Pulsar消息</span>",
'NO_PRE_COMBINE': "<span class='label bg-info'>无预合并</span>",
'NO_IGNORE_FAILED': "<span class='label bg-info'>不忽略写日志错误</span>",
'DISABLE_CHAINING': "<span class='label bg-info'>取消算子合并</span>",
'TRACE_LATEST_OP_TS': "<span class='label bg-info'>跟踪压缩op_ts</span>",
'*': `<span class='label bg-gray-300'>\${${field}}</span>`
},
}
}
function hudiTableTypeMapping(field) {
return {
type: 'mapping',
value: `\${${field}}`,
map: {
'MERGE_ON_READ': "<span class='label bg-info'>MOR</span>",
'COPY_ON_WRITE': "<span class='label bg-info'>COW</span>",
'*': `<span class='label bg-gray-300'>\${${field}}</span>`
},
}
}
function filterModeMapping(field) {
return {
type: 'mapping',
value: `\${${field}}`,
map: {
'NONE': "<span class='label bg-pink-500'>无</span>",
'INCLUDE': "<span class='label bg-purple-500'>包含模式</span>",
'EXCLUDE': "<span class='label bg-cyan-500'>排除模式</span>",
'*': `<span class='label bg-gray-300'>\${${field}}</span>`
},
}
}
function subscriptionTypeMapping(field) {
return {
type: 'mapping',
value: `\${${field}}`,
map: {
'Exclusive': "<span class='label bg-pink-500'>独占</span>",
'Shared': "<span class='label bg-purple-500'>共享</span>",
'Failover': "<span class='label bg-cyan-500'>灾备</span>",
'Key_Shared': "<span class='label bg-green-500'>Key</span>",
'*': `<span class='label bg-gray-300'>\${${field}}</span>`
},
}
}
function publishTypeMapping(field) {
return {
type: 'mapping',
value: `\${${field}}`,
map: {
'Shared': "<span class='label bg-pink-500'>共享</span>",
'Exclusive': "<span class='label bg-purple-500'>独占</span>",
'WaiteForExclusive': "<span class='label bg-cyan-500'>等待独占</span>",
'*': `<span class='label bg-gray-300'>\${${field}}</span>`
},
}
}