function crudCommonOptions() {
return {
affixHeader: false,
stopAutoRefreshWhenModalIsOpen: true,
resizable: false,
syncLocation: false,
silentPolling: true,
}
}
function readOnlyDialogOptions() {
return {
actions: [],
showCloseButton: false,
closeOnEsc: true,
closeOnOutside: true,
disabled: true,
}
}
function paginationCommonOptions(perPage = true, maxButtons = 5) {
let option = {
type: 'pagination',
layout: [
'pager'
],
maxButtons: maxButtons,
showPageInput: false,
perPageAvailable: [10, 15, 20, 50, 100, 200],
}
if (perPage) {
option.layout.push('perPage')
}
return option
}
function paginationTemplate(perPage = 20, maxButtons = 5) {
return {
perPage: perPage,
headerToolbar: [
"reload",
paginationCommonOptions(true, maxButtons),
],
footerToolbar: [
"statistics",
paginationCommonOptions(true, maxButtons),
],
}
}
function timeAndFrom(field, fromNow, emptyText = '未停止', showSource = true) {
let tpl = "${IF(" + field + " === 0, '(" + emptyText + ")', CONCATENATE(''," + fromNow + ",''))}"
if (showSource) {
return {
align: 'center',
width: 80,
type: 'tooltip-wrapper',
content: "${DATETOSTR(DATE(" + field + "))}",
inline: true,
className: 'mr-2',
disabledOn: `\${${field} === 0}`,
body: {
type: 'tpl',
tpl: tpl,
}
}
} else {
return {
align: 'center',
width: 140,
type: 'tpl',
tpl: "${DATETOSTR(DATE(" + field + "))}",
}
}
}
function applicationLogDialog() {
return {
type: 'action',
level: 'link',
actionType: 'dialog',
dialog: {
title: '应用日志',
size: 'xl',
actions: [],
body: [
{
type: 'service',
api: {
method: 'GET',
url: '${base}/log/query_application_log',
data: {
application_id: '${id}',
}
},
body: [
{
disabled: true,
type: 'editor',
name: 'detail',
label: '应用日志',
size: 'xxl',
placeholder: '没有内容',
options: {
wordWrap: 'on',
}
}
]
},
]
}
}
}
function yarnQueueCrud(clusters = undefined, queueNames = undefined) {
return {
type: 'crud',
api: {
method: 'get',
url: '${base}/yarn/queue_list',
data: {
clusters: '${cluster|default:undefined}',
names: '${queueName|default:undefined}'
},
},
affixHeader: false,
defaultParams: {
...(clusters ? {cluster: clusters} : {}),
...(queueNames ? {queueName: queueNames} : {}),
},
interval: 10000,
syncLocation: false,
silentPolling: true,
headerToolbar: [
"reload",
],
columns: [
{
name: 'queueName',
label: '队列名称',
width: 130,
type: 'tooltip-wrapper',
body: '${TRUNCATE(queueName, 20)}',
content: '${queueName}',
},
{
label: '当前容量',
type: 'progress',
value: '${ROUND((absoluteUsedCapacity * 100 / absoluteMaxCapacity), 0)}',
stripe: true,
animate: true,
showLabel: false,
map: [
{
value: 30,
color: "#28a745"
}, {
value: 90,
color: "#007bff"
},
{
value: 100,
color: "#dc3545"
}
],
},
{
label: '进度',
width: 35,
align: 'center',
type: 'tpl',
tpl: '${ROUND((absoluteUsedCapacity * 100 / absoluteMaxCapacity), 0)}%',
},
{
type: "operation",
label: "操作",
width: 100,
fixed: 'right',
buttons: [
{
label: "详情",
type: "button",
level: "link",
tooltip: '查看队列详情',
visibleOn: '${!root}',
actionType: 'dialog',
dialog: {
closeOnEsc: true,
closeOnOutside: true,
size: 'md',
close: true,
title: '队列详情',
body: {
type: 'property',
items: [
{label: 'CPU', content: '${resourcesUsed.vCores}'},
// 有空看看这个值的单位
{label: '内存', content: '${resourcesUsed.memory}', span: 2},
{label: '容量(%)', content: '${capacity}'},
{label: '最大容量(%)', content: '${maxCapacity}'},
{label: '已用容量(%)', content: '${usedCapacity}'},
{label: '绝对容量(%)', content: '${absoluteCapacity}'},
{label: '绝对最大容量(%)', content: '${absoluteMaxCapacity}'},
{label: '绝对已用容量(%)', content: '${absoluteUsedCapacity}'},
{label: '应用数量', content: '${numApplications}', span: 3},
{label: '最大应用数量', content: '${maxApplications}'},
{label: '活跃应用数量', content: '${numActiveApplications}'},
{label: '等待应用数量', content: '${numPendingApplications}'},
{label: '容器数量', content: '${numContainers}', span: 3},
{label: '已分配容器数量', content: '${numApplications}'},
{label: '预留容器数量数量', content: '${numApplications}'},
{label: '等待容器数量', content: '${numApplications}'},
]
},
actions: []
}
},
{
visibleOn: "${webUrl}",
label: "管理页面",
type: "action",
level: "link",
tooltip: '打开管理页面',
actionType: 'url',
url: '${webUrl}',
blank: true,
}
]
}
]
}
}
function yarnCrudColumns() {
return [
{
label: '名称',
className: 'nowrap',
type: 'tpl',
tpl: "${IF(syncApplication, 'S', IF(compactionApplication, 'C', ''))}${IF(hudiApplication, '', '')}${IF(syncApplication, flinkJobName, IF(compactionApplication, alias, name))}",
backgroundScale: {
min: 0.001,
max: 1.000,
source: '${compactionCompletionRatio}',
colors: [
'#FFFFFF',
'#DD4150',
]
}
},
{
name: 'cluster',
label: '集群',
width: 65,
align: 'center',
type: 'tpl',
tpl: '${cluster}'
},
{
label: '用户',
width: 80,
type: 'tooltip-wrapper',
body: '${TRUNCATE(user, 8)}',
content: '${user}',
align: 'center',
},
{
name: 'state',
label: '运行状态',
width: 70,
align: 'center',
type: 'mapping',
canAccessSuperData: false,
map: {
'NEW': '创建中',
'NEW_SAVING': '已创建',
'SUBMITTED': '已提交',
'ACCEPTED': '调度中',
'RUNNING': '运行中',
'FINISHED': '已结束',
'FAILED': '已失败',
'KILLED': '被停止',
'*': '${state}'
},
filterable: {
multiple: true,
options: [
{label: '创建中', value: 'NEW'},
{label: '已创建', value: 'NEW_SAVING'},
{label: '已提交', value: 'SUBMITTED'},
{label: '调度中', value: 'ACCEPTED'},
{label: '运行中', value: 'RUNNING'},
{label: '已结束', value: 'FINISHED'},
{label: '已失败', value: 'FAILED'},
{label: '被停止', value: 'KILLED'},
]
}
},
{
name: 'finalStatus',
label: '最终状态',
width: 70,
align: 'center',
type: 'mapping',
canAccessSuperData: false,
map: {
'UNDEFINED': '运行',
'SUCCEEDED': '成功',
'FAILED': '失败',
'KILLED': '终止',
'ENDED': '结束',
'*': '${state}'
},
filterable: {
multiple: true,
options: [
{label: '运行', value: 'UNDEFINED'},
{label: '成功', value: 'SUCCEEDED'},
{label: '失败', value: 'FAILED'},
{label: '终止', value: 'KILLED'},
{label: '结束', value: 'ENDED'},
]
}
},
{
name: 'startedTime',
label: '启动时间',
...timeAndFrom('startedTime', 'startTimeFromNow'),
sortable: true,
canAccessSuperData: false,
},
{
name: 'finishedTime',
label: '停止时间',
...timeAndFrom('finishedTime', 'finishTimeFromNow'),
sortable: true,
align: 'center',
canAccessSuperData: false,
},
{
type: "operation",
width: 160,
label: "操作",
fixed: 'right',
className: 'nowrap',
buttons: [
/*{
disabled: true,
label: "停止",
type: "button",
level: "link",
tooltip: '从队列中移除该任务',
actionType: 'dialog',
dialog: {
closeOnEsc: true,
closeOnOutside: true,
size: 'sm',
close: true,
title: '停止确认',
body: '确认停止任务:
${name}
?',
actions: [
{
type: 'button',
label: '取消',
actionType: 'cancel'
},
{
type: 'button',
label: '确认移除',
level: 'danger',
actionType: 'ajax',
api: '',
primary: true,
message: {
success: '操作成功',
failed: '操作失败'
}
}
]
}
},*/
{
visibleOn: 'flinkJobId',
label: 'FID',
type: 'action',
actionType: 'copy',
content: '${flinkJobId}',
level: 'link',
tooltip: '${flinkJobId}'
},
{
visibleOn: 'alias',
label: 'ALIAS',
type: 'action',
actionType: 'copy',
content: '${alias}',
level: 'link',
tooltip: '${alias}'
},
{
label: 'ID',
type: 'action',
actionType: 'copy',
content: '${id}',
level: 'link',
tooltip: '${id}'
},
{
disabledOn: '${trackingUrl == null}',
disabledTip: '无应用页面',
label: "页面",
type: "action",
level: "link",
tooltip: '打开应用页面: ${trackingUrl}',
actionType: 'url',
url: '${trackingUrl}',
blank: true,
},
{
label: '日志',
...applicationLogDialog(),
}
]
}
]
}
function simpleYarnDialog(cluster, title, filterField) {
return {
title: title,
actions: [],
data: {
base: '${base}',
name: `\${${filterField}}`,
flinkJob: '${flinkJob}',
tableMeta: '${tableMeta}',
},
size: 'xl',
body: [
{
type: 'service',
api: {
method: 'get',
url: '${base}/yarn/job_current',
data: {
clusters: `${cluster}`,
name: '${name}',
}
},
silentPolling: false,
body: [
{
type: 'wrapper',
size: 'none',
visibleOn: '${hasCurrent}',
body: [
{
type: 'service',
api: {
method: 'get',
url: '${base}/flink/overview',
data: {
url: '${current.trackingUrl}'
}
},
silentPolling: false,
body: [
{
type: 'property',
title: 'Flink 基本信息',
column: 4,
items: [
{label: 'Flink 版本', content: '${flinkVersion}'},
{label: 'Flink 小版本', content: '${flinkCommit}', span: 3},
{label: '运行中', content: '${jobsRunning}'},
{label: '已结束', content: '${jobsFinished}'},
{label: '已失败', content: '${jobsFailed}'},
{label: '被取消', content: '${jobsCanceled}'},
{
label: 'Slot (可用/总数)',
content: '${slotsAvailable}/${slotsTotal}',
span: 4
},
]
},
]
},
{
type: 'service',
api: {
method: 'get',
url: '${base}/flink/jobs',
data: {
url: '${current.trackingUrl}',
schema: '${tableMeta.schema}',
table: "${tableMeta.table}",
mode: "${flinkJob.runMode}",
}
},
silentPolling: false,
body: [
{
type: 'table',
title: '任务详情',
source: '${items}',
affixHeader: false,
columns: [
{
name: 'name',
label: '名称',
width: 2000,
},
{
label: 'Checkpoint',
width: 60,
align: 'center',
fixed: 'right',
type: 'tpl',
tpl: "${IF(checkpointMetrics, checkpointMetrics.complete + '/' + checkpointMetrics.inProgress +'/' + checkpointMetrics.failed, '-')}",
},
{
name: 'metrics.readRecords',
label: '读记录数',
width: 60,
align: 'center',
fixed: 'right',
},
{
name: 'metrics.writeRecords',
label: '写记录数',
width: 60,
align: 'center',
fixed: 'right',
},
{
label: '操作',
width: 60,
align: 'center',
fixed: 'right',
type: 'wrapper',
size: 'none',
body: [
{
disabled: true,
type: 'button',
label: '详情',
level: 'link',
size: 'xs',
actionType: 'url',
blank: true,
url: '${page}',
}
],
},
]
}
]
},
]
},
{
type: 'tpl',
tpl: '没有正在运行的任务',
visibleOn: '${!hasCurrent}',
},
],
},
{type: 'divider'},
{
type: 'crud',
api: {
method: 'get',
url: `\${base}/yarn/job_list`,
data: {
clusters: `${cluster}`,
page: '${page|default:undefined}',
count: '${perPage|default:undefined}',
order: '${orderBy|default:undefined}',
direction: '${orderDir|default:undefined}',
filter_state: '${state|default:undefined}',
filter_final_status: '${finalStatus|default:undefined}',
search_id: '${id|default:undefined}',
search_name: '${name}',
precise: true,
}
},
affixHeader: false,
interval: 10000,
syncLocation: false,
silentPolling: true,
resizable: false,
perPage: 10,
headerToolbar: [
"reload",
paginationCommonOptions(),
],
footerToolbar: [],
columns: yarnCrudColumns(),
}
],
}
}
function copyField(field, tips = '复制', ignoreLength = 0) {
let tpl = ignoreLength === 0 ? `\${${field}}` : `\${TRUNCATE(${field}, ${ignoreLength})}`
return {
type: 'wrapper',
size: 'none',
body: [
{
type: 'tpl',
className: 'mr-1',
tpl: tpl,
},
{
type: 'action',
level: 'link',
label: '',
icon: 'fa fa-copy',
size: 'xs',
actionType: 'copy',
content: `\$${field}`,
tooltip: `${tips}`,
},
]
}
}
function flinkJobProperty(id, name, runMode) {
return {
type: 'property',
title: 'Flink Job 配置',
items: [
{label: 'ID', content: copyField(id, '复制 ID')},
{label: '任务名称', content: copyField(name, '复制任务名')},
{
label: '任务模式',
content: {
...mappingField(`${runMode}`, runModeMapping),
}
},
],
}
}
function runMetaProperty(runMode) {
return {
type: 'property',
title: `${runMode} 运行时信息`,
items: [
{label: '运行集群', content: `\${${runMode}Runtime.cluster}`},
{label: '运行主机', content: copyField(`${runMode}Runtime.host`)},
{label: '进程', content: copyField(`${runMode}Runtime.jvmPid`)},
{label: '任务 ID', content: copyField(`${runMode}Runtime.applicationId`), span: 2},
{label: 'Jar 版本', content: `\${${runMode}Runtime.executorVersion}`},
{label: '任务名称', content: `\${${runMode}Runtime.flinkJobName}`},
{label: '容器 ID', content: `\${${runMode}Runtime.containerId}`, span: 2},
{label: '容器路径', content: copyField(`${runMode}Runtime.containerPath`, undefined, 120), span: 3},
]
}
}
function statisticsProperty(title, statistic) {
return {
type: 'property',
title: title,
column: 3,
items: [
{label: '扫描总时间', content: `\${${statistic}.totalScanTime}`},
{label: '压缩日志文件总数', content: `\${${statistic}.totalLogFilesCompacted}`},
{label: '压缩日志文件大小', content: `\${${statistic}.totalLogFilesSize}`},
{label: '删除记录数', content: `\${${statistic}.totalRecordsDeleted}`},
{label: '更新记录数', content: `\${${statistic}.totalRecordsUpdated}`},
{label: '压缩日志数', content: `\${${statistic}.totalRecordsCompacted}`},
],
}
}
function flinkJobDialog() {
return {
title: 'Flink job 详情',
actions: [],
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
size: 'md',
body: [
flinkJobProperty('flinkJobId', 'flinkJob.name', 'flinkJob.runMode'),
{type: 'divider'},
{
type: 'action',
label: '打开同步详情',
actionType: 'dialog',
dialog: simpleYarnDialog('b5-sync', '同步详情', 'syncJobName')
},
{type: 'divider'},
{
type: 'service',
api: {
method: 'get',
url: '${base}/table/list_metas',
data: {
flink_job_id: '${flinkJobId}'
},
},
canAccessSuperData: true,
body: [
{
type: "table",
title: "包含 Hudi 同步表",
source: "${items}",
columns: [
{
label: '别名',
type: 'wrapper',
size: 'none',
body: [
{
type: 'action',
level: 'link',
label: '${tableMeta.alias}',
size: 'xs',
actionType: 'dialog',
tooltip: '查看详情',
dialog: tableMetaDialog(),
},
{
type: 'action',
level: 'link',
label: '',
icon: 'fa fa-copy',
size: 'xs',
actionType: 'copy',
content: '${tableMeta.alias}',
tooltip: '复制别名',
},
],
},
{
label: '并行度',
name: 'tableMeta.hudi.writeTasks',
align: 'center',
},
{
label: 'Bucket 数量',
name: 'tableMeta.hudi.bucketIndexNumber',
align: 'center',
},
{
label: '写并行度',
name: 'tableMeta.hudi.writeTasks',
align: 'center',
},
{
label: '压缩并行度',
name: 'tableMeta.hudi.compactionTasks',
align: 'center',
},
]
}
]
}
]
}
}
function timelineColumns() {
return [
{
name: 'timestamp',
label: '时间点',
width: 150,
sortable: true,
},
{
name: 'action',
label: '类型',
width: 100,
...mappingField('action', hudiTimelineActionMapping),
filterable: filterableField(hudiTimelineActionMapping, false),
},
{
name: 'state',
label: ' 状态',
width: 80,
align: 'center',
...mappingField('state', hudiTimelineStateMapping),
filterable: filterableField(hudiTimelineStateMapping, true),
},
{
name: 'fileTime',
label: ' 文件时间',
width: 150,
align: 'center',
type: 'tpl',
tpl: "${DATETOSTR(DATE(fileTime), 'YYYY-MM-DD HH:mm:ss')}",
},
{
name: 'fileName',
label: '文件名',
type: 'wrapper',
size: 'none',
className: 'nowrap',
body: [
{
type: 'tpl',
tpl: '${fileName}'
},
{
visibleOn: "action === 'compaction'",
type: 'action',
icon: 'fa fa-eye ml-1',
level: 'link',
tooltip: '查看压缩计划',
size: 'xs',
actionType: 'dialog',
dialog: {
title: '压缩计划详情',
actions: [],
size: 'lg',
body: {
type: 'crud',
api: {
method: 'get',
url: '${base}/hudi/read_compaction_plan',
data: {
hdfs: '${hdfs|default:undefined}',
flink_job_id: '${flinkJobId|default:undefined}',
alias: '${tableMeta.alias|default:undefined}',
instant: '${timestamp|default:undefined}',
},
adaptor: (payload, response) => {
return {
items: (payload['data']['operations'] ? payload['data']['operations'] : [])
.map(operation => {
if (operation['deltaFilePaths']) {
operation.deltaFilePaths = operation.deltaFilePaths
.map(p => {
return {
path: p
}
})
}
return operation
})
}
}
},
...crudCommonOptions(),
loadDataOnce: true,
...paginationTemplate(20, 8),
columns: [
{
name: 'fileId',
label: '文件 ID',
searchable: true,
},
{
name: 'baseInstantTime',
label: '版本时间点',
width: 120,
align: 'center',
},
{
name: 'partitionPath',
label: '分区',
width: 50,
align: 'center',
},
{
label: '读/写(MB)/数',
type: 'tpl',
tpl: '${metrics[\'TOTAL_IO_READ_MB\']} / ${metrics[\'TOTAL_IO_WRITE_MB\']} / ${metrics[\'TOTAL_LOG_FILES\']}',
align: 'center',
width: 90,
},
{
type: 'operation',
label: '操作',
width: 150,
buttons: [
{
label: '数据文件名',
type: 'action',
level: 'link',
size: 'xs',
actionType: 'copy',
content: '${dataFilePath}',
tooltip: '复制 ${dataFilePath}',
},
{
label: '日志文件',
type: 'action',
level: 'link',
size: 'xs',
actionType: 'dialog',
dialog: {
title: '操作日志文件列表',
size: 'md',
...readOnlyDialogOptions(),
body: {
type: 'crud',
source: '${deltaFilePaths}',
mode: 'list',
...crudCommonOptions(),
loadDataOnce: true,
title: null,
listItem: {
title: '${path}'
}
}
}
}
]
}
]
}
}
},
{
visibleOn: "action === 'rollback'",
type: 'action',
icon: 'fa fa-eye ml-1',
level: 'link',
tooltip: '查看回滚计划',
size: 'xs',
actionType: 'dialog',
dialog: {
title: '回滚计划详情',
actions: [],
size: 'lg',
body: {
type: 'service',
api: {
method: 'get',
url: '${base}/hudi/read_rollback_plan',
data: {
hdfs: '${hdfs|default:undefined}',
flink_job_id: '${flinkJobId|default:undefined}',
alias: '${tableMeta.alias|default:undefined}',
instant: '${timestamp|default:undefined}',
},
},
body: [
{
type: 'property',
title: '回滚目标',
items: [
{
label: 'Action',
content: {
value: '${instantToRollback.action}',
...mappingField('instantToRollback.action', hudiTimelineActionMapping)
},
},
{label: '时间点', content: '${instantToRollback.commitTime}', span: 2},
],
},
{
type: 'crud',
source: '${rollbackRequests}',
...crudCommonOptions(),
columns: [
{
name: 'fileId',
label: '文件 ID',
searchable: true,
},
{
name: 'partitionPath',
label: '分区',
width: 50,
},
{
name: 'latestBaseInstant',
label: '数据文件版本',
},
]
}
]
}
}
},
{
visibleOn: "action === 'clean'",
type: 'action',
icon: 'fa fa-eye ml-1',
level: 'link',
tooltip: '查看清理计划',
size: 'xs',
actionType: 'dialog',
dialog: {
title: '清理计划详情',
actions: [],
size: 'xl',
body: {
type: 'service',
api: {
method: 'get',
url: '${base}/hudi/read_cleaner_plan',
data: {
hdfs: '${hdfs|default:undefined}',
flink_job_id: '${flinkJobId|default:undefined}',
alias: '${tableMeta.alias|default:undefined}',
instant: '${timestamp|default:undefined}',
},
adaptor: (payload, response) => {
if (payload.data['filePathsToBeDeletedPerPartition']) {
let map = payload.data['filePathsToBeDeletedPerPartition']
let list = []
Object.keys(map)
.forEach(key => {
map[key].forEach(file => list.push({
partitionPath: key,
file: file['filePath'],
}))
})
payload.data['filePathsToBeDeletedPerPartition'] = list
}
return payload
}
},
body: [
{
type: 'property',
title: '最早回滚时间点',
items: [
{label: '策略', content: '${policy}', span: 3},
{
label: '操作',
content: {
value: '${earliestInstantToRetain.action}',
...mappingField('earliestInstantToRetain.action', hudiTimelineActionMapping)
},
},
{
label: '状态',
content: {
value: '${earliestInstantToRetain.state}',
...mappingField('earliestInstantToRetain.state', hudiTimelineStateMapping)
},
},
{label: '时间点', content: '${earliestInstantToRetain.timestamp}'},
],
},
{
type: 'crud',
source: '${filePathsToBeDeletedPerPartition}',
...crudCommonOptions(),
...paginationTemplate(20, 8),
loadDataOnce: true,
title: '分区删除文件',
columns: [
{
name: 'partitionPath',
label: '分区',
width: 50,
align: 'center',
},
{
name: 'file',
label: '清理文件',
className: 'nowrap',
},
{
type: 'operation',
label: '操作',
width: 100,
buttons: [
{
label: '复制路经',
type: 'action',
level: 'link',
size: 'xs',
actionType: 'copy',
content: '${file}',
tooltip: '复制 ${file}',
},
]
}
]
}
]
}
}
},
]
},
{
name: 'type',
label: '来源',
width: 60,
align: 'center',
...mappingField('type', hudiTimelineTypeMapping),
filterable: filterableField(hudiTimelineTypeMapping, true),
},
]
}
function tableMetaDialog() {
return {
title: 'Table 详情',
actions: [],
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
size: 'lg',
body: [
{
type: 'wrapper',
size: 'none',
body: [
{
type: 'tpl',
className: 'block font-bold mb-2',
tpl: '常用操作',
},
{
type: 'button-group',
tiled: false,
buttons: [
{
label: '同步情况',
type: 'button',
icon: 'fa fa-arrows-rotate',
actionType: 'dialog',
dialog: simpleYarnDialog('b5-sync', '同步详情', 'syncJobName')
},
{
label: '压缩情况',
type: 'action',
icon: 'fa fa-minimize',
actionType: 'dialog',
dialog: simpleYarnDialog('b1', '压缩详情', 'compactionJobName')
},
{
label: '历史压缩',
type: 'action',
icon: 'fa fa-list',
actionType: 'dialog',
dialog: {
title: 'Hudi 表时间线',
actions: [],
size: 'lg',
body: {
type: 'crud',
api: {
method: 'get',
url: '${base}/table/list_compaction_metrics',
data: {
page: '${page|default:undefined}',
count: '${perPage|default:undefined}',
order: '${orderBy|default:update_time}',
direction: '${orderDir|default:DESC}',
search_flink_job_id: '${flinkJobId|default:undefined}',
search_alias: '${tableMeta.alias|default:undefined}',
filter_completes: '${complete|default:undefined}',
},
defaultParams: {
filter_type: 'active',
},
},
...crudCommonOptions(),
perPage: 15,
headerToolbar: [
"reload",
paginationCommonOptions(),
],
footerToolbar: [
paginationCommonOptions(),
],
columns: [
{
name: 'compactionPlanInstant',
label: '压缩时间点',
width: 160,
...copyField('compactionPlanInstant')
},
{
name: 'cluster',
label: '集群',
width: 65,
align: 'center',
type: 'tpl',
tpl: '${cluster}'
},
{
name: 'applicationId',
label: '应用',
...copyField('applicationId')
},
{
name: 'complete',
label: '状态',
...mappingField('complete', compactionMetricsStateMapping),
filterable: filterableField(compactionMetricsStateMapping, false),
},
{
name: 'startedTime',
label: '启动时间',
...timeAndFrom('startedTime', 'startedTimeFromNow'),
sortable: true,
canAccessSuperData: false,
},
{
name: 'finishedTime',
label: '停止时间',
...timeAndFrom('finishedTime', 'finishedTimeFromNow'),
sortable: true,
align: 'center',
canAccessSuperData: false,
},
{
type: "operation",
width: 50,
label: "操作",
fixed: 'right',
className: 'nowrap',
buttons: [
{
label: '详情',
type: 'action',
level: 'link',
actionType: 'dialog',
dialog: {
title: '压缩详情',
size: 'xl',
actions: [],
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
body: [
statisticsProperty('压缩预扫描', 'before'),
{type: 'divider'},
statisticsProperty('压缩成果', 'after'),
]
}
}
]
}
],
}
}
},
{
type: 'button',
label: '时间线',
icon: 'fa fa-timeline',
actionType: 'dialog',
dialog: {
title: 'Hudi 表时间线',
actions: [],
size: 'lg',
body: {
type: 'crud',
api: {
method: 'get',
url: '${base}/hudi/timeline/list',
data: {
page: '${page|default:undefined}',
count: '${perPage|default:undefined}',
order: '${orderBy|default:undefined}',
direction: '${orderDir|default:undefined}',
flink_job_id: '${flinkJobId|default:undefined}',
alias: '${tableMeta.alias|default:undefined}',
filter_type: "${type|default:active}",
filter_action: '${action|default:undefined}',
filter_state: '${state|default:undefined}',
},
},
...crudCommonOptions(),
perPage: 15,
headerToolbar: [
"reload",
paginationCommonOptions(),
],
footerToolbar: [
paginationCommonOptions(),
],
columns: timelineColumns(),
}
}
},
{
type: 'button',
label: 'Pulsar 队列',
icon: 'fa fa-message',
actionType: 'dialog',
dialog: {
title: '队列详情',
actions: [],
size: 'lg',
body: {
type: 'service',
api: {
method: 'get',
url: '${base}/pulsar/topic',
data: {
pulsar_url: '${tableMeta.pulsarAddress|default:undefined}',
topic: '${tableMeta.topic|default:undefined}',
}
},
body: [
{
type: 'property',
title: '基本信息',
items: [
{label: 'Topic', content: copyField('name'), span: 2},
{label: '最末位移', content: copyField('lastMessageId')},
],
},
{type: 'divider'},
{
type: 'property',
title: '指标信息',
column: 4,
items: [
{label: '入队列消息速率', content: '${state.messageRateIn}'},
{
label: '出队列消息速率',
content: '${state.messageRateOut}'
},
{
label: '入队列消息吞吐量',
content: '${state.messageThroughputIn}'
},
{
label: '出队列消息吞吐量',
content: '${state.messageThroughputOut}'
},
{
label: '入队列消息数量',
content: '${state.messageInCounter}'
},
{
label: '出队列消息数量',
content: '${state.messageOutCounter}'
},
{
label: '入队列消息字节数',
content: '${state.byteInCounter}'
},
{
label: '出队列消息字节数',
content: '${state.byteOutCounter}'
},
{label: '存储消息大小', content: '${state.storageSize}'},
{label: '积压消息大小', content: '${state.backlogSize}'},
{
label: '平均消息大小',
content: '${state.averageMessageSize}'
},
],
},
{type: 'divider'},
{
type: 'table',
title: '消费者们',
source: '${subscriptionStateVOS}',
itemAction: {
type: 'action',
actionType: 'dialog',
dialog: {
title: '详情',
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
actions: [],
size: 'md',
body: [
{
type: 'property',
column: 1,
items: [
{
label: '在此订阅上传递的消息的总速率(msg/s)',
content: '${messageRateOut}'
},
{
label: '此订阅提供的总吞吐量(字节/秒)',
content: '${messageThroughputOut}'
},
{
label: '传送给消费者的总字节数(字节)',
content: '${bytesOutCounter}'
},
{
label: '传递给消费者的消息总数(msg)',
content: '${messageOutCounter}'
},
{
label: '此订阅上重新传递的消息的总速率(msg/s)',
content: '${messageRateRedeliver}'
},
{
label: '分块消息调度速率',
content: '${chunkedMessageRate}'
},
{
label: '积压的大小(字节)',
content: '${backlogSize}'
},
{
label: '积压中不包含延迟消息的消息数',
content: '${messageBacklogNoDelayed}'
},
{
label: '验证订阅是否由于达到未确认消息的阈值而被阻止',
content: '${blockedSubscriptionOnUnackedMessages}'
},
{
label: '当前正在跟踪的延迟消息数',
content: '${messageDelayed}'
},
{
label: '订阅的未确认消息数',
content: '${unackedMessages}'
},
{
label: '单个活跃消费者订阅处于活跃状态的消费者名称(故障转移、独占)',
content: '${activeConsumerName}'
},
{
label: '此订阅上过期的消息总速率(msg/s)',
content: '${messageRateExpired}'
},
{
label: '此订阅上过期的消息总数',
content: '${totalMessageExpired}'
},
{
label: '最后一条消息过期时间',
content: {
type: 'tpl',
tpl: '${lastExpireTimestamp|date:YYYY-MM-DD HH\\:mm\\:ss:x}',
}
},
{
label: '上次接收的消费流命令时间',
content: {
type: 'tpl',
tpl: '${lastConsumedFlowTimestamp|date:YYYY-MM-DD HH\\:mm\\:ss:x}',
}
},
{
label: '上次消费消息时间',
content: {
type: 'tpl',
tpl: '${lastConsumedTimestamp|date:YYYY-MM-DD HH\\:mm\\:ss:x}',
}
},
{
label: '上次确认消息时间',
content: {
type: 'tpl',
tpl: '${lastAckedTimestamp|date:YYYY-MM-DD HH\\:mm\\:ss:x}',
}
},
{
label: '此订阅是持久订阅还是临时订阅',
content: '${durable}'
},
{
label: '标记订阅状态在不同区域之间保持同步',
content: '${replicated}'
},
]
},
]
}
},
columns: [
{
name: 'name',
label: "订阅名称",
...copyField('name')
},
{
name: 'type',
label: '订阅类型',
fixed: 'right',
...mappingField('type', subscriptionTypeMapping)
},
{name: 'messageBacklog', label: '积压', fixed: 'right'},
]
},
{type: 'divider'},
{
type: 'table',
title: '生产者们',
source: '${state.publishers}',
itemAction: {
type: 'action',
actionType: 'dialog',
dialog: {
title: '详情',
closeOnEsc: true,
closeOnOutside: true,
showCloseButton: false,
actions: [],
size: 'md',
body: [
{
type: 'property',
column: 1,
items: [
{
label: '发布消息速率(msg/s)',
content: '${messageRateIn}'
},
{
label: '发布消息吞吐量(字节/秒)',
content: '${messageThroughputIn}'
},
{
label: '消息平均大小(字节)',
content: '${averageMessageSize}'
},
{
label: '接收到的分块消息总数(msg)',
content: '${chunkedMessageRate}'
},
{label: '生产者地址', content: '${address}'},
{label: '客户端版本', content: '${clientVersion}'},
]
},
]
}
},
columns: [
{
name: 'producerId',
label: "ID",
width: 50,
},
{
name: 'producerName',
label: "名称",
...copyField('producerName'),
},
{
name: 'connectedSince',
label: "连接时间",
type: 'tpl',
tpl: '${connectedSince}'
},
{
name: 'accessMode',
label: '发布类型',
fixed: 'right',
...mappingField('accessMode', publishTypeMapping),
},
]
},
]
}
}
},
{
type: 'button',
label: 'Hudi 表结构',
icon: 'fa fa-table',
actionType: 'dialog',
dialog: {
title: 'Hudi 表结构',
actions: [],
body: {
type: 'service',
api: {
method: 'get',
url: '${base}/hudi/schema',
data: {
flink_job_id: '${flinkJobId|default:undefined}',
alias: '${tableMeta.alias|default:undefined}',
},
},
body: {
type: 'page',
body: {
type: 'json',
source: '${detail}',
levelExpand: 3,
}
}
}
}
},
]
},
]
},
{type: 'divider', visibleOn: '${syncRuntime}'},
{
visibleOn: '${syncRuntime}',
...runMetaProperty('sync'),
},
{type: 'divider', visibleOn: '${compactionRuntime}'},
{
visibleOn: '${compactionRuntime}',
...runMetaProperty('compaction'),
},
{type: 'divider'},
flinkJobProperty('flinkJobId', 'flinkJob.name', 'flinkJob.runMode'),
{type: 'divider'},
{
type: 'property',
title: 'Table 配置',
items: [
{label: '别名', content: copyField('tableMeta.alias', '复制别名')},
{
label: '分片表',
content: '${IF(tableMeta.type === \'sharding\', \'是\', \'否\')}'
},
{label: '分区字段', content: copyField('tableMeta.partitionField', '复制分区字段')},
{label: '源端', content: copyField('tableMeta.source', '复制源端')},
{label: '源库名', content: copyField('tableMeta.schema', '复制库名')},
{label: '源表名', content: copyField('tableMeta.table', '复制表名')},
{label: '源表类型', content: '${tableMeta.sourceType}'},
{label: '优先级', content: '${tableMeta.priority}', span: 2},
{
label: '标签',
content: {
type: 'each',
source: '${SPLIT(tableMeta.tags, ",")}',
items: mappingField('item', tagsMapping),
},
span: 3,
},
{label: '订阅 Topic', content: copyField('tableMeta.topic', '复制 Topic'), span: 3},
{
label: 'Pulsar 地址',
content: copyField('tableMeta.pulsarAddress', '复制地址', 130),
span: 3
},
{label: '过滤模式', content: mappingField('tableMeta.filterType', filterModeMapping)},
{label: '过滤字段', content: '${tableMeta.filterField}', span: 2},
{
label: '过滤内容',
content: {
type: 'each',
source: '${SPLIT(tableMeta.filterValues, ",")}',
items: {
type: 'tpl',
tpl: "${item}",
}
},
span: 3,
},
],
},
{type: 'divider'},
{
type: 'property',
title: 'Hudi 配置',
items: [
{label: '表类型', content: mappingField('tableMeta.hudi.targetTableType', hudiTableTypeMapping)},
{label: '库名', content: copyField('tableMeta.hudi.targetDataSource', '复制库名')},
{label: '表名', content: copyField('tableMeta.hudi.targetTable', '复制表名')},
{
label: 'HDFS',
content: copyField('tableMeta.hudi.targetHdfsPath', '复制路径'),
span: 3
},
{
label: 'Bucket 数量',
content: '${tableMeta.hudi.bucketIndexNumber}'
},
{
label: '保留文件版本',
content: '${tableMeta.hudi.keepFileVersion}'
},
{
label: '保留提交个数',
content: '${tableMeta.hudi.keepCommitVersion}'
},
{label: '写并行度', content: '${tableMeta.hudi.writeTasks}'},
{label: '读并行度', content: '${tableMeta.hudi.sourceTasks}'},
{
label: '压缩并行度',
content: '${tableMeta.hudi.compactionTasks}'
},
{
label: '写任务最大内存',
content: '${tableMeta.hudi. writeTaskMaxMemory}M'
},
{
label: '写批次大小',
content: '${tableMeta.hudi.writeBatchSize}M'
},
{
label: '写限速',
content: '${tableMeta.hudi.writeRateLimit}条/秒'
},
{
label: '压缩策略',
content: '${tableMeta.hudi. compactionStrategy}'
},
{
label: '压缩增量个数',
content: '${tableMeta.hudi.compactionDeltaCommits}'
},
{
label: '压缩增量时间',
content: '${tableMeta.hudi.compactionDeltaSeconds}秒'
},
],
},
{type: 'divider'},
{
type: 'property',
title: 'Yarn 配置',
column: 2,
items: [
{
label: '同步 JM 内存',
content: '${tableMeta.syncYarn.jobManagerMemory}M'
},
{
label: '同步 TM 内存',
content: '${tableMeta.syncYarn.taskManagerMemory}M'
},
{
label: '压缩 JM 内存',
content: '${tableMeta.compactionYarn.jobManagerMemory}M'
},
{
label: '压缩 TM 内存',
content: '${tableMeta.compactionYarn.taskManagerMemory}M'
},
],
},
{type: 'divider'},
{
type: 'property',
title: '其他配置',
column: 4,
items: [
{
label: '指标发布地址',
content: copyField('tableMeta.config.metricPublishUrl'),
span: 4
},
{
label: '指标打点延迟',
content: '${tableMeta.config.metricPublishDelay}'
},
{
label: '指标打点间隔',
content: '${tableMeta.config.metricPublishPeriod}'
},
{
label: '指标发布超时',
content: '${tableMeta.config.metricPublishTimeout}'
},
{
label: '指标批次数量',
content: '${tableMeta.config.metricPublishBatch}'
},
{
label: 'Prometheus 地址',
content: copyField('tableMeta.config.metricPrometheusUrl'),
span: 4
},
{
label: '事件提交服务地址',
content: copyField('tableMeta.config.metricApiUrl', '复制 URL', 130),
span: 4
},
{
label: 'Checkpoint 存储',
content: copyField('tableMeta.config.checkpointRootPath'),
span: 4
},
{
label: 'Zookeeper 地址',
content: copyField('tableMeta.config.zookeeperUrl'),
span: 4
},
],
},
{type: 'divider'},
{
type: 'table',
title: '表字段详情',
source: '${tableMeta.fields}',
resizable: false,
columns: [
{name: 'sequence', label: '排序', width: 50, align: 'center'},
{name: 'name', label: '字段名'},
{name: 'type', label: '字段类型', width: 100},
{name: 'length', label: '字段长度', width: 60, align: 'center'},
{name: 'scala', label: '字段精度', width: 60, align: 'center'},
{
label: '是否主键',
align: 'center',
width: 50,
type: 'tpl',
tpl: "${primaryKey|isTrue:'':''|raw}",
},
{
label: '是否分片键',
align: 'center',
width: 70,
type: 'tpl',
tpl: "${partitionKey|isTrue:'':''|raw}",
}
],
}
],
}
}
function mappingItem(label, value, color = 'bg-info') {
return {
label: label,
value: value,
color: color,
}
}
let runModeMapping = [
mappingItem('1对1', 'ONE_IN_ONE', 'bg-pink-300'),
mappingItem('1对多', 'ALL_IN_ONE', 'bg-purple-300'),
mappingItem('按表1对多', 'ALL_IN_ONE_BY_TABLE', 'bg-cyan-300'),
mappingItem('按库1对多', 'ALL_IN_ONE_BY_SCHEMA', 'bg-indigo-300'),
]
let compactionStatusMapping = [
mappingItem('调度', 'SCHEDULE'),
mappingItem('开始', 'START', 'bg-primary'),
mappingItem('完成', 'FINISH', 'bg-success'),
mappingItem('失败', 'FAIL', 'bg-danger'),
]
let tagsMapping = [
mappingItem('不压缩', 'NO_COMPACT'),
mappingItem('备份Pulsar消息', 'PULSAR_BACKUP'),
mappingItem('无预合并', 'NO_PRE_COMBINE'),
mappingItem('不忽略写日志错误', 'NO_IGNORE_FAILED'),
mappingItem('取消算子合并', 'DISABLE_CHAINING'),
mappingItem('跟踪压缩op_ts', 'TRACE_LATEST_OP_TS'),
]
let hudiTableTypeMapping = [
mappingItem('MOR', 'MERGE_ON_READ'),
mappingItem('COW', 'COPY_ON_WRITE'),
]
let filterModeMapping = [
mappingItem('无', 'NONE', 'bg-pink-500'),
mappingItem('包含模式', 'INCLUDE', 'bg-purple-500'),
mappingItem('排除模式', 'EXCLUDE', 'bg-cyan-500'),
]
let subscriptionTypeMapping = [
mappingItem('独占', 'Exclusive', 'bg-pink-500'),
mappingItem('共享', 'Shared', 'bg-purple-500'),
mappingItem('灾备', 'Failover', 'bg-cyan-500'),
mappingItem('Key', 'Key_Shared', 'bg-green-500'),
]
let publishTypeMapping = [
mappingItem('共享', 'Shared', 'bg-pink-500'),
mappingItem('独占', 'Exclusive', 'bg-purple-500'),
mappingItem('等待独占', 'WaiteForExclusive', 'bg-cyan-500'),
]
let hudiTimelineActionMapping = [
mappingItem('Commit', 'commit'),
mappingItem('Delta Commit', 'deltacommit'),
mappingItem('Clean', 'clean', 'bg-cyan-500'),
mappingItem('Rollback', 'rollback', 'label-danger'),
mappingItem('Savepoint', 'savepoint'),
mappingItem('Replace Commit', 'replacecommit', 'label-warning'),
mappingItem('Compaction', 'compaction', 'bg-purple-500'),
mappingItem('Restore', 'restore', 'label-warning'),
mappingItem('Indexing', 'indexing'),
mappingItem('Schema Commit', 'schemacommit', 'label-warning'),
]
let hudiTimelineStateMapping = [
mappingItem('已提交', 'REQUESTED'),
mappingItem('操作中', 'INFLIGHT', 'label-warning'),
mappingItem('已完成', 'COMPLETED', 'label-success'),
mappingItem('错误', 'INVALID', 'label-danger'),
]
let hudiTimelineTypeMapping = [
mappingItem('活跃', 'active'),
mappingItem('归档', 'archive', 'bg-gray-300'),
]
let tableRunningStateMapping = [
mappingItem('运行中', 'true', 'label-success'),
mappingItem('未运行', 'false'),
]
let versionUpdateStateMapping = [
mappingItem('已跨天', 'true', 'label-success'),
mappingItem('未跨天', 'false', 'label-danger'),
]
let compactionMetricsStateMapping = [
mappingItem('成功', 'true', 'label-success'),
mappingItem('失败', 'false', 'label-danger'),
]
function mappingField(field, mapping) {
let mapData = {
'*': `\${${field}}`,
}
mapping.forEach(item => {
mapData[item['value']] = `${item['label']}`
})
return {
type: 'mapping',
map: mapData,
}
}
function filterableField(mapping, multiple = false) {
return {
multiple: multiple,
options: [
...mapping
]
}
}
function formReloadFlinkJobIdTextInputAndAliasTextInput(id) {
return {
onEvent: {
change: {
actions: [
{
actionType: 'reload',
componentId: `flink-job-id-input-select-${id}`
},
{
actionType: 'reload',
componentId: `alias-input-select-${id}`
}
]
}
}
}
}
function flinkJobIdTextInput(id, require = false) {
return {
id: `flink-job-id-input-select-${id}`,
type: 'select',
name: 'flinkJobId',
label: 'Flink job id',
placeholder: '通过 ID 搜索',
clearable: true,
required: require,
searchable: true,
source: {
method: 'get',
url: '${base}/table/all_flink_job_id',
data: {
alias: '${alias|default:undefined}'
}
},
/*onEvent: {
change: {
actions: [
{
actionType: 'reload',
componentId: `alias-input-select-${id}`,
},
]
}
}*/
}
}
function aliasTextInput(id, require = false) {
return {
id: `alias-input-select-${id}`,
type: 'select',
name: 'alias',
label: '别名',
placeholder: '通过别名搜索',
clearable: true,
required: require,
searchable: true,
source: {
method: 'get',
url: '${base}/table/all_alias',
data: {
flink_job_id: '${flinkJobId|default:undefined}'
}
},
/*onEvent: {
change: {
actions: [
{
actionType: 'reload',
componentId: `flink-job-id-input-select-${id}`
},
]
}
}*/
}
}