lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(common): Record消息结构更新

Browse Source 
新增info字段用于存一些公共字段，statement加入消息加密能力
This commit is contained in:
v-zhangjc9
2024-07-09 15:36:10 +08:00
parent ae6e4855e0
commit 6c9f43d310
11 changed files with 455 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
service-executor/service-executor-task/src/main/java/com/lanyuanxiaoyao/service/executor/task/functions/police/PulsarMessage2Prisoner.java
View File

@@ -4,17 +4,15 @@ import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.common.Constants;
import com.lanyuanxiaoyao.service.common.entity.Record;
import com.lanyuanxiaoyao.service.executor.core.TaskContext;
import com.lanyuanxiaoyao.service.common.utils.RecordHelper;
import com.lanyuanxiaoyao.service.executor.task.entity.Prisoner;
import com.lanyuanxiaoyao.service.executor.task.entity.RecordView;
import com.lanyuanxiaoyao.service.executor.task.helper.JacksonHelper;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
import org.eclipse.collections.api.factory.Lists;
/**
@@ -23,7 +21,6 @@ import org.eclipse.collections.api.factory.Lists;
public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner> {
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final Pattern OPTS_PATTERN = Pattern.compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$");
private final ObjectMapper mapper = JacksonHelper.getMapper();
private final long startTime;
private final long endTime;
@@ -41,18 +38,19 @@ public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner
public Prisoner map(RecordView value) {
Record record;
try {
record = mapper.readValue(value.getData(), Record.class);
record = RecordHelper.parse(value.getData());
} catch (Throwable e) {
return new Prisoner(value.getFile(), StrUtil.format("{}: {}", e.getMessage(), value.getData()));
}
Record.Info info = record.getInfo();
Record.Statement statement = record.getStatement();
if (ObjectUtil.isNull(statement) || StrUtil.isBlank(statement.getOpTs())) {
if (ObjectUtil.isNull(statement) || StrUtil.isBlank(info.getOpTs())) {
return new Prisoner(value.getFile(), StrUtil.format("Invalid statement: {}", value.getData()));
}
long timestamp;
try {
if (OPTS_PATTERN.matcher(statement.getOpTs()).matches()) {
timestamp = LocalDateTime.parse(statement.getOpTs(), FORMATTER).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
if (OPTS_PATTERN.matcher(info.getOpTs()).matches()) {
timestamp = LocalDateTime.parse(info.getOpTs(), FORMATTER).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
} else {
throw new Exception("opts not match " + OPTS_PATTERN.pattern());
}
@@ -63,9 +61,9 @@ public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner
return null;
}
Map<String, Object> fields;
if (StrUtil.equalsAny(statement.getOpType(), Constants.INSERT, Constants.UPDATE)) {
if (StrUtil.equalsAny(info.getOpType(), Constants.INSERT, Constants.UPDATE)) {
fields = record.getStatement().getAfter();
} else if (StrUtil.equals(statement.getOpType(), Constants.DELETE)) {
} else if (StrUtil.equals(info.getOpType(), Constants.DELETE)) {
fields = record.getStatement().getBefore();
} else {
return new Prisoner(value.getFile(), StrUtil.format("Invalid opType: {}", value.getData()));
@@ -78,6 +76,6 @@ public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner
if (StrUtil.isBlank(partitionKey)) {
return new Prisoner(value.getFile(), StrUtil.format("Invalid partitionKey: {}", value.getData()));
}
return new Prisoner(value.getFile(), record.getStatement().getOpType(), primaryKey, partitionKey, timestamp);
return new Prisoner(value.getFile(), record.getInfo().getOpType(), primaryKey, partitionKey, timestamp);
}
}