lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

revert(common): 回滚Record消息结构更新

Browse Source 
This reverts commit 6c9f43d310.
This commit is contained in:
v-zhangjc9
2024-07-29 15:42:06 +08:00
parent 738af7a85f
commit b0c5d04476
11 changed files with 80 additions and 455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
service-executor/service-executor-task/src/main/java/com/lanyuanxiaoyao/service/executor/task/functions/police/PulsarMessage2Prisoner.java
View File

@@ -4,15 +4,17 @@ import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.lanyuanxiaoyao.service.common.Constants;
import com.lanyuanxiaoyao.service.common.entity.Record;
import com.lanyuanxiaoyao.service.common.utils.RecordHelper;
import com.lanyuanxiaoyao.service.executor.core.TaskContext;
import com.lanyuanxiaoyao.service.executor.task.entity.Prisoner;
import com.lanyuanxiaoyao.service.executor.task.entity.RecordView;
import com.lanyuanxiaoyao.service.executor.task.helper.JacksonHelper;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.Map;
import java.util.regex.Pattern;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
import org.eclipse.collections.api.factory.Lists;
/**
@@ -21,6 +23,7 @@ import org.eclipse.collections.api.factory.Lists;
public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner> {
private static final DateTimeFormatter FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final Pattern OPTS_PATTERN = Pattern.compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$");
private final ObjectMapper mapper = JacksonHelper.getMapper();
private final long startTime;
private final long endTime;
@@ -38,19 +41,18 @@ public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner
public Prisoner map(RecordView value) {
Record record;
try {
record = RecordHelper.parse(value.getData());
record = mapper.readValue(value.getData(), Record.class);
} catch (Throwable e) {
return new Prisoner(value.getFile(), StrUtil.format("{}: {}", e.getMessage(), value.getData()));
}
Record.Info info = record.getInfo();
Record.Statement statement = record.getStatement();
if (ObjectUtil.isNull(statement) || StrUtil.isBlank(info.getOpTs())) {
if (ObjectUtil.isNull(statement) || StrUtil.isBlank(statement.getOpTs())) {
return new Prisoner(value.getFile(), StrUtil.format("Invalid statement: {}", value.getData()));
}
long timestamp;
try {
if (OPTS_PATTERN.matcher(info.getOpTs()).matches()) {
timestamp = LocalDateTime.parse(info.getOpTs(), FORMATTER).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
if (OPTS_PATTERN.matcher(statement.getOpTs()).matches()) {
timestamp = LocalDateTime.parse(statement.getOpTs(), FORMATTER).toInstant(ZoneOffset.ofHours(8)).toEpochMilli();
} else {
throw new Exception("opts not match " + OPTS_PATTERN.pattern());
}
@@ -61,9 +63,9 @@ public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner
return null;
}
Map<String, Object> fields;
if (StrUtil.equalsAny(info.getOpType(), Constants.INSERT, Constants.UPDATE)) {
if (StrUtil.equalsAny(statement.getOpType(), Constants.INSERT, Constants.UPDATE)) {
fields = record.getStatement().getAfter();
} else if (StrUtil.equals(info.getOpType(), Constants.DELETE)) {
} else if (StrUtil.equals(statement.getOpType(), Constants.DELETE)) {
fields = record.getStatement().getBefore();
} else {
return new Prisoner(value.getFile(), StrUtil.format("Invalid opType: {}", value.getData()));
@@ -76,6 +78,6 @@ public class PulsarMessage2Prisoner extends RichMapFunction<RecordView, Prisoner
if (StrUtil.isBlank(partitionKey)) {
return new Prisoner(value.getFile(), StrUtil.format("Invalid partitionKey: {}", value.getData()));
}
return new Prisoner(value.getFile(), record.getInfo().getOpType(), primaryKey, partitionKey, timestamp);
return new Prisoner(value.getFile(), record.getStatement().getOpType(), primaryKey, partitionKey, timestamp);
}
}