lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(all): flink job增加tags属性

Browse Source 
flink job级别增加标签属性，用于区分调用测试包和非测试包
This commit is contained in:
v-zhangjc9
2024-07-30 16:42:02 +08:00
parent b0c5d04476
commit a3472340b5
23 changed files with 332 additions and 219 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java
View File

@@ -264,6 +264,7 @@ public class RuntimeInfo {
public static final class HudiInfo {
private String appHdfsPath;
private String appTestHdfsPath;
private String archiveHdfsPath;
private String victoriaPushUrl;
@@ -275,6 +276,14 @@ public class RuntimeInfo {
this.appHdfsPath = appHdfsPath;
}
public String getAppTestHdfsPath() {
return appTestHdfsPath;
}
public void setAppTestHdfsPath(String appTestHdfsPath) {
this.appTestHdfsPath = appTestHdfsPath;
}
public String getArchiveHdfsPath() {
return archiveHdfsPath;
}
@@ -295,6 +304,7 @@ public class RuntimeInfo {
public String toString() {
return "HudiInfo{" +
"appHdfsPath='" + appHdfsPath + '\'' +
", appTestHdfsPath='" + appTestHdfsPath + '\'' +
", archiveHdfsPath='" + archiveHdfsPath + '\'' +
", victoriaPushUrl='" + victoriaPushUrl + '\'' +
'}';

334
service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
View File

@@ -43,7 +43,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication
public class RunnerApplication implements ApplicationRunner {
private static final Logger logger = LoggerFactory.getLogger(RunnerApplication.class);
private static final TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
private final DeployInformationProperties deployInformationProperties;
private final RuntimeInfo runtimeInfo;
private final ImmutableList<ServiceInfoWrapper> serviceInfoList;
@@ -71,6 +71,13 @@ public class RunnerApplication implements ApplicationRunner {
SpringApplication.run(RunnerApplication.class, args);
}
private static void generateTemplate(String templatePath, Map<?, ?> data, Path targetScriptPath) throws IOException {
Template template = engine.getTemplate(templatePath);
String script = template.render(data);
Files.deleteIfExists(targetScriptPath);
Files.write(targetScriptPath, script.getBytes());
}
private List<String> selectHosts(ServiceInfoWrapper serviceInfo) {
return serviceInfo.getReplicas() == 0
? hostInfoList
@@ -111,15 +118,14 @@ public class RunnerApplication implements ApplicationRunner {
String absolutRootPath = root.toAbsolutePath().toString();
logger.info("Current path: {}", absolutRootPath);
TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
Template syncTemplate = engine.getTemplate("check/check.ftl");
String syncScript = syncTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build());
Path checkScriptFile = Paths.get(root.toString(), "check.sh");
Files.deleteIfExists(checkScriptFile);
Files.write(checkScriptFile, syncScript.getBytes());
generateTemplate(
"check/check.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build(),
Paths.get(root.toString(), "check.sh")
);
}
private void generateCloud(Path root) throws IOException {
@@ -133,8 +139,6 @@ public class RunnerApplication implements ApplicationRunner {
deployPlans = mapper.readValue(new String(Files.readAllBytes(planPath)), new TypeReference<Map<String, List<String>>>() {});
}
TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
Template deployTemplate = engine.getTemplate("cloud/deploy.ftl");
for (ServiceInfoWrapper serviceInfo : serviceInfoList) {
logger.info("Generate script for {}", serviceInfo.getName());
List<String> selectedHosts;
@@ -148,71 +152,70 @@ public class RunnerApplication implements ApplicationRunner {
selectedHosts = selectHosts(serviceInfo);
deployPlans.put(serviceInfo.getName(), selectedHosts);
}
String deployScript = deployTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("hosts", hostInfoList
.collect(HostInfoWrapper::getHostnameIp)
.toSortedList((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp())))
.put("selectedHosts", selectedHosts)
.put("runtime", runtimeInfo)
.put("info", serviceInfo)
.put("classpath", String.join(":", serviceInfo.getClasspath()))
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build());
Path deployScriptFile = Paths.get(
root.toString(),
StrUtil.format("deploy-{}.sh", serviceInfo.getName())
generateTemplate(
"cloud/deploy.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("hosts", hostInfoList
.collect(HostInfoWrapper::getHostnameIp)
.toSortedList((o1, o2) -> Comparator.<String>naturalOrder().compare(o1.getIp(), o2.getIp())))
.put("selectedHosts", selectedHosts)
.put("runtime", runtimeInfo)
.put("info", serviceInfo)
.put("classpath", String.join(":", serviceInfo.getClasspath()))
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build(),
Paths.get(
root.toString(),
StrUtil.format("deploy-{}.sh", serviceInfo.getName())
)
);
Files.deleteIfExists(deployScriptFile);
Files.write(deployScriptFile, deployScript.getBytes());
Template stopTemplate = engine.getTemplate("cloud/stop.ftl");
String stopScript = stopTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("hosts", hostInfoList
.collect(HostInfoWrapper::getIp)
.toSortedList(Comparator.naturalOrder()))
.put("runtime", runtimeInfo)
.put("info", serviceInfo)
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build());
Path stopScriptFile = Paths.get(
root.toString(),
StrUtil.format("stop-{}.sh", serviceInfo.getName())
generateTemplate(
"cloud/stop.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("hosts", hostInfoList
.collect(HostInfoWrapper::getIp)
.toSortedList(Comparator.naturalOrder()))
.put("runtime", runtimeInfo)
.put("info", serviceInfo)
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build(),
Paths.get(
root.toString(),
StrUtil.format("stop-{}.sh", serviceInfo.getName())
)
);
Files.deleteIfExists(stopScriptFile);
Files.write(stopScriptFile, stopScript.getBytes());
Template logTemplate = engine.getTemplate("cloud/log.ftl");
String logScript = logTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("hosts", hostInfoList
.collect(HostInfoWrapper::getIp)
.toSortedList(Comparator.naturalOrder()))
.put("selectedHosts", selectedHosts)
.put("runtime", runtimeInfo)
.put("info", serviceInfo)
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build());
Path logScriptFile = Paths.get(
root.toString(),
StrUtil.format("log-{}.sh", serviceInfo.getName())
generateTemplate(
"cloud/log.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("hosts", hostInfoList
.collect(HostInfoWrapper::getIp)
.toSortedList(Comparator.naturalOrder()))
.put("selectedHosts", selectedHosts)
.put("runtime", runtimeInfo)
.put("info", serviceInfo)
.put("arguments", serviceInfo.getArguments())
.put("environments", serviceInfo.getEnvironments())
.build(),
Paths.get(
root.toString(),
StrUtil.format("log-{}.sh", serviceInfo.getName())
)
);
Files.deleteIfExists(logScriptFile);
Files.write(logScriptFile, logScript.getBytes());
}
Template stopTemplate = engine.getTemplate("cloud/stop-script.ftl");
String stopScript = stopTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build());
Path stopScriptFile = Paths.get(root.toString(), "stop.sh");
Files.deleteIfExists(stopScriptFile);
Files.write(stopScriptFile, stopScript.getBytes());
generateTemplate(
"cloud/stop-script.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build(),
Paths.get(root.toString(), "stop.sh")
);
MutableMap<String, MutableList<ServiceInfoWrapper>> groups = Maps.mutable.empty();
for (ServiceInfoWrapper service : serviceInfoList) {
@@ -229,29 +232,28 @@ public class RunnerApplication implements ApplicationRunner {
String group = entry.getKey();
MutableList<ServiceInfoWrapper> infos = entry.getValue();
Template batchDeployTemplate = engine.getTemplate("cloud/batch-deploy.ftl");
String batchDeployScript = batchDeployTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("services", infos.collect(ServiceInfoWrapper::getName))
.build());
Path batchDeployScriptFile = Paths.get(
root.toString(),
StrUtil.format("batch-deploy-{}.sh", group)
generateTemplate(
"cloud/batch-deploy.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("services", infos.collect(ServiceInfoWrapper::getName))
.build(),
Paths.get(
root.toString(),
StrUtil.format("batch-deploy-{}.sh", group)
)
);
Files.deleteIfExists(batchDeployScriptFile);
Files.write(batchDeployScriptFile, batchDeployScript.getBytes());
Template batchStopTemplate = engine.getTemplate("cloud/batch-stop.ftl");
String batchStopScript = batchStopTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("services", infos.collect(ServiceInfoWrapper::getName))
.build());
Path batchStopScriptFile = Paths.get(
root.toString(),
StrUtil.format("batch-stop-{}.sh", group)
generateTemplate(
"cloud/batch-stop.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("services", infos.collect(ServiceInfoWrapper::getName))
.build(),
Paths.get(
root.toString(),
StrUtil.format("batch-stop-{}.sh", group)
)
);
Files.deleteIfExists(batchStopScriptFile);
Files.write(batchStopScriptFile, batchStopScript.getBytes());
}
Files.deleteIfExists(planPath);
@@ -263,35 +265,32 @@ public class RunnerApplication implements ApplicationRunner {
String absolutRootPath = root.toAbsolutePath().toString();
logger.info("Current path: {}", absolutRootPath);
TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
Template commandTemplate = engine.getTemplate(root.toFile().getName() + "/cli.ftl");
String commandScript = commandTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("directly", false)
.build());
Path commandScriptFile = Paths.get(root.toString(), "cli");
Files.deleteIfExists(commandScriptFile);
Files.write(commandScriptFile, commandScript.getBytes());
Template commandDirectlyTemplate = engine.getTemplate(root.toFile().getName() + "/cli.ftl");
String commandDirectlyScript = commandDirectlyTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("directly", true)
.build());
Path commandDirectlyScriptFile = Paths.get(root.toString(), "cli_d");
Files.deleteIfExists(commandDirectlyScriptFile);
Files.write(commandDirectlyScriptFile, commandDirectlyScript.getBytes());
Template updateTemplate = engine.getTemplate(root.toFile().getName() + "/update.ftl");
String updateScript = updateTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build());
Path updateScriptFile = Paths.get(root.toString(), "update.sh");
Files.deleteIfExists(updateScriptFile);
Files.write(updateScriptFile, updateScript.getBytes());
generateTemplate(
root.toFile().getName() + "/cli.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("directly", false)
.build(),
Paths.get(root.toString(), "cli")
);
generateTemplate(
root.toFile().getName() + "/cli.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("directly", true)
.build(),
Paths.get(root.toString(), "cli_d")
);
generateTemplate(
root.toFile().getName() + "/update.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build(),
Paths.get(root.toString(), "update.sh")
);
}
private void generateUploader(Path root) throws IOException {
@@ -299,24 +298,22 @@ public class RunnerApplication implements ApplicationRunner {
String absolutRootPath = root.toAbsolutePath().toString();
logger.info("Current path: {}", absolutRootPath);
TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
Template startTemplate = engine.getTemplate("uploader/start.ftl");
String startScript = startTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build());
Path startScriptFile = Paths.get(root.toString(), "start.sh");
Files.deleteIfExists(startScriptFile);
Files.write(startScriptFile, startScript.getBytes());
Template updateTemplate = engine.getTemplate("uploader/update.ftl");
String updateScript = updateTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build());
Path updateScriptFile = Paths.get(root.toString(), "update.sh");
Files.deleteIfExists(updateScriptFile);
Files.write(updateScriptFile, updateScript.getBytes());
generateTemplate(
"uploader/start.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build(),
Paths.get(root.toString(), "start.sh")
);
generateTemplate(
"uploader/update.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.build(),
Paths.get(root.toString(), "update.sh")
);
Template stopTemplate = engine.getTemplate("cloud/stop-script.ftl");
String stopScript = stopTemplate.render(MapUtil.builder()
@@ -332,29 +329,38 @@ public class RunnerApplication implements ApplicationRunner {
String absolutRootPath = root.toAbsolutePath().toString();
logger.info("Current path: {}", absolutRootPath);
TemplateEngine engine = TemplateUtil.createEngine(new TemplateConfig("template", TemplateConfig.ResourceMode.CLASSPATH));
Template syncTemplate = engine.getTemplate("update-jar.ftl");
String syncScript = syncTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("jarPrefix", "sync")
.put("jarName", "sync-1.0.0-SNAPSHOT.jar")
.put("uploadPath", runtimeInfo.getHudi().getAppHdfsPath())
.build());
Path syncScriptFile = Paths.get(root.toString(), "update-sync.sh");
Files.deleteIfExists(syncScriptFile);
Files.write(syncScriptFile, syncScript.getBytes());
Template taskTemplate = engine.getTemplate("update-jar.ftl");
String taskScript = taskTemplate.render(MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("jarPrefix", "task")
.put("jarName", "service-executor-task-1.0.0-SNAPSHOT.jar")
.put("uploadPath", taskJarPath)
.build());
Path taskScriptFile = Paths.get(root.toString(), "update-task.sh");
Files.deleteIfExists(taskScriptFile);
Files.write(taskScriptFile, taskScript.getBytes());
generateTemplate(
"update-jar.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("jarPrefix", "sync")
.put("jarName", "sync-1.0.0-SNAPSHOT.jar")
.put("uploadPath", runtimeInfo.getHudi().getAppHdfsPath())
.build(),
Paths.get(root.toString(), "update-sync.sh")
);
generateTemplate(
"update-jar.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("jarPrefix", "sync")
.put("jarName", "sync-1.0.0-SNAPSHOT.jar")
.put("uploadPath", runtimeInfo.getHudi().getAppTestHdfsPath())
.build(),
Paths.get(root.toString(), "update-test.sh")
);
generateTemplate(
"update-jar.ftl",
MapUtil.builder()
.put("currentPath", absolutRootPath)
.put("runtime", runtimeInfo)
.put("jarPrefix", "task")
.put("jarName", "service-executor-task-1.0.0-SNAPSHOT.jar")
.put("uploadPath", taskJarPath)
.build(),
Paths.get(root.toString(), "update-task.sh")
);
}
}

1
service-cli/service-cli-runner/src/main/resources/application-b12.yml
View File

@@ -31,6 +31,7 @@ deploy:
hudi:
# hudi业务jar包所在目录
app-hdfs-path: hdfs://b2/apps/datalake/jars/app-b12
app-test-hdfs-path: hdfs://b2/apps/datalake/jars/app-test-b12
# hudi指标推送
victoria-push-url: http://132.126.207.125:35710/api/v1/import/prometheus
loki-push-url: ${deploy.runtime.loki.hudi-push-url}

4
service-cli/service-cli-runner/src/main/resources/application.yml
View File

@@ -37,6 +37,7 @@ deploy:
"[connector.hadoop.kerberos-principal]": ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
"[connector.hadoop.kerberos-keytab-path]": ${deploy.runtime.kerberos-keytab-path}
"[connector.hudi.app-hdfs-path]": ${deploy.runtime.hudi.app-hdfs-path}
"[connector.hudi.app-test-hdfs-path]": ${deploy.runtime.hudi.app-test-hdfs-path}
"[connector.hudi.victoria-push-url]": ${deploy.runtime.hudi.victoria-push-url}
"[connector.hudi.loki-push-url]": ${deploy.runtime.hudi.loki-push-url}
arguments:
@@ -57,6 +58,7 @@ deploy:
"[connector.hadoop.kerberos-principal]": ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
"[connector.hadoop.kerberos-keytab-path]": ${deploy.runtime.kerberos-keytab-path}
"[connector.hudi.app-hdfs-path]": ${deploy.runtime.hudi.app-hdfs-path}
"[connector.hudi.app-test-hdfs-path]": ${deploy.runtime.hudi.app-test-hdfs-path}
"[connector.hudi.victoria-push-url]": ${deploy.runtime.hudi.victoria-push-url}
"[connector.hudi.loki-push-url]": ${deploy.runtime.hudi.loki-push-url}
arguments:
@@ -77,6 +79,7 @@ deploy:
"[connector.hadoop.kerberos-principal]": ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
"[connector.hadoop.kerberos-keytab-path]": ${deploy.runtime.kerberos-keytab-path}
"[connector.hudi.app-hdfs-path]": ${deploy.runtime.hudi.app-hdfs-path}
"[connector.hudi.app-test-hdfs-path]": ${deploy.runtime.hudi.app-test-hdfs-path}
"[connector.hudi.victoria-push-url]": ${deploy.runtime.hudi.victoria-push-url}
"[connector.hudi.loki-push-url]": ${deploy.runtime.hudi.loki-push-url}
arguments:
@@ -97,6 +100,7 @@ deploy:
"[connector.hadoop.kerberos-principal]": ${deploy.runtime.user}/$\{hostname}.hdp.dc@ECLD.COM
"[connector.hadoop.kerberos-keytab-path]": ${deploy.runtime.kerberos-keytab-path}
"[connector.hudi.app-hdfs-path]": ${deploy.runtime.hudi.app-hdfs-path}
"[connector.hudi.app-test-hdfs-path]": ${deploy.runtime.hudi.app-test-hdfs-path}
"[connector.hudi.victoria-push-url]": ${deploy.runtime.hudi.victoria-push-url}
"[connector.hudi.loki-push-url]": ${deploy.runtime.hudi.loki-push-url}
arguments: