diff --git a/bin/build-exporter.sh b/bin/build-exporter.sh new file mode 100755 index 0000000..4cf87c7 --- /dev/null +++ b/bin/build-exporter.sh @@ -0,0 +1,4 @@ +#!/bin/bash +mvn -pl service-dependencies,service-configuration,service-forest clean deploy -D skipTests -P local -s ~/.m2/settings-development.xml +mvn -pl service-exporter clean package spring-boot:repackage -D skipTests -s ~/.m2/settings-development.xml +ytp-transfer2 /Users/lanyuanxiaoyao/Project/IdeaProjects/hudi-service/service-exporter/target/service-exporter-1.0.0-SNAPSHOT.jar \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8bc92df..69c1849 100644 --- a/pom.xml +++ b/pom.xml @@ -26,6 +26,7 @@ service-uploader service-executor service-cloud-query + service-exporter diff --git a/service-cli/service-cli-runner/src/main/resources/application.yml b/service-cli/service-cli-runner/src/main/resources/application.yml index 2bff1e9..f605fc1 100644 --- a/service-cli/service-cli-runner/src/main/resources/application.yml +++ b/service-cli/service-cli-runner/src/main/resources/application.yml @@ -149,3 +149,6 @@ deploy: executor_history-server-archive-dir: ${deploy.runtime.executor.archive-hdfs-path} executor_task-jar-path: ${deploy.runtime.executor.task-jar-path} executor_task-result-path: ${deploy.runtime.executor.task-result-path} + - name: service-exporter + source-jar: service-exporter-1.0.0-SNAPSHOT.jar + replicas: 3 diff --git a/service-exporter/pom.xml b/service-exporter/pom.xml new file mode 100644 index 0000000..4e5c4a0 --- /dev/null +++ b/service-exporter/pom.xml @@ -0,0 +1,45 @@ + + + 4.0.0 + + com.lanyuanxiaoyao + hudi-service + 1.0.0-SNAPSHOT + + + service-exporter + 用于提供系统外部接口 + + + + com.lanyuanxiaoyao + service-dependencies + 1.0.0-SNAPSHOT + + + com.lanyuanxiaoyao + service-configuration + 1.0.0-SNAPSHOT + + + com.lanyuanxiaoyao + service-forest + 1.0.0-SNAPSHOT + + + + + + + org.apache.maven.plugins + maven-source-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + + \ No newline at end of file diff --git a/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/ExporterApplication.java b/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/ExporterApplication.java new file mode 100644 index 0000000..5b9a82b --- /dev/null +++ b/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/ExporterApplication.java @@ -0,0 +1,33 @@ +package com.lanyuanxiaoyao.service.exporter; + +import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.retry.annotation.EnableRetry; + +/** + * 启动类 + * + * @author lanyuanxiaoyao + * @date 2023-04-30 + */ +@EnableDiscoveryClient +@SpringBootApplication( + scanBasePackages = {"com.lanyuanxiaoyao.service"}, + exclude = { + GsonAutoConfiguration.class, + DataSourceAutoConfiguration.class + } +) +@EnableConfigurationProperties +@EnableEncryptableProperties +@EnableRetry +public class ExporterApplication { + public static void main(String[] args) { + SpringApplication.run(ExporterApplication.class, args); + } +} diff --git a/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/controller/ExporterController.java b/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/controller/ExporterController.java new file mode 100644 index 0000000..5a8700b --- /dev/null +++ b/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/controller/ExporterController.java @@ -0,0 +1,46 @@ +package com.lanyuanxiaoyao.service.exporter.controller; + +import com.eshore.odcp.hudi.connector.utils.NameHelper; +import com.lanyuanxiaoyao.service.configuration.entity.zookeeper.ZookeeperNode; +import com.lanyuanxiaoyao.service.exporter.entity.FlinkJobIdAndName; +import com.lanyuanxiaoyao.service.forest.service.InfoService; +import com.lanyuanxiaoyao.service.forest.service.ZookeeperService; +import org.eclipse.collections.api.list.ImmutableList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.cache.annotation.CacheConfig; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 出口 + * + * @author lanyuanxiaoyao + * @date 2024-02-02 + */ +@RestController +@RequestMapping("exporter") +@CacheConfig(cacheManager = "normal-cache") +@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection") +public class ExporterController { + private static final Logger logger = LoggerFactory.getLogger(ExporterController.class); + + private final InfoService infoService; + private final ZookeeperService zookeeperService; + + public ExporterController(InfoService infoService, ZookeeperService zookeeperService) { + this.infoService = infoService; + this.zookeeperService = zookeeperService; + } + + @GetMapping("un_running_flink_job") + @Cacheable(value = "un_running_flink_job", sync = true) + public ImmutableList unRunningFlinkJob() { + ImmutableList locks = zookeeperService.getChildren(NameHelper.ZK_SYNC_RUNNING_LOCK_PATH).collect(ZookeeperNode::getPath); + return infoService.flinkJobList() + .reject(job -> locks.contains(NameHelper.syncFlinkName(job.getId(), job.getName()))) + .collect(job -> new FlinkJobIdAndName(job.getId(), job.getName())); + } +} diff --git a/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/entity/FlinkJobIdAndName.java b/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/entity/FlinkJobIdAndName.java new file mode 100644 index 0000000..4e0275d --- /dev/null +++ b/service-exporter/src/main/java/com/lanyuanxiaoyao/service/exporter/entity/FlinkJobIdAndName.java @@ -0,0 +1,31 @@ +package com.lanyuanxiaoyao.service.exporter.entity; + +/** + * @author lanyuanxiaoyao + * @date 2024-02-02 + */ +public class FlinkJobIdAndName { + private final Long id; + private final String name; + + public FlinkJobIdAndName(Long id, String name) { + this.id = id; + this.name = name; + } + + public Long getId() { + return id; + } + + public String getName() { + return name; + } + + @Override + public String toString() { + return "FlinkJobIdAndName{" + + "id=" + id + + ", name='" + name + '\'' + + '}'; + } +} diff --git a/service-exporter/src/main/resources/application.yml b/service-exporter/src/main/resources/application.yml new file mode 100644 index 0000000..3b6d1c6 --- /dev/null +++ b/service-exporter/src/main/resources/application.yml @@ -0,0 +1,5 @@ +spring: + application: + name: service-exporter + profiles: + include: random-port,common,discovery,metrics,forest \ No newline at end of file diff --git a/service-exporter/src/main/resources/logback-spring.xml b/service-exporter/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..f272f36 --- /dev/null +++ b/service-exporter/src/main/resources/logback-spring.xml @@ -0,0 +1,52 @@ + + + + + + + + + + + true + + ${LOKI_PUSH_URL:-http://localhost/loki/api/v1/push} + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %p [${HOSTNAME}] [%t] %logger #@# %m%n%wEx + + true + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %clr(%5p) %clr([${HOSTNAME}]){yellow} %clr([%t]){magenta} %clr(%logger{40}){cyan} #@# %m%n%wEx + + + + + ${LOGGING_PARENT:-.}/${APP_NAME:-run}.log + + ${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz + 7 + + + %d{yyyy-MM-dd HH:mm:ss.SSS} %p [${HOSTNAME}] [%t] %logger #@# %m%n%wEx + + + + + + + + + + + + \ No newline at end of file diff --git a/service-gateway/src/main/java/com/lanyuanxiaoyao/service/gateway/GatewayApplication.java b/service-gateway/src/main/java/com/lanyuanxiaoyao/service/gateway/GatewayApplication.java index 31ee931..67c65d5 100644 --- a/service-gateway/src/main/java/com/lanyuanxiaoyao/service/gateway/GatewayApplication.java +++ b/service-gateway/src/main/java/com/lanyuanxiaoyao/service/gateway/GatewayApplication.java @@ -50,6 +50,7 @@ public class GatewayApplication { .route("scheduler", createRoute("/hudi_services/service_scheduler", "lb://service-scheduler")) .route("web", createRoute("/hudi_services/service_web", "lb://service-web")) .route("services", createRoute("/hudi_services/service_cloud_query", "lb://service-cloud-query")) + .route("exporter", createRoute("/hudi_services/service-exporter", "lb://service-exporter")) .build(); } }