lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(loki-query): 拆分服务和hudi使用的loki，查询服务分开部署

Browse Source
This commit is contained in:
兰缘小妖
2024-01-10 09:49:11 +08:00
parent 557aa22343
commit 86ad6df899
11 changed files with 184 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
service-cli/service-cli-core/src/main/java/com/lanyuanxiaoyao/service/cli/core/RuntimeInfo.java
View File

@@ -13,9 +13,9 @@ public class RuntimeInfo {
private String logPath;
private String dataPath;
private String kerberosKeytabPath;
private String lokiUrl;
private LokiInfo loki;
private String zkUrl;
private String eurekaUrl;
private String connectorZkUrl;
private HudiInfo hudi;
private SecurityInfo security;
@@ -67,12 +67,12 @@ public class RuntimeInfo {
this.kerberosKeytabPath = kerberosKeytabPath;
}
public String getLokiUrl() {
return lokiUrl;
public LokiInfo getLoki() {
return loki;
}
public void setLokiUrl(String lokiUrl) {
this.lokiUrl = lokiUrl;
public void setLoki(LokiInfo loki) {
this.loki = loki;
}
public String getZkUrl() {
@@ -83,12 +83,12 @@ public class RuntimeInfo {
this.zkUrl = zkUrl;
}
public String getEurekaUrl() {
return eurekaUrl;
public String getConnectorZkUrl() {
return connectorZkUrl;
}
public void setEurekaUrl(String eurekaUrl) {
this.eurekaUrl = eurekaUrl;
public void setConnectorZkUrl(String connectorZkUrl) {
this.connectorZkUrl = connectorZkUrl;
}
public HudiInfo getHudi() {
@@ -110,18 +110,116 @@ public class RuntimeInfo {
@Override
public String toString() {
return "RuntimeInfo{" +
"user='" + user + '\'' +
", jarPath='" + jarPath + '\'' +
", jdkPath='" + jdkPath + '\'' +
", logPath='" + logPath + '\'' +
", dataPath='" + dataPath + '\'' +
", kerberosKeytabPath='" + kerberosKeytabPath + '\'' +
", lokiUrl='" + lokiUrl + '\'' +
", zkUrl='" + zkUrl + '\'' +
", eurekaUrl='" + eurekaUrl + '\'' +
", hudi=" + hudi +
", security=" + security +
'}';
"user='" + user + '\'' +
", jarPath='" + jarPath + '\'' +
", jdkPath='" + jdkPath + '\'' +
", logPath='" + logPath + '\'' +
", dataPath='" + dataPath + '\'' +
", kerberosKeytabPath='" + kerberosKeytabPath + '\'' +
", loki=" + loki +
", zkUrl='" + zkUrl + '\'' +
", connectorZkUrl='" + connectorZkUrl + '\'' +
", hudi=" + hudi +
", security=" + security +
'}';
}
public static final class LokiInfo {
private String serviceUrl;
private String servicePushUrl;
private String hudiUrl;
private String hudiPushUrl;
public String getServiceUrl() {
return serviceUrl;
}
public void setServiceUrl(String serviceUrl) {
this.serviceUrl = serviceUrl;
}
public String getServicePushUrl() {
return servicePushUrl;
}
public void setServicePushUrl(String servicePushUrl) {
this.servicePushUrl = servicePushUrl;
}
public String getHudiUrl() {
return hudiUrl;
}
public void setHudiUrl(String hudiUrl) {
this.hudiUrl = hudiUrl;
}
public String getHudiPushUrl() {
return hudiPushUrl;
}
public void setHudiPushUrl(String hudiPushUrl) {
this.hudiPushUrl = hudiPushUrl;
}
@Override
public String toString() {
return "LokiInfo{" +
"serviceUrl='" + serviceUrl + '\'' +
", servicePushUrl='" + servicePushUrl + '\'' +
", hudiUrl='" + hudiUrl + '\'' +
", hudiPushUrl='" + hudiPushUrl + '\'' +
'}';
}
}
public static final class ExecutorInfo {
private String stagingPath;
private String archiveHdfsPath;
private String taskJarPath;
private String taskResultPath;
public String getStagingPath() {
return stagingPath;
}
public void setStagingPath(String stagingPath) {
this.stagingPath = stagingPath;
}
public String getArchiveHdfsPath() {
return archiveHdfsPath;
}
public void setArchiveHdfsPath(String archiveHdfsPath) {
this.archiveHdfsPath = archiveHdfsPath;
}
public String getTaskJarPath() {
return taskJarPath;
}
public void setTaskJarPath(String taskJarPath) {
this.taskJarPath = taskJarPath;
}
public String getTaskResultPath() {
return taskResultPath;
}
public void setTaskResultPath(String taskResultPath) {
this.taskResultPath = taskResultPath;
}
@Override
public String toString() {
return "ExecutorInfo{" +
"stagingPath='" + stagingPath + '\'' +
", archiveHdfsPath='" + archiveHdfsPath + '\'' +
", taskJarPath='" + taskJarPath + '\'' +
", taskResultPath='" + taskResultPath + '\'' +
'}';
}
}
public static final class HudiInfo {
@@ -156,10 +254,10 @@ public class RuntimeInfo {
@Override
public String toString() {
return "HudiInfo{" +
"appHdfsPath='" + appHdfsPath + '\'' +
", archiveHdfsPath='" + archiveHdfsPath + '\'' +
", victoriaPushUrl='" + victoriaPushUrl + '\'' +
'}';
"appHdfsPath='" + appHdfsPath + '\'' +
", archiveHdfsPath='" + archiveHdfsPath + '\'' +
", victoriaPushUrl='" + victoriaPushUrl + '\'' +
'}';
}
}
@@ -195,10 +293,10 @@ public class RuntimeInfo {
@Override
public String toString() {
return "SecurityInfo{" +
"authority='" + authority + '\'' +
", username='" + username + '\'' +
", darkcode='" + darkcode + '\'' +
'}';
"authority='" + authority + '\'' +
", username='" + username + '\'' +
", darkcode='" + darkcode + '\'' +
'}';
}
}
}

1
service-cli/service-cli-runner/src/main/java/com/lanyuanxiaoyao/service/cli/runner/RunnerApplication.java
View File

@@ -85,6 +85,7 @@ public class RunnerApplication implements ApplicationRunner {
@Override
public void run(ApplicationArguments args) throws IOException {
logger.info("Loading runtime info: {}", mapper.writeValueAsString(deployInformationProperties.getRuntime()));
Path planPath = Paths.get("deploy.plan");
Map<String, List<String>> deployPlans = new HashMap<>();
if (Files.exists(planPath) && !deployInformationProperties.getShuffler()) {

7
service-cli/service-cli-runner/src/main/resources/application-b12.yml
View File

@@ -6,8 +6,11 @@ deploy:
log-path: /data/datalake/logs
data-path: /data/datalake/data
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
loki-base-url: http://132.126.207.125:33100
loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
loki:
service-url: http://132.126.207.125:33100
service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push
hudi-url: http://132.126.207.126:33100
hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push
zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
connector-zk-url: b1m2.hdp.dc:2181,b1m3.hdp.dc:2181,b1m4.hdp.dc:2181,b1m5.hdp.dc:2181,b1m6.hdp.dc:2181
hudi:

7
service-cli/service-cli-runner/src/main/resources/application-b5.yml
View File

@@ -6,8 +6,11 @@ deploy:
log-path: /apps/datalake/hudi/logs
data-path: /apps/datalake/hudi/data
kerberos-keytab-path: /etc/security/keytabs/datalake.app.keytab
loki-base-url: http://132.122.116.142:33100
loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
loki:
service-url: http://132.122.116.142:33100
service-push-url: ${deploy.runtime.loki.service-url}/loki/api/v1/push
hudi-url: http://132.122.116.142:33100
hudi-push-url: ${deploy.runtime.loki.hudi-url}/loki/api/v1/push
zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
connector-zk-url: b5m1.hdp.dc:2181,b5m2.hdp.dc:2181,b5m3.hdp.dc:2181
hudi:

36
service-cli/service-cli-runner/src/main/resources/application-t5.yml
View File

@@ -1,36 +0,0 @@
deploy:
runtime:
user: datalake
jar-path: /home/datalake/hudi/jars
jdk-path: /opt/jdk1.8.0_162/bin/java
log-path: /home/datalake/hudi/logs
data-path: /home/datalake/hudi/data
kerberos-keytab-path: /opt/TDP/keytab/datalake.keytab
loki-base-url: http://132.121.126.84:33100
loki-url: ${deploy.runtime.loki-base-url}/loki/api/v1/push
zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
connector-zk-url: t5s1.hdp.dc:2181,t5s2.hdp.dc:2181,t5s3.hdp.dc:2181
hudi:
app-hdfs-path: jfs://ns1/apps/datalake/hudi/jars
archive-hdfs-path: jfs://ns1/apps/datalake/hudi/archive
victoria-push-url: http://132.121.126.84:35710/api/v1/import/prometheus
executor:
staging-path: hdfs://b2/apps/datalake/yarn
archive-hdfs-path: hdfs://b2/apps/flink/completed-jobs/
task-jar-path: hdfs://b2/apps/datalake/jars/service/service-executor-task-1.0.0-SNAPSHOT.jar
task-result-path: hdfs://b2/apps/datalake/task-results
security:
authority: ENC(GXKnbq1LS11U2HaONspvH+D/TkIx13aWTaokdkzaF7HSvq6Z0Rv1+JUWFnYopVXu)
username: ENC(moIO5mO39V1Z+RDwROK9JXY4GfM8ZjDgM6Si7wRZ1MPVjbhTpmLz3lz28rAiw7c2LeCmizfJzHkEXIwGlB280g==)
darkcode: ENC(0jzpQ7T6S+P7bZrENgYsUoLhlqGvw7DA2MN3BRqEOwq7plhtg72vuuiPQNnr3DaYz0CpyTvxInhpx11W3VZ1trD6NINh7O3LN70ZqO5pWXk=)
hosts:
- host: t5s1
ip: 132.121.126.83
- host: t5s2
ip: 132.121.126.84
- host: t5s3
ip: 132.121.126.86
- host: t5s4
ip: 132.121.126.89
- host: t5s5
ip: 132.121.126.90

9
service-cli/service-cli-runner/src/main/resources/application.yml
View File

@@ -99,10 +99,17 @@ deploy:
source-jar: service-info-query-1.0.0-SNAPSHOT.jar
replicas: 10
- name: service-loki-query
source-jar: service-loki-query-1.0.0-SNAPSHOT.jar
replicas: 5
arguments:
spring_application_name: service-loki-query
loki_host: ${deploy.runtime.loki.service-url}
- name: service-loki-hudi-query
source-jar: service-loki-query-1.0.0-SNAPSHOT.jar
replicas: 10
arguments:
loki_host: ${deploy.runtime.loki-base-url}
spring_application_name: service-loki-hudi-query
loki_host: ${deploy.runtime.loki.hudi-url}
- name: service-yarn-query
source-jar: service-yarn-query-1.0.0-SNAPSHOT.jar
replicas: 20

2
service-cli/service-cli-runner/src/main/resources/template/deploy.ftl
View File

@@ -7,7 +7,7 @@
)-->
datetime=`date +%Y%m%d%H%M%S`
log_path='${runtime.logPath}'
loki_url='${runtime.lokiUrl}'
loki_url='${runtime.loki.servicePushUrl}'
zk_url='${runtime.zkUrl}'
security_authority='${runtime.security.authority}'
security_username='${runtime.security.username}'

11
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiHudiService.java Normal file
View File

@@ -0,0 +1,11 @@
package com.lanyuanxiaoyao.service.forest.service.loki;
import com.dtflys.forest.annotation.BaseRequest;
/**
* @author lanyuanxiaoyao
* @date 2023-05-25
*/
@BaseRequest(baseURL = "http://service-loki-hudi-query")
public interface LokiHudiService extends LokiService {
}

6
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/LokiService.java → service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiService.java
View File

@@ -1,13 +1,11 @@
package com.lanyuanxiaoyao.service.forest.service;
package com.lanyuanxiaoyao.service.forest.service.loki;
import com.dtflys.forest.annotation.BaseRequest;
import com.dtflys.forest.annotation.Get;
import com.dtflys.forest.annotation.Query;
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
import org.eclipse.collections.api.list.ImmutableList;
import org.springframework.web.bind.annotation.RequestParam;
import java.util.Map;
import org.eclipse.collections.api.list.ImmutableList;
/**
* @author lanyuanxiaoyao

16
service-forest/src/main/java/com/lanyuanxiaoyao/service/forest/service/loki/LokiServiceService.java Normal file
View File

@@ -0,0 +1,16 @@
package com.lanyuanxiaoyao.service.forest.service.loki;
import com.dtflys.forest.annotation.BaseRequest;
import com.dtflys.forest.annotation.Get;
import com.dtflys.forest.annotation.Query;
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
import java.util.Map;
import org.eclipse.collections.api.list.ImmutableList;
/**
* @author lanyuanxiaoyao
* @date 2023-05-25
*/
@BaseRequest(baseURL = "http://service-loki-query")
public interface LokiServiceService extends LokiService {
}

14
service-web/src/main/java/com/lanyuanxiaoyao/service/web/controller/LogController.java
View File

@@ -3,7 +3,7 @@ package com.lanyuanxiaoyao.service.web.controller;
import cn.hutool.core.map.MapUtil;
import com.lanyuanxiaoyao.service.configuration.entity.loki.LokiLogLine;
import com.lanyuanxiaoyao.service.forest.service.InfoService;
import com.lanyuanxiaoyao.service.forest.service.LokiService;
import com.lanyuanxiaoyao.service.forest.service.loki.LokiHudiService;
import com.lanyuanxiaoyao.service.web.controller.base.AmisCrudResponse;
import com.lanyuanxiaoyao.service.web.controller.base.AmisDetailResponse;
import com.lanyuanxiaoyao.service.web.controller.base.AmisResponse;
@@ -26,12 +26,12 @@ import org.springframework.web.bind.annotation.RestController;
public class LogController extends BaseController {
private static final Logger logger = LoggerFactory.getLogger(LogController.class);
private final LokiService lokiService;
private final LokiHudiService lokiHudiService;
private final InfoService infoService;
@SuppressWarnings("SpringJavaInjectionPointsAutowiringInspection")
public LogController(LokiService lokiService, InfoService infoService) {
this.lokiService = lokiService;
public LogController(LokiHudiService lokiHudiService, InfoService infoService) {
this.lokiHudiService = lokiHudiService;
this.infoService = infoService;
}
@@ -43,7 +43,7 @@ public class LogController extends BaseController {
@GetMapping("query_sync_log")
public AmisDetailResponse querySyncLog(@RequestParam("flink_job_id") Long flinkJobId) {
return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.<String, String>builder()
return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.<String, String>builder()
.put("app", "hudi-sync")
.put("run_type", "sync")
.put("flink_job_id", flinkJobId.toString())
@@ -56,7 +56,7 @@ public class LogController extends BaseController {
@GetMapping("query_compaction_log")
public AmisDetailResponse queryCompactionLog(@RequestParam("flink_job_id") Long flinkJobId, @RequestParam("alias") String alias) {
return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.<String, String>builder()
return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.<String, String>builder()
.put("app", "hudi-sync")
.put("run_type", "compaction")
.put("flink_job_id", flinkJobId.toString())
@@ -70,7 +70,7 @@ public class LogController extends BaseController {
@GetMapping("query_application_log")
public AmisDetailResponse queryApplicationLog(@RequestParam("application_id") String applicationId) {
return AmisResponse.responseDetailData(lokiService.queryRange(MapUtil.<String, String>builder()
return AmisResponse.responseDetailData(lokiHudiService.queryRange(MapUtil.<String, String>builder()
.put("app", "hudi-sync")
.put("app_id", applicationId)
.build())