refactor(all): 初始化项目
迁移项目到独立的 service 进行集中开发 包含以下组件的查询 API 服务 flink hudi database(info) pulsar yarn 包含前端服务和 UI web 包含公共代码 configuration
This commit is contained in:
@@ -0,0 +1,249 @@
|
||||
package com.lanyuanxiaoyao.service.pulsar;
|
||||
|
||||
import cn.hutool.core.util.ObjectUtil;
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import com.eshore.odcp.hudi.connector.exception.PulsarInfoNotFoundException;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.pulsar.PulsarInfo;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.pulsar.PulsarNamespace;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.pulsar.PulsarTenant;
|
||||
import com.lanyuanxiaoyao.service.configuration.entity.pulsar.PulsarTopic;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import org.apache.pulsar.client.admin.PulsarAdmin;
|
||||
import org.apache.pulsar.client.admin.PulsarAdminException;
|
||||
import org.apache.pulsar.client.api.MessageId;
|
||||
import org.apache.pulsar.client.api.PulsarClientException;
|
||||
import org.apache.pulsar.common.policies.data.*;
|
||||
import org.eclipse.collections.api.factory.Lists;
|
||||
import org.eclipse.collections.api.list.ImmutableList;
|
||||
import org.eclipse.collections.api.list.MutableList;
|
||||
import org.springframework.boot.SpringApplication;
|
||||
import org.springframework.boot.autoconfigure.SpringBootApplication;
|
||||
import org.springframework.boot.autoconfigure.gson.GsonAutoConfiguration;
|
||||
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
import org.springframework.context.annotation.ComponentScans;
|
||||
import org.springframework.retry.annotation.EnableRetry;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* @author lanyuanxiaoyao
|
||||
* @date 2023-04-27
|
||||
*/
|
||||
@EnableDiscoveryClient
|
||||
@SpringBootApplication(exclude = {GsonAutoConfiguration.class, DataSourceAutoConfiguration.class})
|
||||
@ComponentScans({
|
||||
@ComponentScan("com.lanyuanxiaoyao.service")
|
||||
})
|
||||
@EnableConfigurationProperties
|
||||
@EnableRetry
|
||||
@RestController
|
||||
@RequestMapping("pulsar")
|
||||
public class PulsarQueryApplication {
|
||||
public static void main(String[] args) {
|
||||
SpringApplication.run(PulsarQueryApplication.class, args);
|
||||
}
|
||||
|
||||
private PulsarInfo getInfo(String name) {
|
||||
if (PulsarInfo.PULSAR_NAME_INFO_MAPPING.containsKey(name)) {
|
||||
return PulsarInfo.PULSAR_NAME_INFO_MAPPING.get(name);
|
||||
}
|
||||
throw new PulsarInfoNotFoundException("Pulsar info not found for " + name);
|
||||
}
|
||||
|
||||
private String adminUrl(PulsarInfo info) {
|
||||
return StrUtil.format("http://{}/admin/v2", info.getAdmin());
|
||||
}
|
||||
|
||||
@RequestMapping("names")
|
||||
public ImmutableList<String> names() {
|
||||
return Lists.immutable.ofAll(PulsarInfo.DEFAULT_INFOS).collect(PulsarInfo::getName);
|
||||
}
|
||||
|
||||
@RequestMapping("name")
|
||||
public String name(@RequestParam("pulsar_url") String pulsarUrl) {
|
||||
String[] urls = pulsarUrl.replaceAll("pulsar://", "").split(",");
|
||||
if (ObjectUtil.isNotEmpty(urls)) {
|
||||
return PulsarInfo.PULSAR_CLIENT_NAME_MAPPING.get(urls[0]);
|
||||
}
|
||||
throw new PulsarInfoNotFoundException("Pulsar name not found for " + pulsarUrl);
|
||||
}
|
||||
|
||||
@RequestMapping("tenants")
|
||||
public ImmutableList<PulsarTenant> tenants(@RequestParam("name") String name) throws PulsarClientException, PulsarAdminException {
|
||||
PulsarInfo info = getInfo(name);
|
||||
try (PulsarAdmin admin = PulsarAdmin.builder()
|
||||
.serviceHttpUrl(adminUrl(info))
|
||||
.build()) {
|
||||
MutableList<PulsarTenant> result = Lists.mutable.empty();
|
||||
for (String tenant : admin.tenants().getTenants()) {
|
||||
result.add(new PulsarTenant(tenant));
|
||||
}
|
||||
return result.toImmutable();
|
||||
}
|
||||
}
|
||||
|
||||
@RequestMapping("namespaces")
|
||||
public ImmutableList<PulsarNamespace> namespaces(@RequestParam("name") String name, @RequestParam("tenant") String tenant) throws PulsarClientException, PulsarAdminException {
|
||||
PulsarInfo info = getInfo(name);
|
||||
try (PulsarAdmin admin = PulsarAdmin.builder()
|
||||
.serviceHttpUrl(adminUrl(info))
|
||||
.build()) {
|
||||
MutableList<PulsarNamespace> result = Lists.mutable.empty();
|
||||
for (String namespace : admin.namespaces().getNamespaces(tenant)) {
|
||||
result.add(new PulsarNamespace(tenant, namespace));
|
||||
}
|
||||
return result.toImmutable();
|
||||
}
|
||||
}
|
||||
|
||||
@RequestMapping("topics")
|
||||
public ImmutableList<PulsarTopic> topics(@RequestParam("name") String name, @RequestParam("namespace") String namespace) throws PulsarClientException, PulsarAdminException {
|
||||
PulsarInfo info = getInfo(name);
|
||||
try (PulsarAdmin admin = PulsarAdmin.builder()
|
||||
.serviceHttpUrl(adminUrl(info))
|
||||
.build()) {
|
||||
MutableList<PulsarTopic> result = Lists.mutable.empty();
|
||||
for (String topic : admin.topics().getList(namespace)) {
|
||||
result.add(convert(admin, topic));
|
||||
}
|
||||
return result.toImmutable();
|
||||
}
|
||||
}
|
||||
|
||||
@RequestMapping("topic")
|
||||
public PulsarTopic topic(@RequestParam("name") String name, @RequestParam("topic") String topic) throws PulsarClientException, PulsarAdminException {
|
||||
PulsarInfo info = getInfo(name);
|
||||
try (PulsarAdmin admin = PulsarAdmin.builder()
|
||||
.serviceHttpUrl(adminUrl(info))
|
||||
.build()) {
|
||||
return convert(admin, topic);
|
||||
}
|
||||
}
|
||||
|
||||
private static PulsarTopic convert(PulsarAdmin admin, String topic) throws PulsarAdminException {
|
||||
TopicStats stats = admin.topics().getStats(topic);
|
||||
MessageId lastMessageId = admin.topics().getLastMessageId(topic);
|
||||
List<String> subscriptions = admin.topics().getSubscriptions(topic);
|
||||
return new PulsarTopic(
|
||||
topic,
|
||||
ObjectUtil.isNull(stats) ? null : new PulsarTopic.State(
|
||||
stats.getMsgRateIn(),
|
||||
stats.getMsgRateOut(),
|
||||
stats.getMsgThroughputIn(),
|
||||
stats.getMsgThroughputOut(),
|
||||
stats.getBytesInCounter(),
|
||||
stats.getBytesOutCounter(),
|
||||
stats.getMsgInCounter(),
|
||||
stats.getMsgOutCounter(),
|
||||
stats.getAverageMsgSize(),
|
||||
stats.isMsgChunkPublished(),
|
||||
stats.getStorageSize(),
|
||||
stats.getBacklogSize(),
|
||||
stats.getOffloadedStorageSize(),
|
||||
ObjectUtil.isNull(stats.getPublishers()) ? null : new ArrayList<PulsarTopic.State.PublisherState>() {{
|
||||
for (PublisherStats publisher : stats.getPublishers()) {
|
||||
add(new PulsarTopic.State.PublisherState(
|
||||
publisher.getAccessMode().name(),
|
||||
publisher.getMsgRateIn(),
|
||||
publisher.getMsgThroughputIn(),
|
||||
publisher.getAverageMsgSize(),
|
||||
publisher.getChunkedMessageRate(),
|
||||
publisher.getProducerId(),
|
||||
publisher.getProducerName(),
|
||||
publisher.getAddress(),
|
||||
publisher.getConnectedSince(),
|
||||
publisher.getClientVersion(),
|
||||
publisher.getMetadata()
|
||||
));
|
||||
}
|
||||
}},
|
||||
stats.getWaitingPublishers(),
|
||||
ObjectUtil.isNull(stats.getSubscriptions()) ? null : new HashMap<String, PulsarTopic.State.SubscriptionState>() {{
|
||||
for (Entry<String, ? extends SubscriptionStats> entry : stats.getSubscriptions().entrySet()) {
|
||||
SubscriptionStats state = entry.getValue();
|
||||
put(entry.getKey(), new PulsarTopic.State.SubscriptionState(
|
||||
state.getMsgRateOut(),
|
||||
state.getMsgThroughputOut(),
|
||||
state.getBytesOutCounter(),
|
||||
state.getMsgOutCounter(),
|
||||
state.getMsgRateRedeliver(),
|
||||
state.getChunkedMessageRate(),
|
||||
state.getMsgBacklog(),
|
||||
state.getBacklogSize(),
|
||||
state.getMsgBacklogNoDelayed(),
|
||||
state.isBlockedSubscriptionOnUnackedMsgs(),
|
||||
state.getMsgDelayed(),
|
||||
state.getUnackedMessages(),
|
||||
state.getType(),
|
||||
state.getActiveConsumerName(),
|
||||
state.getMsgRateExpired(),
|
||||
state.getTotalMsgExpired(),
|
||||
state.getLastExpireTimestamp(),
|
||||
state.getLastConsumedFlowTimestamp(),
|
||||
state.getLastConsumedTimestamp(),
|
||||
state.getLastAckedTimestamp(),
|
||||
state.getLastMarkDeleteAdvancedTimestamp(),
|
||||
new ArrayList<PulsarTopic.State.ConsumerState>() {{
|
||||
for (ConsumerStats consumer : state.getConsumers()) {
|
||||
add(new PulsarTopic.State.ConsumerState(
|
||||
consumer.getMsgRateOut(),
|
||||
consumer.getMsgThroughputOut(),
|
||||
consumer.getBytesOutCounter(),
|
||||
consumer.getMsgOutCounter(),
|
||||
consumer.getMsgRateRedeliver(),
|
||||
consumer.getChunkedMessageRate(),
|
||||
consumer.getConsumerName(),
|
||||
consumer.getAvailablePermits(),
|
||||
consumer.getUnackedMessages(),
|
||||
consumer.getAvgMessagesPerEntry(),
|
||||
consumer.isBlockedConsumerOnUnackedMsgs(),
|
||||
consumer.getReadPositionWhenJoining(),
|
||||
consumer.getAddress(),
|
||||
consumer.getConnectedSince(),
|
||||
consumer.getClientVersion(),
|
||||
consumer.getLastAckedTimestamp(),
|
||||
consumer.getLastConsumedTimestamp(),
|
||||
consumer.getKeyHashRanges(),
|
||||
consumer.getMetadata()
|
||||
));
|
||||
}
|
||||
}},
|
||||
state.isDurable(),
|
||||
state.isReplicated(),
|
||||
state.getConsumersAfterMarkDeletePosition()
|
||||
));
|
||||
}
|
||||
}},
|
||||
ObjectUtil.isNull(stats.getReplication()) ? null : new HashMap<String, PulsarTopic.State.ReplicateState>() {{
|
||||
for (Entry<String, ? extends ReplicatorStats> entry : stats.getReplication().entrySet()) {
|
||||
ReplicatorStats state = entry.getValue();
|
||||
put(entry.getKey(), new PulsarTopic.State.ReplicateState(
|
||||
state.getMsgRateIn(),
|
||||
state.getMsgRateOut(),
|
||||
state.getMsgThroughputIn(),
|
||||
state.getMsgThroughputOut(),
|
||||
state.getMsgRateExpired(),
|
||||
state.getReplicationBacklog(),
|
||||
state.isConnected(),
|
||||
state.getReplicationDelayInSeconds(),
|
||||
state.getInboundConnection(),
|
||||
state.getInboundConnectedSince(),
|
||||
state.getOutboundConnection(),
|
||||
state.getOutboundConnectedSince()
|
||||
));
|
||||
}
|
||||
}},
|
||||
stats.getDeduplicationStatus(),
|
||||
stats.getTopicEpoch()
|
||||
),
|
||||
lastMessageId.toString(),
|
||||
subscriptions
|
||||
);
|
||||
}
|
||||
}
|
||||
27
service-pulsar-query/src/main/resources/application.yml
Normal file
27
service-pulsar-query/src/main/resources/application.yml
Normal file
@@ -0,0 +1,27 @@
|
||||
server:
|
||||
port: 0
|
||||
spring:
|
||||
application:
|
||||
name: service-pulsar-query
|
||||
main:
|
||||
banner-mode: off
|
||||
management:
|
||||
endpoints:
|
||||
web:
|
||||
exposure:
|
||||
include: "*"
|
||||
endpoint:
|
||||
prometheus:
|
||||
enabled: true
|
||||
metrics:
|
||||
export:
|
||||
jmx:
|
||||
enabled: false
|
||||
eureka:
|
||||
instance:
|
||||
hostname: localhost
|
||||
prefer-ip-address: true
|
||||
instance-id: ${spring.application.name}-${eureka.instance.hostname}-${random.uuid}-${datetime}
|
||||
client:
|
||||
service-url:
|
||||
defaultZone: http://localhost:35670/eureka/
|
||||
50
service-pulsar-query/src/main/resources/logback-spring.xml
Normal file
50
service-pulsar-query/src/main/resources/logback-spring.xml
Normal file
@@ -0,0 +1,50 @@
|
||||
<configuration>
|
||||
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
|
||||
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
|
||||
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
|
||||
|
||||
<springProperty scope="context" name="LOKI_PUSH_URL" source="loki.url"/>
|
||||
<springProperty scope="context" name="LOGGING_PARENT" source="logging.parent"/>
|
||||
<springProperty scope="context" name="APP_NAME" source="spring.application.name"/>
|
||||
|
||||
<appender name="Loki" class="com.github.loki4j.logback.Loki4jAppender">
|
||||
<metricsEnabled>true</metricsEnabled>
|
||||
<http class="com.github.loki4j.logback.ApacheHttpSender">
|
||||
<url>${LOKI_PUSH_URL:-http://localhost/loki/api/v1/push}</url>
|
||||
</http>
|
||||
<format>
|
||||
<label>
|
||||
<pattern>app=${APP_NAME:- },host=${HOSTNAME},level=%level</pattern>
|
||||
</label>
|
||||
<message>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</message>
|
||||
<sortByTime>true</sortByTime>
|
||||
</format>
|
||||
</appender>
|
||||
|
||||
<appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
|
||||
<encoder>
|
||||
<pattern>${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${LOGGING_PARENT:-.}/${APP_NAME:-run}.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<fileNamePattern>${LOGGING_PARENT:-.}/archive/${APP_NAME:-run}-%d{yyyy-MM-dd}.gz</fileNamePattern>
|
||||
<MaxHistory>7</MaxHistory>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${FILE_LOG_PATTERN:-%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}} [${HOSTNAME}] ${LOG_LEVEL_PATTERN:-%5p} ${PID:- } --- [%t] %-40.40logger{39} : %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<logger name="com.zaxxer.hikari" level="ERROR"/>
|
||||
|
||||
<root level="INFO">
|
||||
<appender-ref ref="Loki"/>
|
||||
<appender-ref ref="Console"/>
|
||||
<appender-ref ref="RollingFile"/>
|
||||
</root>
|
||||
</configuration>
|
||||
Reference in New Issue
Block a user