feat(all): 迁移common、sync、executor项目
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
package com.lanyuanxiaoyao.service.executor;
|
||||
|
||||
import org.apache.flink.client.deployment.ClusterClientFactory;
|
||||
import org.apache.flink.client.deployment.ClusterDescriptor;
|
||||
import org.apache.flink.client.deployment.ClusterSpecification;
|
||||
import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader;
|
||||
import org.apache.flink.client.deployment.application.ApplicationConfiguration;
|
||||
import org.apache.flink.client.program.ClusterClient;
|
||||
import org.apache.flink.client.program.ClusterClientProvider;
|
||||
import org.apache.flink.configuration.Configuration;
|
||||
import org.apache.flink.runtime.security.SecurityConfiguration;
|
||||
import org.apache.flink.runtime.security.SecurityUtils;
|
||||
import org.apache.hadoop.yarn.api.records.ApplicationId;
|
||||
|
||||
/**
|
||||
* 启动类 #
|
||||
*
|
||||
* @author ZhangJiacheng
|
||||
* @date 2022-06-01
|
||||
*/
|
||||
public class Runner {
|
||||
public static ApplicationId run(Configuration inputConfiguration, String className, String[] args) throws Exception {
|
||||
Configuration configuration = new Configuration(inputConfiguration);
|
||||
SecurityUtils.install(new SecurityConfiguration(configuration));
|
||||
return SecurityUtils.getInstalledContext()
|
||||
.runSecured(() -> {
|
||||
DefaultClusterClientServiceLoader yarnServiceLoader = new DefaultClusterClientServiceLoader();
|
||||
ApplicationConfiguration applicationConfiguration = new ApplicationConfiguration(args, className);
|
||||
//ApplicationDeployer deployer = new ApplicationClusterDeployer(new DefaultClusterClientServiceLoader());
|
||||
//deployer.run(configuration, applicationConfiguration);
|
||||
final ClusterClientFactory<ApplicationId> clientFactory = yarnServiceLoader.getClusterClientFactory(configuration);
|
||||
try (final ClusterDescriptor<ApplicationId> clusterDescriptor = clientFactory.createClusterDescriptor(configuration)) {
|
||||
final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(configuration);
|
||||
ClusterClientProvider<ApplicationId> provider = clusterDescriptor.deployApplicationCluster(clusterSpecification, applicationConfiguration);
|
||||
ClusterClient<ApplicationId> clusterClient = provider.getClusterClient();
|
||||
if (clusterClient == null) {
|
||||
return null;
|
||||
}
|
||||
return clusterClient.getClusterId();
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package com.lanyuanxiaoyao.service.executor.metrics;
|
||||
|
||||
import org.apache.flink.annotation.docs.Documentation;
|
||||
import org.apache.flink.configuration.ConfigOption;
|
||||
|
||||
import static org.apache.flink.configuration.ConfigOptions.key;
|
||||
|
||||
/**
|
||||
* Config options for the {@link VictoriaMetricsReporter}.
|
||||
*/
|
||||
@Documentation.SuffixOption
|
||||
public class VictoriaMetricsOptions {
|
||||
public static final ConfigOption<String> ENDPOINT =
|
||||
key("endpoint")
|
||||
.stringType()
|
||||
.noDefaultValue()
|
||||
.withDescription("Victoria metrics endpoint. eg: http://localhost:8428/api/v1/import/prometheus");
|
||||
|
||||
public static final ConfigOption<Integer> TIMEOUT =
|
||||
key("timeout")
|
||||
.intType()
|
||||
.defaultValue(60000)
|
||||
.withDescription("Http push timeout. Default 1 minute");
|
||||
|
||||
public static final ConfigOption<String> TAGS =
|
||||
key("tags")
|
||||
.stringType()
|
||||
.defaultValue("")
|
||||
.withDescription("Extra tags for every metric");
|
||||
|
||||
public static final ConfigOption<Boolean> ENABLE_AUTH =
|
||||
key("enable.auth")
|
||||
.booleanType()
|
||||
.defaultValue(false)
|
||||
.withDescription("Enable metric server http basic auth");
|
||||
|
||||
public static final ConfigOption<String> AUTH_USERNAME =
|
||||
key("auth.username")
|
||||
.stringType()
|
||||
.defaultValue("")
|
||||
.withDescription("Basic auth username");
|
||||
|
||||
public static final ConfigOption<String> AUTH_PASSWORD =
|
||||
key("auth.password")
|
||||
.stringType()
|
||||
.defaultValue("")
|
||||
.withDescription("Basic auth password");
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package com.lanyuanxiaoyao.service.executor.metrics;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import cn.hutool.http.HttpUtil;
|
||||
import io.prometheus.client.CollectorRegistry;
|
||||
import io.prometheus.client.exporter.common.TextFormat;
|
||||
import java.io.IOException;
|
||||
import java.io.StringWriter;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.flink.metrics.prometheus.AbstractPrometheusReporter;
|
||||
import org.apache.flink.metrics.reporter.Scheduled;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
/**
|
||||
* @author ZhangJiacheng
|
||||
* @date 2022-06-22
|
||||
*/
|
||||
public class VictoriaMetricsReporter extends AbstractPrometheusReporter implements Scheduled {
|
||||
private static final Logger logger = LoggerFactory.getLogger(VictoriaMetricsReporter.class);
|
||||
|
||||
private final String endpoint;
|
||||
private final Integer timout;
|
||||
private final Map<String, String> tags;
|
||||
private final Boolean enableBasicAuth;
|
||||
private final String basicAuthUsername;
|
||||
private final String basicAuthPassword;
|
||||
|
||||
public VictoriaMetricsReporter(String endpoint, Integer timout, Map<String, String> tags, Boolean enableBasicAuth, String basicAuthUsername, String basicAuthPassword) {
|
||||
this.endpoint = endpoint;
|
||||
this.timout = timout;
|
||||
this.tags = tags;
|
||||
this.enableBasicAuth = enableBasicAuth;
|
||||
this.basicAuthUsername = basicAuthUsername;
|
||||
this.basicAuthPassword = basicAuthPassword;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void report() {
|
||||
try (StringWriter writer = new StringWriter()) {
|
||||
TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples());
|
||||
|
||||
String query = tags.entrySet()
|
||||
.stream()
|
||||
.map(entry -> StrUtil.format("extra_label={}={}", entry.getKey(), entry.getValue()))
|
||||
.collect(Collectors.joining("&"));
|
||||
HttpRequest request = HttpUtil.createPost(StrUtil.format("{}?{}", endpoint, query))
|
||||
.body(writer.toString())
|
||||
.timeout(timout);
|
||||
if (enableBasicAuth) {
|
||||
request.basicAuth(basicAuthUsername, basicAuthPassword);
|
||||
}
|
||||
HttpResponse response = request.execute();
|
||||
if (!response.isOk()) {
|
||||
logger.warn("Fail to push metrics: {}, {}, endpoint: {}, tags: {}", response.getStatus(), response.body(), endpoint, tags);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
logger.error("Fail to write metrics, endpoint: {}, tags: {}, exception: {}", endpoint, tags, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package com.lanyuanxiaoyao.service.executor.metrics;
|
||||
|
||||
import cn.hutool.core.util.StrUtil;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import org.apache.flink.metrics.MetricConfig;
|
||||
import org.apache.flink.metrics.reporter.InterceptInstantiationViaReflection;
|
||||
import org.apache.flink.metrics.reporter.MetricReporterFactory;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import static com.lanyuanxiaoyao.service.executor.metrics.VictoriaMetricsOptions.*;
|
||||
|
||||
/**
|
||||
* @author ZhangJiacheng
|
||||
* @date 2022-06-22
|
||||
*/
|
||||
@InterceptInstantiationViaReflection(
|
||||
reporterClassName = "com.eshore.odcp.hudi.connector.utils.executor.metrics.VictoriaMetricsReporter")
|
||||
public class VictoriaMetricsReporterFactory implements MetricReporterFactory {
|
||||
private static final Logger logger = LoggerFactory.getLogger(VictoriaMetricsReporterFactory.class);
|
||||
|
||||
@Override
|
||||
public VictoriaMetricsReporter createMetricReporter(Properties properties) {
|
||||
MetricConfig metricConfig = (MetricConfig) properties;
|
||||
String endpoint = metricConfig.getString(ENDPOINT.key(), ENDPOINT.defaultValue());
|
||||
int timeout = metricConfig.getInteger(TIMEOUT.key(), TIMEOUT.defaultValue());
|
||||
String tagsText = metricConfig.getString(TAGS.key(), TAGS.defaultValue());
|
||||
Boolean enableAuth = metricConfig.getBoolean(ENABLE_AUTH.key(), ENABLE_AUTH.defaultValue());
|
||||
String authUsername = metricConfig.getString(AUTH_USERNAME.key(), AUTH_USERNAME.defaultValue());
|
||||
String authPassword = metricConfig.getString(AUTH_PASSWORD.key(), AUTH_PASSWORD.defaultValue());
|
||||
|
||||
Map<String, String> tags = new HashMap<>(10);
|
||||
if (StrUtil.isNotBlank(tagsText)) {
|
||||
for (String item : tagsText.split(";")) {
|
||||
String[] parsed = item.split("=");
|
||||
tags.put(parsed[0], parsed[1]);
|
||||
}
|
||||
}
|
||||
logger.info("Create victoria metric reporter for endpoint {} timeout: {}, tags: {}, enable_auth: {}, auth_username: {}, auth_password: {}", endpoint, timeout, tags, enableAuth, authUsername, authPassword);
|
||||
return new VictoriaMetricsReporter(endpoint, timeout, tags, enableAuth, authUsername, authPassword);
|
||||
}
|
||||
}
|
||||
54
utils/executor/src/main/resources/log4j.properties
Normal file
54
utils/executor/src/main/resources/log4j.properties
Normal file
@@ -0,0 +1,54 @@
|
||||
################################################################################
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
################################################################################
|
||||
|
||||
# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds.
|
||||
monitorInterval=30
|
||||
|
||||
# This affects logging for both user code and Flink
|
||||
rootLogger.level = INFO
|
||||
rootLogger.appenderRef.file.ref = MainAppender
|
||||
|
||||
# Uncomment this if you want to _only_ change Flink's logging
|
||||
#logger.flink.name = org.apache.flink
|
||||
#logger.flink.level = INFO
|
||||
|
||||
# The following lines keep the log level of common libraries/connectors on
|
||||
# log level INFO. The root logger does not override this. You have to manually
|
||||
# change the log levels here.
|
||||
logger.akka.name = akka
|
||||
logger.akka.level = INFO
|
||||
logger.kafka.name= org.apache.kafka
|
||||
logger.kafka.level = INFO
|
||||
logger.hadoop.name = org.apache.hadoop
|
||||
logger.hadoop.level = INFO
|
||||
logger.zookeeper.name = org.apache.zookeeper
|
||||
logger.zookeeper.level = INFO
|
||||
logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3
|
||||
logger.shaded_zookeeper.level = INFO
|
||||
logger.hudi.name=org.apache.hudi
|
||||
logger.hudi.level=INFO
|
||||
|
||||
# Log all infos in the given file
|
||||
appender.main.name = MainAppender
|
||||
appender.main.type = Console
|
||||
appender.main.layout.type = PatternLayout
|
||||
appender.main.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
|
||||
|
||||
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
|
||||
logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline
|
||||
logger.netty.level = OFF
|
||||
1
utils/executor/src/main/resources/log4j2.properties
Normal file
1
utils/executor/src/main/resources/log4j2.properties
Normal file
@@ -0,0 +1 @@
|
||||
log4j.rootLogger=INFO
|
||||
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
org.apache.flink.yarn.YarnClusterClientFactory
|
||||
@@ -0,0 +1,17 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
org.apache.flink.yarn.executors.YarnJobClusterExecutorFactory
|
||||
org.apache.flink.yarn.executors.YarnSessionClusterExecutorFactory
|
||||
@@ -0,0 +1 @@
|
||||
com.eshore.odcp.hudi.connector.utils.executor.metrics.VictoriaMetricsReporterFactory
|
||||
@@ -0,0 +1,18 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
org.apache.flink.table.planner.delegation.BlinkPlannerFactory
|
||||
org.apache.flink.table.planner.delegation.BlinkExecutorFactory
|
||||
org.apache.flink.table.planner.delegation.DefaultParserFactory
|
||||
Reference in New Issue
Block a user