lanyuanxiaoyao/hudi-service
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(all): 迁移common、sync、executor项目

Browse Source
This commit is contained in:
兰缘小妖
2024-02-29 15:32:14 +08:00
parent 0683068a02
commit 5a2e9fdfb8
73 changed files with 10204 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
utils/executor/pom.xml Normal file
View File

@@ -0,0 +1,161 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>com.lanyuanxiaoyao</groupId>
<artifactId>hudi-service</artifactId>
<version>1.0.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>executor</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime-web_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-optimizer_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-metrics-core</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-metrics-prometheus_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-container_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-yarn_${scala.major.version}</artifactId>
<version>${flink.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-hadoop-fs</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-uber_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-uber-blink_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-state-processor-api_${scala.major.version}</artifactId>
<version>${flink.version}</version>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-shaded-hadoop-3-uber</artifactId>
<version>3.1.1.7.2.9.0-173-9.0</version>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-format</artifactId>
<version>2.4.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<createDependencyReducedPom>false</createDependencyReducedPom>
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>com.eshore.odcp.hudi.connector.utils.executor.Runner</mainClass>
</transformer>
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>reference.conf</resource>
</transformer>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>log4j-surefire*.properties</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

43
utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/Runner.java Normal file
View File

@@ -0,0 +1,43 @@
package com.lanyuanxiaoyao.service.executor;
import org.apache.flink.client.deployment.ClusterClientFactory;
import org.apache.flink.client.deployment.ClusterDescriptor;
import org.apache.flink.client.deployment.ClusterSpecification;
import org.apache.flink.client.deployment.DefaultClusterClientServiceLoader;
import org.apache.flink.client.deployment.application.ApplicationConfiguration;
import org.apache.flink.client.program.ClusterClient;
import org.apache.flink.client.program.ClusterClientProvider;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.security.SecurityConfiguration;
import org.apache.flink.runtime.security.SecurityUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
/**
* 启动类 #
*
* @author ZhangJiacheng
* @date 2022-06-01
*/
public class Runner {
public static ApplicationId run(Configuration inputConfiguration, String className, String[] args) throws Exception {
Configuration configuration = new Configuration(inputConfiguration);
SecurityUtils.install(new SecurityConfiguration(configuration));
return SecurityUtils.getInstalledContext()
.runSecured(() -> {
DefaultClusterClientServiceLoader yarnServiceLoader = new DefaultClusterClientServiceLoader();
ApplicationConfiguration applicationConfiguration = new ApplicationConfiguration(args, className);
//ApplicationDeployer deployer = new ApplicationClusterDeployer(new DefaultClusterClientServiceLoader());
//deployer.run(configuration, applicationConfiguration);
final ClusterClientFactory<ApplicationId> clientFactory = yarnServiceLoader.getClusterClientFactory(configuration);
try (final ClusterDescriptor<ApplicationId> clusterDescriptor = clientFactory.createClusterDescriptor(configuration)) {
final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(configuration);
ClusterClientProvider<ApplicationId> provider = clusterDescriptor.deployApplicationCluster(clusterSpecification, applicationConfiguration);
ClusterClient<ApplicationId> clusterClient = provider.getClusterClient();
if (clusterClient == null) {
return null;
}
return clusterClient.getClusterId();
}
});
}
}

48
utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsOptions.java Normal file
View File

@@ -0,0 +1,48 @@
package com.lanyuanxiaoyao.service.executor.metrics;
import org.apache.flink.annotation.docs.Documentation;
import org.apache.flink.configuration.ConfigOption;
import static org.apache.flink.configuration.ConfigOptions.key;
/**
* Config options for the {@link VictoriaMetricsReporter}.
*/
@Documentation.SuffixOption
public class VictoriaMetricsOptions {
public static final ConfigOption<String> ENDPOINT =
key("endpoint")
.stringType()
.noDefaultValue()
.withDescription("Victoria metrics endpoint. eg: http://localhost:8428/api/v1/import/prometheus");
public static final ConfigOption<Integer> TIMEOUT =
key("timeout")
.intType()
.defaultValue(60000)
.withDescription("Http push timeout. Default 1 minute");
public static final ConfigOption<String> TAGS =
key("tags")
.stringType()
.defaultValue("")
.withDescription("Extra tags for every metric");
public static final ConfigOption<Boolean> ENABLE_AUTH =
key("enable.auth")
.booleanType()
.defaultValue(false)
.withDescription("Enable metric server http basic auth");
public static final ConfigOption<String> AUTH_USERNAME =
key("auth.username")
.stringType()
.defaultValue("")
.withDescription("Basic auth username");
public static final ConfigOption<String> AUTH_PASSWORD =
key("auth.password")
.stringType()
.defaultValue("")
.withDescription("Basic auth password");
}

64
utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporter.java Normal file
View File

@@ -0,0 +1,64 @@
package com.lanyuanxiaoyao.service.executor.metrics;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import io.prometheus.client.CollectorRegistry;
import io.prometheus.client.exporter.common.TextFormat;
import java.io.IOException;
import java.io.StringWriter;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.flink.metrics.prometheus.AbstractPrometheusReporter;
import org.apache.flink.metrics.reporter.Scheduled;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author ZhangJiacheng
* @date 2022-06-22
*/
public class VictoriaMetricsReporter extends AbstractPrometheusReporter implements Scheduled {
private static final Logger logger = LoggerFactory.getLogger(VictoriaMetricsReporter.class);
private final String endpoint;
private final Integer timout;
private final Map<String, String> tags;
private final Boolean enableBasicAuth;
private final String basicAuthUsername;
private final String basicAuthPassword;
public VictoriaMetricsReporter(String endpoint, Integer timout, Map<String, String> tags, Boolean enableBasicAuth, String basicAuthUsername, String basicAuthPassword) {
this.endpoint = endpoint;
this.timout = timout;
this.tags = tags;
this.enableBasicAuth = enableBasicAuth;
this.basicAuthUsername = basicAuthUsername;
this.basicAuthPassword = basicAuthPassword;
}
@Override
public void report() {
try (StringWriter writer = new StringWriter()) {
TextFormat.write004(writer, CollectorRegistry.defaultRegistry.metricFamilySamples());
String query = tags.entrySet()
.stream()
.map(entry -> StrUtil.format("extra_label={}={}", entry.getKey(), entry.getValue()))
.collect(Collectors.joining("&"));
HttpRequest request = HttpUtil.createPost(StrUtil.format("{}?{}", endpoint, query))
.body(writer.toString())
.timeout(timout);
if (enableBasicAuth) {
request.basicAuth(basicAuthUsername, basicAuthPassword);
}
HttpResponse response = request.execute();
if (!response.isOk()) {
logger.warn("Fail to push metrics: {}, {}, endpoint: {}, tags: {}", response.getStatus(), response.body(), endpoint, tags);
}
} catch (IOException e) {
logger.error("Fail to write metrics, endpoint: {}, tags: {}, exception: {}", endpoint, tags, e);
}
}
}

44
utils/executor/src/main/java/com/lanyuanxiaoyao/service/executor/metrics/VictoriaMetricsReporterFactory.java Normal file
View File

@@ -0,0 +1,44 @@
package com.lanyuanxiaoyao.service.executor.metrics;
import cn.hutool.core.util.StrUtil;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import org.apache.flink.metrics.MetricConfig;
import org.apache.flink.metrics.reporter.InterceptInstantiationViaReflection;
import org.apache.flink.metrics.reporter.MetricReporterFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static com.lanyuanxiaoyao.service.executor.metrics.VictoriaMetricsOptions.*;
/**
* @author ZhangJiacheng
* @date 2022-06-22
*/
@InterceptInstantiationViaReflection(
reporterClassName = "com.eshore.odcp.hudi.connector.utils.executor.metrics.VictoriaMetricsReporter")
public class VictoriaMetricsReporterFactory implements MetricReporterFactory {
private static final Logger logger = LoggerFactory.getLogger(VictoriaMetricsReporterFactory.class);
@Override
public VictoriaMetricsReporter createMetricReporter(Properties properties) {
MetricConfig metricConfig = (MetricConfig) properties;
String endpoint = metricConfig.getString(ENDPOINT.key(), ENDPOINT.defaultValue());
int timeout = metricConfig.getInteger(TIMEOUT.key(), TIMEOUT.defaultValue());
String tagsText = metricConfig.getString(TAGS.key(), TAGS.defaultValue());
Boolean enableAuth = metricConfig.getBoolean(ENABLE_AUTH.key(), ENABLE_AUTH.defaultValue());
String authUsername = metricConfig.getString(AUTH_USERNAME.key(), AUTH_USERNAME.defaultValue());
String authPassword = metricConfig.getString(AUTH_PASSWORD.key(), AUTH_PASSWORD.defaultValue());
Map<String, String> tags = new HashMap<>(10);
if (StrUtil.isNotBlank(tagsText)) {
for (String item : tagsText.split(";")) {
String[] parsed = item.split("=");
tags.put(parsed[0], parsed[1]);
}
}
logger.info("Create victoria metric reporter for endpoint {} timeout: {}, tags: {}, enable_auth: {}, auth_username: {}, auth_password: {}", endpoint, timeout, tags, enableAuth, authUsername, authPassword);
return new VictoriaMetricsReporter(endpoint, timeout, tags, enableAuth, authUsername, authPassword);
}
}

54
utils/executor/src/main/resources/log4j.properties Normal file
View File

@@ -0,0 +1,54 @@
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
# Allows this configuration to be modified at runtime. The file will be checked every 30 seconds.
monitorInterval=30
# This affects logging for both user code and Flink
rootLogger.level = INFO
rootLogger.appenderRef.file.ref = MainAppender
# Uncomment this if you want to _only_ change Flink's logging
#logger.flink.name = org.apache.flink
#logger.flink.level = INFO
# The following lines keep the log level of common libraries/connectors on
# log level INFO. The root logger does not override this. You have to manually
# change the log levels here.
logger.akka.name = akka
logger.akka.level = INFO
logger.kafka.name= org.apache.kafka
logger.kafka.level = INFO
logger.hadoop.name = org.apache.hadoop
logger.hadoop.level = INFO
logger.zookeeper.name = org.apache.zookeeper
logger.zookeeper.level = INFO
logger.shaded_zookeeper.name = org.apache.flink.shaded.zookeeper3
logger.shaded_zookeeper.level = INFO
logger.hudi.name=org.apache.hudi
logger.hudi.level=INFO
# Log all infos in the given file
appender.main.name = MainAppender
appender.main.type = Console
appender.main.layout.type = PatternLayout
appender.main.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
# Suppress the irrelevant (wrong) warnings from the Netty channel handler
logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline
logger.netty.level = OFF

1
utils/executor/src/main/resources/log4j2.properties Normal file
View File

@@ -0,0 +1 @@
log4j.rootLogger=INFO

16
utils/executor/src/main/resources/services/org.apache.flink.client.deployment.ClusterClientFactory Normal file
View File

@@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.flink.yarn.YarnClusterClientFactory

17
utils/executor/src/main/resources/services/org.apache.flink.core.execution.PipelineExecutorFactory Normal file
View File

@@ -0,0 +1,17 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.flink.yarn.executors.YarnJobClusterExecutorFactory
org.apache.flink.yarn.executors.YarnSessionClusterExecutorFactory

1
utils/executor/src/main/resources/services/org.apache.flink.metrics.reporter.MetricReporterFactory Normal file
View File

@@ -0,0 +1 @@
com.eshore.odcp.hudi.connector.utils.executor.metrics.VictoriaMetricsReporterFactory

18
utils/executor/src/main/resources/services/org.apache.flink.table.factories.TableFactory Normal file
View File

@@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.flink.table.planner.delegation.BlinkPlannerFactory
org.apache.flink.table.planner.delegation.BlinkExecutorFactory
org.apache.flink.table.planner.delegation.DefaultParserFactory