lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1122] Introduce a kafka implementation of hoodie write commit ca… (#1886)

Browse Source
This commit is contained in:
Mathieu
2020-08-20 23:00:59 +08:00
committed by GitHub
parent bd7814dadf
commit b883b6d268
8 changed files with 258 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
hudi-client/src/main/java/org/apache/hudi/callback/impl/HoodieWriteCommitHttpCallback.java
View File

@@ -21,12 +21,11 @@ import org.apache.hudi.callback.HoodieWriteCommitCallback;
import org.apache.hudi.callback.client.http.HoodieWriteCommitHttpCallbackClient;
import org.apache.hudi.callback.common.HoodieWriteCommitCallbackMessage;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieCommitCallbackException;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.codehaus.jackson.map.ObjectMapper;
import java.io.IOException;
import static org.apache.hudi.callback.util.HoodieWriteCommitCallbackUtil.convertToJsonString;
/**
* A http implementation of {@link HoodieWriteCommitCallback}.
@@ -44,13 +43,7 @@ public class HoodieWriteCommitHttpCallback implements HoodieWriteCommitCallback
@Override
public void call(HoodieWriteCommitCallbackMessage callbackMessage) {
// convert to json
ObjectMapper mapper = new ObjectMapper();
String callbackMsg = null;
try {
callbackMsg = mapper.writeValueAsString(callbackMessage);
} catch (IOException e) {
throw new HoodieCommitCallbackException("Callback service convert message to json failed", e);
}
String callbackMsg = convertToJsonString(callbackMessage);
LOG.info("Try to send callbackMsg, msg = " + callbackMsg);
client.send(callbackMsg);
}

44
hudi-client/src/main/java/org/apache/hudi/callback/util/HoodieWriteCommitCallbackUtil.java Normal file
View File

@@ -0,0 +1,44 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.callback.util;
import org.apache.hudi.exception.HoodieCommitCallbackException;
import org.codehaus.jackson.map.ObjectMapper;
import java.io.IOException;
/**
* Util helps to prepare callback message.
*/
public class HoodieWriteCommitCallbackUtil {
private static ObjectMapper mapper = new ObjectMapper();
/**
* Convert data to json string format.
*/
public static String convertToJsonString(Object obj) {
try {
return mapper.writeValueAsString(obj);
} catch (IOException e) {
throw new HoodieCommitCallbackException("Callback service convert data to json failed", e);
}
}
}

13
hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteCommitCallbackConfig.java
View File

@@ -29,17 +29,18 @@ import java.util.Properties;
*/
public class HoodieWriteCommitCallbackConfig extends DefaultHoodieConfig {
public static final String CALLBACK_ON = "hoodie.write.commit.callback.on";
public static final String CALLBACK_PREFIX = "hoodie.write.commit.callback.";
public static final String CALLBACK_ON = CALLBACK_PREFIX + "on";
public static final boolean DEFAULT_CALLBACK_ON = false;
public static final String CALLBACK_CLASS_PROP = "hoodie.write.commit.callback.class";
public static final String CALLBACK_CLASS_PROP = CALLBACK_PREFIX + "class";
public static final String DEFAULT_CALLBACK_CLASS_PROP = "org.apache.hudi.callback.impl.HoodieWriteCommitHttpCallback";
// ***** REST callback configs *****
public static final String CALLBACK_HTTP_URL_PROP = "hoodie.write.commit.callback.http.url";
public static final String CALLBACK_HTTP_API_KEY = "hoodie.write.commit.callback.http.api.key";
// ***** HTTP callback configs *****
public static final String CALLBACK_HTTP_URL_PROP = CALLBACK_PREFIX + "http.url";
public static final String CALLBACK_HTTP_API_KEY = CALLBACK_PREFIX + "http.api.key";
public static final String DEFAULT_CALLBACK_HTTP_API_KEY = "hudi_write_commit_http_callback";
public static final String CALLBACK_HTTP_TIMEOUT_SECONDS = "hoodie.write.commit.callback.http.timeout.seconds";
public static final String CALLBACK_HTTP_TIMEOUT_SECONDS = CALLBACK_PREFIX + "http.timeout.seconds";
public static final int DEFAULT_CALLBACK_HTTP_TIMEOUT_SECONDS = 3;
private HoodieWriteCommitCallbackConfig(Properties props) {