lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[checkstyle] Add ConstantName java checkstyle rule (#1066)

Browse Source 
* add SimplifyBooleanExpression java checkstyle rule
* collapse empty tags in scalastyle file
This commit is contained in:
lamber-ken
2019-12-04 18:59:15 +08:00
committed by leesf
parent b2d9638bea
commit b3e0ebbc4a
38 changed files with 192 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java
View File

@@ -176,8 +176,8 @@ public class KafkaOffsetGen {
private static final String KAFKA_TOPIC_NAME = "hoodie.deltastreamer.source.kafka.topic";
private static final String MAX_EVENTS_FROM_KAFKA_SOURCE_PROP = "hoodie.deltastreamer.kafka.source.maxEvents";
private static final KafkaResetOffsetStrategies DEFAULT_AUTO_RESET_OFFSET = KafkaResetOffsetStrategies.LARGEST;
public static final long defaultMaxEventsFromKafkaSource = 5000000;
public static long DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE = defaultMaxEventsFromKafkaSource;
public static final long DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE = 5000000;
public static long maxEventsFromKafkaSource = DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE;
}
private final HashMap<String, String> kafkaParams;
@@ -234,9 +234,9 @@ public class KafkaOffsetGen {
// Come up with final set of OffsetRanges to read (account for new partitions, limit number of events)
long maxEventsToReadFromKafka = props.getLong(Config.MAX_EVENTS_FROM_KAFKA_SOURCE_PROP,
Config.DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE);
Config.maxEventsFromKafkaSource);
maxEventsToReadFromKafka = (maxEventsToReadFromKafka == Long.MAX_VALUE || maxEventsToReadFromKafka == Integer.MAX_VALUE)
? Config.DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE : maxEventsToReadFromKafka;
? Config.maxEventsFromKafkaSource : maxEventsToReadFromKafka;
long numEvents = sourceLimit == Long.MAX_VALUE ? maxEventsToReadFromKafka : sourceLimit;
OffsetRange[] offsetRanges = CheckpointUtils.computeOffsetRanges(fromOffsets, toOffsets, numEvents);

6
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java
View File

@@ -90,7 +90,7 @@ public class TestKafkaSource extends UtilitiesTestBase {
props.setProperty("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.setProperty("hoodie.deltastreamer.kafka.source.maxEvents",
maxEventsToReadFromKafkaSource != null ? String.valueOf(maxEventsToReadFromKafkaSource) :
String.valueOf(Config.DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE));
String.valueOf(Config.maxEventsFromKafkaSource));
return props;
}
@@ -151,7 +151,7 @@ public class TestKafkaSource extends UtilitiesTestBase {
Source jsonSource = new JsonKafkaSource(props, jsc, sparkSession, schemaProvider);
SourceFormatAdapter kafkaSource = new SourceFormatAdapter(jsonSource);
Config.DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE = 500;
Config.maxEventsFromKafkaSource = 500;
/*
1. Extract without any checkpoint => get all the data, respecting default upper cap since both sourceLimit and
@@ -168,7 +168,7 @@ public class TestKafkaSource extends UtilitiesTestBase {
assertEquals(1500, fetch2.getBatch().get().count());
//reset the value back since it is a static variable
Config.DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE = Config.defaultMaxEventsFromKafkaSource;
Config.maxEventsFromKafkaSource = Config.DEFAULT_MAX_EVENTS_FROM_KAFKA_SOURCE;
}
@Test