[HUDI-346] Set allowMultipleEmptyLines to false for EmptyLineSeparator rule (#1025)
This commit is contained in:
@@ -35,7 +35,6 @@ import org.apache.log4j.Logger;
|
||||
import org.apache.spark.api.java.JavaRDD;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
|
||||
|
||||
public class HoodieCompactor {
|
||||
|
||||
private static volatile Logger logger = LogManager.getLogger(HoodieCompactor.class);
|
||||
|
||||
@@ -75,7 +75,6 @@ import org.apache.spark.sql.Row;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
import scala.collection.JavaConversions;
|
||||
|
||||
|
||||
/**
|
||||
* Sync's one batch of data to hoodie dataset
|
||||
*/
|
||||
@@ -155,7 +154,6 @@ public class DeltaSync implements Serializable {
|
||||
*/
|
||||
private final HoodieTableType tableType;
|
||||
|
||||
|
||||
public DeltaSync(HoodieDeltaStreamer.Config cfg, SparkSession sparkSession, SchemaProvider schemaProvider,
|
||||
HoodieTableType tableType, TypedProperties props, JavaSparkContext jssc, FileSystem fs, HiveConf hiveConf,
|
||||
Function<HoodieWriteClient, Boolean> onInitializingHoodieWriteClient) throws IOException {
|
||||
|
||||
@@ -65,7 +65,6 @@ import org.apache.log4j.Logger;
|
||||
import org.apache.spark.api.java.JavaSparkContext;
|
||||
import org.apache.spark.sql.SparkSession;
|
||||
|
||||
|
||||
/**
|
||||
* An Utility which can incrementally take the output from {@link HiveIncrementalPuller} and apply it to the target
|
||||
* dataset. Does not maintain any state, queries at runtime to see how far behind the target dataset is from the source
|
||||
@@ -267,11 +266,9 @@ public class HoodieDeltaStreamer implements Serializable {
|
||||
@Parameter(names = {"--checkpoint"}, description = "Resume Delta Streamer from this checkpoint.")
|
||||
public String checkpoint = null;
|
||||
|
||||
|
||||
@Parameter(names = {"--help", "-h"}, help = true)
|
||||
public Boolean help = false;
|
||||
|
||||
|
||||
public boolean isAsyncCompactionEnabled() {
|
||||
return continuousMode && !forceDisableCompaction
|
||||
&& HoodieTableType.MERGE_ON_READ.equals(HoodieTableType.valueOf(storageType));
|
||||
|
||||
@@ -57,7 +57,6 @@ public class SchedulerConfGenerator {
|
||||
compactionMinShare.toString());
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Helper to set Spark Scheduling Configs dynamically
|
||||
*
|
||||
|
||||
@@ -44,7 +44,6 @@ public final class SourceFormatAdapter {
|
||||
|
||||
private final Source source;
|
||||
|
||||
|
||||
public SourceFormatAdapter(Source source) {
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
@@ -48,7 +48,6 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
|
||||
|
||||
private final String outputDateFormat;
|
||||
|
||||
|
||||
/**
|
||||
* Supported configs
|
||||
*/
|
||||
|
||||
@@ -209,7 +209,6 @@ public class TimelineServerPerf implements Serializable {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private static class PerfStats implements Serializable {
|
||||
|
||||
private final String partition;
|
||||
|
||||
@@ -63,7 +63,6 @@ public class HiveIncrPullSource extends AvroSource {
|
||||
|
||||
private final String incrPullRootPath;
|
||||
|
||||
|
||||
/**
|
||||
* Configs supported
|
||||
*/
|
||||
|
||||
@@ -50,7 +50,6 @@ public class AvroConvertor implements Serializable {
|
||||
*/
|
||||
private transient Injection<GenericRecord, byte[]> recordInjection;
|
||||
|
||||
|
||||
public AvroConvertor(String schemaStr) {
|
||||
this.schemaStr = schemaStr;
|
||||
}
|
||||
@@ -79,7 +78,6 @@ public class AvroConvertor implements Serializable {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public GenericRecord fromJson(String json) throws IOException {
|
||||
initSchema();
|
||||
initJsonConvertor();
|
||||
@@ -90,7 +88,6 @@ public class AvroConvertor implements Serializable {
|
||||
return new Schema.Parser().parse(schemaStr);
|
||||
}
|
||||
|
||||
|
||||
public GenericRecord fromAvroBinary(byte[] avroBinary) {
|
||||
initSchema();
|
||||
initInjection();
|
||||
|
||||
@@ -43,7 +43,6 @@ import scala.collection.mutable.ArrayBuffer;
|
||||
import scala.collection.mutable.StringBuilder;
|
||||
import scala.util.Either;
|
||||
|
||||
|
||||
/**
|
||||
* Source to read data from Kafka, incrementally
|
||||
*/
|
||||
@@ -250,7 +249,6 @@ public class KafkaOffsetGen {
|
||||
return checkpointOffsetReseter ? earliestOffsets : checkpointOffsets;
|
||||
}
|
||||
|
||||
|
||||
public String getTopicName() {
|
||||
return topicName;
|
||||
}
|
||||
|
||||
@@ -61,7 +61,6 @@ public class TestHDFSParquetImporter implements Serializable {
|
||||
private static MiniDFSCluster dfsCluster;
|
||||
private static DistributedFileSystem dfs;
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void initClass() throws Exception {
|
||||
hdfsTestService = new HdfsTestService();
|
||||
|
||||
@@ -78,7 +78,6 @@ public class TestKafkaSource extends UtilitiesTestBase {
|
||||
testUtils.teardown();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testJsonKafkaSource() throws IOException {
|
||||
|
||||
@@ -132,7 +131,6 @@ public class TestKafkaSource extends UtilitiesTestBase {
|
||||
assertEquals(Option.empty(), fetch4AsRows.getBatch());
|
||||
}
|
||||
|
||||
|
||||
private static HashMap<TopicAndPartition, LeaderOffset> makeOffsetMap(int[] partitions, long[] offsets) {
|
||||
HashMap<TopicAndPartition, LeaderOffset> map = new HashMap<>();
|
||||
for (int i = 0; i < partitions.length; i++) {
|
||||
|
||||
Reference in New Issue
Block a user