1
0

[HUDI-346] Set allowMultipleEmptyLines to false for EmptyLineSeparator rule (#1025)

This commit is contained in:
谢磊
2019-11-19 18:44:42 +08:00
committed by leesf
parent 66492498f7
commit 804e348d0e
82 changed files with 5 additions and 108 deletions

View File

@@ -35,7 +35,6 @@ import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
public class HoodieCompactor {
private static volatile Logger logger = LogManager.getLogger(HoodieCompactor.class);

View File

@@ -75,7 +75,6 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import scala.collection.JavaConversions;
/**
* Sync's one batch of data to hoodie dataset
*/
@@ -155,7 +154,6 @@ public class DeltaSync implements Serializable {
*/
private final HoodieTableType tableType;
public DeltaSync(HoodieDeltaStreamer.Config cfg, SparkSession sparkSession, SchemaProvider schemaProvider,
HoodieTableType tableType, TypedProperties props, JavaSparkContext jssc, FileSystem fs, HiveConf hiveConf,
Function<HoodieWriteClient, Boolean> onInitializingHoodieWriteClient) throws IOException {

View File

@@ -65,7 +65,6 @@ import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
/**
* An Utility which can incrementally take the output from {@link HiveIncrementalPuller} and apply it to the target
* dataset. Does not maintain any state, queries at runtime to see how far behind the target dataset is from the source
@@ -267,11 +266,9 @@ public class HoodieDeltaStreamer implements Serializable {
@Parameter(names = {"--checkpoint"}, description = "Resume Delta Streamer from this checkpoint.")
public String checkpoint = null;
@Parameter(names = {"--help", "-h"}, help = true)
public Boolean help = false;
public boolean isAsyncCompactionEnabled() {
return continuousMode && !forceDisableCompaction
&& HoodieTableType.MERGE_ON_READ.equals(HoodieTableType.valueOf(storageType));

View File

@@ -57,7 +57,6 @@ public class SchedulerConfGenerator {
compactionMinShare.toString());
}
/**
* Helper to set Spark Scheduling Configs dynamically
*

View File

@@ -44,7 +44,6 @@ public final class SourceFormatAdapter {
private final Source source;
public SourceFormatAdapter(Source source) {
this.source = source;
}

View File

@@ -48,7 +48,6 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
private final String outputDateFormat;
/**
* Supported configs
*/

View File

@@ -209,7 +209,6 @@ public class TimelineServerPerf implements Serializable {
}
}
private static class PerfStats implements Serializable {
private final String partition;

View File

@@ -63,7 +63,6 @@ public class HiveIncrPullSource extends AvroSource {
private final String incrPullRootPath;
/**
* Configs supported
*/

View File

@@ -50,7 +50,6 @@ public class AvroConvertor implements Serializable {
*/
private transient Injection<GenericRecord, byte[]> recordInjection;
public AvroConvertor(String schemaStr) {
this.schemaStr = schemaStr;
}
@@ -79,7 +78,6 @@ public class AvroConvertor implements Serializable {
}
}
public GenericRecord fromJson(String json) throws IOException {
initSchema();
initJsonConvertor();
@@ -90,7 +88,6 @@ public class AvroConvertor implements Serializable {
return new Schema.Parser().parse(schemaStr);
}
public GenericRecord fromAvroBinary(byte[] avroBinary) {
initSchema();
initInjection();

View File

@@ -43,7 +43,6 @@ import scala.collection.mutable.ArrayBuffer;
import scala.collection.mutable.StringBuilder;
import scala.util.Either;
/**
* Source to read data from Kafka, incrementally
*/
@@ -250,7 +249,6 @@ public class KafkaOffsetGen {
return checkpointOffsetReseter ? earliestOffsets : checkpointOffsets;
}
public String getTopicName() {
return topicName;
}

View File

@@ -61,7 +61,6 @@ public class TestHDFSParquetImporter implements Serializable {
private static MiniDFSCluster dfsCluster;
private static DistributedFileSystem dfs;
@BeforeClass
public static void initClass() throws Exception {
hdfsTestService = new HdfsTestService();

View File

@@ -78,7 +78,6 @@ public class TestKafkaSource extends UtilitiesTestBase {
testUtils.teardown();
}
@Test
public void testJsonKafkaSource() throws IOException {
@@ -132,7 +131,6 @@ public class TestKafkaSource extends UtilitiesTestBase {
assertEquals(Option.empty(), fetch4AsRows.getBatch());
}
private static HashMap<TopicAndPartition, LeaderOffset> makeOffsetMap(int[] partitions, long[] offsets) {
HashMap<TopicAndPartition, LeaderOffset> map = new HashMap<>();
for (int i = 0; i < partitions.length; i++) {