[HUDI-346] Set allowMultipleEmptyLines to false for EmptyLineSeparator rule (#1025)
This commit is contained in:
谢磊
@@ -39,7 +39,6 @@ public class HoodieAvroWriteSupport extends AvroWriteSupport {
|
||||
public static final String HOODIE_MIN_RECORD_KEY_FOOTER = "hoodie_min_record_key";
|
||||
public static final String HOODIE_MAX_RECORD_KEY_FOOTER = "hoodie_max_record_key";
|
||||
|
||||
|
||||
public HoodieAvroWriteSupport(MessageType schema, Schema avroSchema, BloomFilter bloomFilter) {
|
||||
super(schema, avroSchema);
|
||||
this.bloomFilter = bloomFilter;
|
||||
|
||||
@@ -80,7 +80,6 @@ public class HoodieJsonPayload implements HoodieRecordPayload<HoodieJsonPayload>
|
||||
return baos.toByteArray();
|
||||
}
|
||||
|
||||
|
||||
private String unCompressData(byte[] data) throws IOException {
|
||||
InflaterInputStream iis = new InflaterInputStream(new ByteArrayInputStream(data));
|
||||
try {
|
||||
|
||||
@@ -51,7 +51,6 @@ public class HoodiePartitionMetadata {
|
||||
|
||||
private static Logger log = LogManager.getLogger(HoodiePartitionMetadata.class);
|
||||
|
||||
|
||||
/**
|
||||
* Construct metadata from existing partition
|
||||
*/
|
||||
|
||||
@@ -98,7 +98,6 @@ public class HoodieRecord<T extends HoodieRecordPayload> implements Serializable
|
||||
this.data = null;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Sets the current currentLocation of the record. This should happen exactly-once
|
||||
*/
|
||||
|
||||
@@ -114,7 +114,6 @@ public class HoodieTableConfig implements Serializable {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Read the table type from the table properties and if not found, return the default
|
||||
*/
|
||||
|
||||
@@ -267,7 +267,6 @@ public class HoodieTableMetaClient implements Serializable {
|
||||
return archivedTimeline;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Helper method to initialize a dataset, with given basePath, tableType, name, archiveFolder
|
||||
*/
|
||||
@@ -410,7 +409,6 @@ public class HoodieTableMetaClient implements Serializable {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Helper method to scan all hoodie-instant metafiles and construct HoodieInstant objects
|
||||
*
|
||||
|
||||
@@ -294,7 +294,6 @@ class HoodieLogFileReader implements HoodieLogFormat.Reader {
|
||||
return new HoodieLogFormatVersion(inputStream.readInt());
|
||||
}
|
||||
|
||||
|
||||
private boolean readMagic() throws IOException {
|
||||
try {
|
||||
boolean hasMagic = hasNextMagic();
|
||||
|
||||
@@ -97,7 +97,6 @@ public interface HoodieLogFormat {
|
||||
public HoodieLogBlock prev() throws IOException;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Builder class to construct the default log format writer
|
||||
*/
|
||||
|
||||
@@ -45,7 +45,6 @@ public class HoodieDeleteBlock extends HoodieLogBlock {
|
||||
this.keysToDelete = keysToDelete;
|
||||
}
|
||||
|
||||
|
||||
private HoodieDeleteBlock(Option<byte[]> content, FSDataInputStream inputStream, boolean readBlockLazily,
|
||||
Option<HoodieLogBlockContentLocation> blockContentLocation, Map<HeaderMetadataType, String> header,
|
||||
Map<HeaderMetadataType, String> footer) {
|
||||
|
||||
@@ -171,7 +171,6 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
|
||||
(Function<HoodieInstant, Option<byte[]>> & Serializable) this::getInstantDetails);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Get only the cleaner action (inflight and completed) in the active timeline
|
||||
*/
|
||||
@@ -364,7 +363,6 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
|
||||
return inflight;
|
||||
}
|
||||
|
||||
|
||||
private void transitionState(HoodieInstant fromInstant, HoodieInstant toInstant, Option<byte[]> data) {
|
||||
Preconditions.checkArgument(fromInstant.getTimestamp().equals(toInstant.getTimestamp()));
|
||||
Path commitFilePath = new Path(metaClient.getMetaPath(), toInstant.getFileName());
|
||||
|
||||
@@ -93,7 +93,6 @@ public class HoodieArchivedTimeline extends HoodieDefaultTimeline {
|
||||
in.defaultReadObject();
|
||||
}
|
||||
|
||||
|
||||
public static Path getArchiveLogPath(String archiveFolder) {
|
||||
return new Path(archiveFolder, HOODIE_COMMIT_ARCHIVE_LOG_FILE);
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.apache.hudi.common.table.timeline.HoodieInstant;
|
||||
|
||||
|
||||
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||
public class InstantDTO {
|
||||
|
||||
|
||||
@@ -655,7 +655,6 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
|
||||
.map(Option::get);
|
||||
}
|
||||
|
||||
|
||||
protected Option<HoodieDataFile> getLatestDataFile(HoodieFileGroup fileGroup) {
|
||||
return Option
|
||||
.fromJavaOptional(fileGroup.getAllDataFiles().filter(df -> !isDataFileDueToPendingCompaction(df)).findFirst());
|
||||
|
||||
@@ -130,7 +130,6 @@ public class FileSystemViewManager {
|
||||
return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Create an in-memory file System view for a dataset
|
||||
*
|
||||
|
||||
@@ -111,7 +111,6 @@ public class AvroUtils {
|
||||
return serializeAvroMetadata(compactionWorkload, HoodieCompactionPlan.class);
|
||||
}
|
||||
|
||||
|
||||
public static Option<byte[]> serializeCleanerPlan(HoodieCleanerPlan cleanPlan) throws IOException {
|
||||
return serializeAvroMetadata(cleanPlan, HoodieCleanerPlan.class);
|
||||
}
|
||||
|
||||
@@ -63,7 +63,6 @@ public interface ConsistencyGuard {
|
||||
*/
|
||||
void waitTillAllFilesDisappear(String dirPath, List<String> files) throws IOException, TimeoutException;
|
||||
|
||||
|
||||
/**
|
||||
* Wait Till target visibility is reached
|
||||
*
|
||||
|
||||
@@ -109,7 +109,6 @@ public class FSUtils {
|
||||
return String.format("%d-%d-%d", taskPartitionId, stageId, taskAttemptId);
|
||||
}
|
||||
|
||||
|
||||
public static String makeDataFileName(String commitTime, String writeToken, String fileId) {
|
||||
return String.format("%s_%s_%s.parquet", fileId, writeToken, commitTime);
|
||||
}
|
||||
@@ -150,7 +149,6 @@ public class FSUtils {
|
||||
return fullFileName.split("_")[0];
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Gets all partition paths assuming date partitioning (year, month, day) three levels down.
|
||||
*/
|
||||
|
||||
@@ -183,7 +183,6 @@ public class HoodieAvroUtils {
|
||||
return record;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Given a avro record with a given schema, rewrites it into the new schema while setting fields only from the old
|
||||
* schema
|
||||
|
||||
@@ -32,7 +32,6 @@ import java.lang.reflect.InvocationTargetException;
|
||||
import org.apache.hudi.exception.HoodieSerializationException;
|
||||
import org.objenesis.instantiator.ObjectInstantiator;
|
||||
|
||||
|
||||
/**
|
||||
* {@link SerializationUtils} class internally uses {@link Kryo} serializer for serializing / deserializing objects.
|
||||
*/
|
||||
|
||||
@@ -29,7 +29,6 @@ import org.apache.hudi.common.util.collection.Pair;
|
||||
import org.apache.log4j.LogManager;
|
||||
import org.apache.log4j.Logger;
|
||||
|
||||
|
||||
public class TimelineDiffHelper {
|
||||
|
||||
protected static Logger log = LogManager.getLogger(TimelineDiffHelper.class);
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
|
||||
package org.apache.hudi.common.util.queue;
|
||||
|
||||
import java.util.Arrays;
|
||||
@@ -143,7 +142,6 @@ public class BoundedInMemoryExecutor<I, O, E> {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public boolean isRemaining() {
|
||||
return queue.iterator().hasNext();
|
||||
}
|
||||
|
||||
@@ -20,7 +20,6 @@ package org.apache.hudi.common.util.queue;
|
||||
|
||||
import java.util.Iterator;
|
||||
|
||||
|
||||
/**
|
||||
* Consume entries from queue and execute callback function
|
||||
*/
|
||||
@@ -59,5 +58,4 @@ public abstract class BoundedInMemoryQueueConsumer<I, O> {
|
||||
*/
|
||||
protected abstract O getResult();
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -123,5 +123,4 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
|
||||
assertArrayEquals(new Text("data3").getBytes(), archivedTimeline.getInstantDetails(instant3).get());
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -471,7 +471,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAppendAndReadOnCorruptedLog() throws IOException, URISyntaxException, InterruptedException {
|
||||
Writer writer =
|
||||
@@ -556,7 +555,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
|
||||
reader.close();
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testAvroLogRecordReaderBasic() throws IOException, URISyntaxException, InterruptedException {
|
||||
Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema());
|
||||
|
||||
@@ -287,8 +287,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
|
||||
* HELPER METHODS
|
||||
*********************************************************************************************************
|
||||
*/
|
||||
|
||||
|
||||
/**
|
||||
* Helper to run one or more rounds of cleaning, incrementally syncing the view and then validate
|
||||
*/
|
||||
|
||||
@@ -41,7 +41,6 @@ public class TestDFSPropertiesConfiguration {
|
||||
private static MiniDFSCluster dfsCluster;
|
||||
private static DistributedFileSystem dfs;
|
||||
|
||||
|
||||
@BeforeClass
|
||||
public static void initClass() throws Exception {
|
||||
hdfsTestService = new HdfsTestService();
|
||||
|
||||
@@ -24,7 +24,6 @@ import org.codehaus.jackson.JsonNode;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
|
||||
|
||||
public class TestHoodieAvroUtils {
|
||||
|
||||
private static String EXAMPLE_SCHEMA = "{\"type\": \"record\"," + "\"name\": \"testrec\"," + "\"fields\": [ "
|
||||
|
||||
Reference in New Issue
Block a user