lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-346] Set allowMultipleEmptyLines to false for EmptyLineSeparator rule (#1025)

Browse Source
This commit is contained in:
谢磊
2019-11-19 18:44:42 +08:00
committed by leesf
parent 66492498f7
commit 804e348d0e
82 changed files with 5 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hudi-cli/src/main/java/org/apache/hudi/cli/HoodieCLI.java
View File

@@ -35,7 +35,6 @@ public class HoodieCLI {
public static HoodieTableMetaClient tableMetadata;
public static HoodieTableMetaClient syncTableMetadata;
public enum CLIState {
INIT, DATASET, SYNC
}

1
hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
View File

@@ -70,7 +70,6 @@ public class RepairsCommand implements CommandMarker {
return "Deduplication failed ";
}
@CliCommand(value = "repair addpartitionmeta", help = "Add partition metadata to a dataset, if not present")
public String addPartitionMeta(
@CliOption(key = {"dryrun"}, help = "Should we actually add or just print what would be done",

3
hudi-cli/src/main/java/org/apache/hudi/cli/commands/SavepointsCommand.java
View File

@@ -53,7 +53,6 @@ public class SavepointsCommand implements CommandMarker {
return HoodieCLI.tableMetadata != null;
}
@CliAvailabilityIndicator({"savepoint create"})
public boolean isCreateSavepointAvailable() {
return HoodieCLI.tableMetadata != null;
@@ -127,7 +126,6 @@ public class SavepointsCommand implements CommandMarker {
return "Savepoint " + commitTime + " rolled back";
}
@CliCommand(value = "savepoints refresh", help = "Refresh the savepoints")
public String refreshMetaClient() throws IOException {
HoodieCLI.refreshTableMetadata();
@@ -140,5 +138,4 @@ public class SavepointsCommand implements CommandMarker {
return new HoodieWriteClient(jsc, config, false);
}
}

1
hudi-cli/src/main/java/org/apache/hudi/cli/utils/InputStreamConsumer.java
View File

@@ -57,5 +57,4 @@ public class InputStreamConsumer extends Thread {
stdout.start();
}
}

1
hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
View File

@@ -132,7 +132,6 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig {
return this;
}
public Builder withAutoClean(Boolean autoClean) {
props.setProperty(AUTO_CLEAN_PROP, String.valueOf(autoClean));
return this;

1
hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
View File

@@ -74,7 +74,6 @@ public class HoodieMetricsConfig extends DefaultHoodieConfig {
return this;
}
public Builder on(boolean metricsOn) {
props.setProperty(METRICS_ON, String.valueOf(metricsOn));
return this;

1
hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -147,7 +147,6 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return Integer.parseInt(props.getProperty(ROLLBACK_PARALLELISM));
}
public int getWriteBufferLimitBytes() {
return Integer.parseInt(props.getProperty(WRITE_BUFFER_LIMIT_BYTES, DEFAULT_WRITE_BUFFER_LIMIT_BYTES));
}

1
hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java
View File

@@ -18,7 +18,6 @@
package org.apache.hudi.exception;
/**
* <p>
* Exception thrown when dependent system is not available

1
hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java
View File

@@ -27,7 +27,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.spark.api.java.function.Function2;
/**
* Map function that handles a sorted stream of HoodieRecords
*/

1
hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java
View File

@@ -52,7 +52,6 @@ public abstract class LazyIterableIterator<I, O> implements Iterable<O>, Iterato
*/
protected abstract O computeNext();
/**
* Called once, after all elements are processed.
*/

2
hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java
View File

@@ -47,7 +47,6 @@ public abstract class HoodieIndex<T extends HoodieRecordPayload> implements Seri
this.config = config;
}
public static <T extends HoodieRecordPayload> HoodieIndex<T> createIndex(HoodieWriteConfig config,
JavaSparkContext jsc) throws HoodieIndexException {
switch (config.getIndexType()) {
@@ -108,7 +107,6 @@ public abstract class HoodieIndex<T extends HoodieRecordPayload> implements Seri
*/
public abstract boolean canIndexLogFiles();
/**
* An index is "implicit" with respect to storage, if just writing new data to a file slice, updates the index as
* well. This is used by storage, to save memory footprint in certain cases.

1
hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java
View File

@@ -38,7 +38,6 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
/**
* Hoodie Index implementation backed by an in-memory Hash map.
* <p>

1
hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
View File

@@ -268,7 +268,6 @@ public class HoodieBloomIndex<T extends HoodieRecordPayload> extends HoodieIndex
}
}
@Override
public boolean rollbackCommit(String commitTime) {
// Nope, don't need to do anything.

1
hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java
View File

@@ -170,7 +170,6 @@ public class HoodieCleanHelper<T extends HoodieRecordPayload<T>> implements Seri
return deletePaths;
}
/**
* Selects the versions for file for cleaning, such that it
* <p>

1
hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java
View File

@@ -23,7 +23,6 @@ import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
public abstract class HoodieIOHandle<T extends HoodieRecordPayload> {
protected final String instantTime;

1
hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
View File

@@ -86,7 +86,6 @@ public class HoodieMergeHandle<T extends HoodieRecordPayload> extends HoodieWrit
dataFileToBeMerged);
}
public static Schema createHoodieWriteSchema(Schema originalSchema) {
return HoodieAvroUtils.addMetadataFields(originalSchema);
}

1
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java
View File

@@ -16,7 +16,6 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
import com.google.common.annotations.VisibleForTesting;

1
hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java
View File

@@ -51,7 +51,6 @@ public class HoodieParquetWriter<T extends HoodieRecordPayload, R extends Indexe
private final String commitTime;
private final Schema schema;
public HoodieParquetWriter(String commitTime, Path file, HoodieParquetConfig parquetConfig, Schema schema)
throws IOException {
super(HoodieWrapperFileSystem.convertToHoodiePath(file, parquetConfig.getHadoopConf()),

1
hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
View File

@@ -80,7 +80,6 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
/**
* Implementation of a very heavily read-optimized Hoodie Table where
* <p>

1
hudi-client/src/main/java/org/apache/hudi/table/RollbackExecutor.java
View File

@@ -212,7 +212,6 @@ public class RollbackExecutor implements Serializable {
return results;
}
private Map<HeaderMetadataType, String> generateHeader(String commit) {
// generate metadata
Map<HeaderMetadataType, String> header = Maps.newHashMap();

1
hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java
View File

@@ -49,7 +49,6 @@ public class WorkloadProfile<T extends HoodieRecordPayload> implements Serializa
private final WorkloadStat globalStat;
public WorkloadProfile(JavaRDD<HoodieRecord<T>> taggedRecords) {
this.taggedRecords = taggedRecords;
this.partitionPathStatMap = new HashMap<>();

1
hudi-client/src/test/java/HoodieClientExample.java
View File

@@ -67,7 +67,6 @@ public class HoodieClientExample {
cli.run();
}
public void run() throws Exception {
SparkConf sparkConf = new SparkConf().setAppName("hoodie-client-example");

1
hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java
View File

@@ -70,7 +70,6 @@ public class TestConsistencyGuard extends HoodieClientTestHarness {
.asList(basePath + "/partition/path/f1_1-0-2_000.parquet", basePath + "/partition/path/f2_1-0-2_000.parquet"));
}
@Test(expected = TimeoutException.class)
public void testCheckFailingAppears() throws Exception {
HoodieClientTestUtils.fakeDataFile(basePath, "partition/path", "000", "f1");

1
hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java
View File

@@ -92,7 +92,6 @@ public class HoodieClientTestUtils {
new File(parentPath + "/" + commitTime + suffix).createNewFile();
}
public static void fakeCommitFile(String basePath, String commitTime) throws IOException {
fakeMetaFile(basePath, commitTime, HoodieTimeline.COMMIT_EXTENSION);
}

2
hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java
View File

@@ -79,7 +79,6 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa
return partitionPath;
}
@Override
public TestRawTripPayload preCombine(TestRawTripPayload another) {
return another;
@@ -129,7 +128,6 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa
return baos.toByteArray();
}
private String unCompressData(byte[] data) throws IOException {
try (InflaterInputStream iis = new InflaterInputStream(new ByteArrayInputStream(data))) {
return FileIOUtils.readAsUTFString(iis, dataSize);

2
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
View File

@@ -291,7 +291,6 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
}
}
@Test
public void testTagLocation() throws Exception {
// We have some records to be tagged (two different partitions)
@@ -433,7 +432,6 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
}
}
@Test
public void testBloomFilterFalseError() throws IOException, InterruptedException {
// We have two hoodie records

1
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java
View File

@@ -196,7 +196,6 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness {
assertEquals(new HashSet<>(Arrays.asList("f4", "f1")), new HashSet<>(recordKeyToFileComps.get("005")));
}
@Test
public void testTagLocation() throws Exception {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();

4
hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java
View File

@@ -250,7 +250,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
assertEquals(4, writeStatus.getStat().getNumWrites());// 3 rewritten records + 1 new record
}
private List<HoodieRecord> newHoodieRecords(int n, String time) throws Exception {
List<HoodieRecord> records = new ArrayList<>();
for (int i = 0; i < n; i++) {
@@ -387,7 +386,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
assertEquals("If the number of records are more than 1150, then there should be a new file", 3, counts);
}
private UpsertPartitioner getUpsertPartitioner(int smallFileSize, int numInserts, int numUpdates, int fileSize,
String testPartitionPath, boolean autoSplitInserts) throws Exception {
HoodieWriteConfig config = makeHoodieClientConfigBuilder()
@@ -419,7 +417,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
return partitioner;
}
@Test
public void testUpsertPartitioner() throws Exception {
final String testPartitionPath = "2016/09/26";
@@ -429,7 +426,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
assertEquals("Total of 2 insert buckets", 2, insertBuckets.size());
}
@Test
public void testUpsertPartitionerWithSmallInsertHandling() throws Exception {
final String testPartitionPath = "2016/09/26";

1
hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroWriteSupport.java
View File

@@ -39,7 +39,6 @@ public class HoodieAvroWriteSupport extends AvroWriteSupport {
public static final String HOODIE_MIN_RECORD_KEY_FOOTER = "hoodie_min_record_key";
public static final String HOODIE_MAX_RECORD_KEY_FOOTER = "hoodie_max_record_key";
public HoodieAvroWriteSupport(MessageType schema, Schema avroSchema, BloomFilter bloomFilter) {
super(schema, avroSchema);
this.bloomFilter = bloomFilter;

1
hudi-common/src/main/java/org/apache/hudi/common/HoodieJsonPayload.java
View File

@@ -80,7 +80,6 @@ public class HoodieJsonPayload implements HoodieRecordPayload<HoodieJsonPayload>
return baos.toByteArray();
}
private String unCompressData(byte[] data) throws IOException {
InflaterInputStream iis = new InflaterInputStream(new ByteArrayInputStream(data));
try {

1
hudi-common/src/main/java/org/apache/hudi/common/model/HoodiePartitionMetadata.java
View File

@@ -51,7 +51,6 @@ public class HoodiePartitionMetadata {
private static Logger log = LogManager.getLogger(HoodiePartitionMetadata.class);
/**
* Construct metadata from existing partition
*/

1
hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecord.java
View File

@@ -98,7 +98,6 @@ public class HoodieRecord<T extends HoodieRecordPayload> implements Serializable
this.data = null;
}
/**
* Sets the current currentLocation of the record. This should happen exactly-once
*/

1
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
View File

@@ -114,7 +114,6 @@ public class HoodieTableConfig implements Serializable {
}
}
/**
* Read the table type from the table properties and if not found, return the default
*/

2
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
View File

@@ -267,7 +267,6 @@ public class HoodieTableMetaClient implements Serializable {
return archivedTimeline;
}
/**
* Helper method to initialize a dataset, with given basePath, tableType, name, archiveFolder
*/
@@ -410,7 +409,6 @@ public class HoodieTableMetaClient implements Serializable {
}
}
/**
* Helper method to scan all hoodie-instant metafiles and construct HoodieInstant objects
*

1
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
View File

@@ -294,7 +294,6 @@ class HoodieLogFileReader implements HoodieLogFormat.Reader {
return new HoodieLogFormatVersion(inputStream.readInt());
}
private boolean readMagic() throws IOException {
try {
boolean hasMagic = hasNextMagic();

1
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormat.java
View File

@@ -97,7 +97,6 @@ public interface HoodieLogFormat {
public HoodieLogBlock prev() throws IOException;
}
/**
* Builder class to construct the default log format writer
*/

1
hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDeleteBlock.java
View File

@@ -45,7 +45,6 @@ public class HoodieDeleteBlock extends HoodieLogBlock {
this.keysToDelete = keysToDelete;
}
private HoodieDeleteBlock(Option<byte[]> content, FSDataInputStream inputStream, boolean readBlockLazily,
Option<HoodieLogBlockContentLocation> blockContentLocation, Map<HeaderMetadataType, String> header,
Map<HeaderMetadataType, String> footer) {

2
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieActiveTimeline.java
View File

@@ -171,7 +171,6 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
(Function<HoodieInstant, Option<byte[]>> & Serializable) this::getInstantDetails);
}
/**
* Get only the cleaner action (inflight and completed) in the active timeline
*/
@@ -364,7 +363,6 @@ public class HoodieActiveTimeline extends HoodieDefaultTimeline {
return inflight;
}
private void transitionState(HoodieInstant fromInstant, HoodieInstant toInstant, Option<byte[]> data) {
Preconditions.checkArgument(fromInstant.getTimestamp().equals(toInstant.getTimestamp()));
Path commitFilePath = new Path(metaClient.getMetaPath(), toInstant.getFileName());

1
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
View File

@@ -93,7 +93,6 @@ public class HoodieArchivedTimeline extends HoodieDefaultTimeline {
in.defaultReadObject();
}
public static Path getArchiveLogPath(String archiveFolder) {
return new Path(archiveFolder, HOODIE_COMMIT_ARCHIVE_LOG_FILE);
}

1
hudi-common/src/main/java/org/apache/hudi/common/table/timeline/dto/InstantDTO.java
View File

@@ -22,7 +22,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.hudi.common.table.timeline.HoodieInstant;
@JsonIgnoreProperties(ignoreUnknown = true)
public class InstantDTO {

1
hudi-common/src/main/java/org/apache/hudi/common/table/view/AbstractTableFileSystemView.java
View File

@@ -655,7 +655,6 @@ public abstract class AbstractTableFileSystemView implements SyncableFileSystemV
.map(Option::get);
}
protected Option<HoodieDataFile> getLatestDataFile(HoodieFileGroup fileGroup) {
return Option
.fromJavaOptional(fileGroup.getAllDataFiles().filter(df -> !isDataFileDueToPendingCompaction(df)).findFirst());

1
hudi-common/src/main/java/org/apache/hudi/common/table/view/FileSystemViewManager.java
View File

@@ -130,7 +130,6 @@ public class FileSystemViewManager {
return new SpillableMapBasedFileSystemView(metaClient, timeline, viewConf);
}
/**
* Create an in-memory file System view for a dataset
*

1
hudi-common/src/main/java/org/apache/hudi/common/util/AvroUtils.java
View File

@@ -111,7 +111,6 @@ public class AvroUtils {
return serializeAvroMetadata(compactionWorkload, HoodieCompactionPlan.class);
}
public static Option<byte[]> serializeCleanerPlan(HoodieCleanerPlan cleanPlan) throws IOException {
return serializeAvroMetadata(cleanPlan, HoodieCleanerPlan.class);
}

1
hudi-common/src/main/java/org/apache/hudi/common/util/ConsistencyGuard.java
View File

@@ -63,7 +63,6 @@ public interface ConsistencyGuard {
*/
void waitTillAllFilesDisappear(String dirPath, List<String> files) throws IOException, TimeoutException;
/**
* Wait Till target visibility is reached
*

2
hudi-common/src/main/java/org/apache/hudi/common/util/FSUtils.java
View File

@@ -109,7 +109,6 @@ public class FSUtils {
return String.format("%d-%d-%d", taskPartitionId, stageId, taskAttemptId);
}
public static String makeDataFileName(String commitTime, String writeToken, String fileId) {
return String.format("%s_%s_%s.parquet", fileId, writeToken, commitTime);
}
@@ -150,7 +149,6 @@ public class FSUtils {
return fullFileName.split("_")[0];
}
/**
* Gets all partition paths assuming date partitioning (year, month, day) three levels down.
*/

1
hudi-common/src/main/java/org/apache/hudi/common/util/HoodieAvroUtils.java
View File

@@ -183,7 +183,6 @@ public class HoodieAvroUtils {
return record;
}
/**
* Given a avro record with a given schema, rewrites it into the new schema while setting fields only from the old
* schema

1
hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java
View File

@@ -32,7 +32,6 @@ import java.lang.reflect.InvocationTargetException;
import org.apache.hudi.exception.HoodieSerializationException;
import org.objenesis.instantiator.ObjectInstantiator;
/**
* {@link SerializationUtils} class internally uses {@link Kryo} serializer for serializing / deserializing objects.
*/

1
hudi-common/src/main/java/org/apache/hudi/common/util/TimelineDiffHelper.java
View File

@@ -29,7 +29,6 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
public class TimelineDiffHelper {
protected static Logger log = LogManager.getLogger(TimelineDiffHelper.class);

2
hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryExecutor.java
View File

@@ -16,7 +16,6 @@
* limitations under the License.
*/
package org.apache.hudi.common.util.queue;
import java.util.Arrays;
@@ -143,7 +142,6 @@ public class BoundedInMemoryExecutor<I, O, E> {
}
}
public boolean isRemaining() {
return queue.iterator().hasNext();
}

2
hudi-common/src/main/java/org/apache/hudi/common/util/queue/BoundedInMemoryQueueConsumer.java
View File

@@ -20,7 +20,6 @@ package org.apache.hudi.common.util.queue;
import java.util.Iterator;
/**
* Consume entries from queue and execute callback function
*/
@@ -59,5 +58,4 @@ public abstract class BoundedInMemoryQueueConsumer<I, O> {
*/
protected abstract O getResult();
}

1
hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
View File

@@ -123,5 +123,4 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
assertArrayEquals(new Text("data3").getBytes(), archivedTimeline.getInstantDetails(instant3).get());
}
}

2
hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieLogFormat.java
View File

@@ -471,7 +471,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
}
@Test
public void testAppendAndReadOnCorruptedLog() throws IOException, URISyntaxException, InterruptedException {
Writer writer =
@@ -556,7 +555,6 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
reader.close();
}
@Test
public void testAvroLogRecordReaderBasic() throws IOException, URISyntaxException, InterruptedException {
Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema());

2
hudi-common/src/test/java/org/apache/hudi/common/table/view/TestIncrementalFSViewSync.java
View File

@@ -287,8 +287,6 @@ public class TestIncrementalFSViewSync extends HoodieCommonTestHarness {
* HELPER METHODS
*********************************************************************************************************
*/
/**
* Helper to run one or more rounds of cleaning, incrementally syncing the view and then validate
*/

1
hudi-common/src/test/java/org/apache/hudi/common/util/TestDFSPropertiesConfiguration.java
View File

@@ -41,7 +41,6 @@ public class TestDFSPropertiesConfiguration {
private static MiniDFSCluster dfsCluster;
private static DistributedFileSystem dfs;
@BeforeClass
public static void initClass() throws Exception {
hdfsTestService = new HdfsTestService();

1
hudi-common/src/test/java/org/apache/hudi/common/util/TestHoodieAvroUtils.java
View File

@@ -24,7 +24,6 @@ import org.codehaus.jackson.JsonNode;
import org.junit.Assert;
import org.junit.Test;
public class TestHoodieAvroUtils {
private static String EXAMPLE_SCHEMA = "{\"type\": \"record\"," + "\"name\": \"testrec\"," + "\"fields\": [ "

1
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/HoodieROTablePathFilter.java
View File

@@ -64,7 +64,6 @@ public class HoodieROTablePathFilter implements PathFilter, Serializable {
private transient FileSystem fs;
public HoodieROTablePathFilter() {
hoodiePathCache = new HashMap<>();
nonHoodiePathCache = new HashSet<>();

1
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/SafeParquetRecordReaderWrapper.java
View File

@@ -42,7 +42,6 @@ public class SafeParquetRecordReaderWrapper implements RecordReader<NullWritable
// Number of fields in Value Schema
private final int numValueFields;
public SafeParquetRecordReaderWrapper(RecordReader<NullWritable, ArrayWritable> parquetReader) {
this.parquetReader = parquetReader;
ArrayWritable arrayWritable = parquetReader.createValue();

1
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieParquetRealtimeInputFormat.java
View File

@@ -150,7 +150,6 @@ public class HoodieParquetRealtimeInputFormat extends HoodieParquetInputFormat i
return rtSplits.toArray(new InputSplit[rtSplits.size()]);
}
@Override
public FileStatus[] listStatus(JobConf job) throws IOException {
// Call the HoodieInputFormat::listStatus to obtain all latest parquet files, based on commit

1
hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/realtime/HoodieRealtimeFileSplit.java
View File

@@ -73,7 +73,6 @@ public class HoodieRealtimeFileSplit extends FileSplit {
return new String(bytes, StandardCharsets.UTF_8);
}
@Override
public void write(DataOutput out) throws IOException {
super.write(out);

1
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/InputFormatTestUtil.java
View File

@@ -113,7 +113,6 @@ public class InputFormatTestUtil {
return partitionPath;
}
public static File prepareSimpleParquetDataset(TemporaryFolder basePath, Schema schema, int numberOfFiles,
int numberOfRecords, String commitNumber) throws Exception {
basePath.create();

2
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java
View File

@@ -32,8 +32,8 @@ import org.junit.Before;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
/**
*
*/
public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {

1
hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestRecordReaderValueIterator.java
View File

@@ -58,7 +58,6 @@ public class TestRecordReaderValueIterator {
this.entries = entries;
}
@Override
public boolean next(IntWritable key, Text value) throws IOException {
if (currIndex >= entries.size()) {

1
hudi-hive/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
View File

@@ -157,7 +157,6 @@ public class HiveSyncTool {
}
}
/**
* Syncs the list of storage parititions passed in (checks if the partition is in hive, if not adds it or if the
* partition path does not match, it updates the partition path)

3
hudi-hive/src/main/java/org/apache/hudi/hive/HoodieHiveClient.java
View File

@@ -234,7 +234,6 @@ public class HoodieHiveClient {
return events;
}
/**
* Scan table partitions
*/
@@ -531,8 +530,6 @@ public class HoodieHiveClient {
return responses;
}
private void createHiveConnection() {
if (connection == null) {
try {

2
hudi-hive/src/main/java/org/apache/hudi/hive/util/SchemaUtil.java
View File

@@ -132,7 +132,6 @@ public class SchemaUtil {
return false;
}
/**
* Returns equivalent Hive table schema read from a parquet file
*
@@ -296,7 +295,6 @@ public class SchemaUtil {
return finalStr;
}
private static String hiveCompatibleFieldName(String fieldName, boolean isNested) {
String result = fieldName;
if (isNested) {

1
hudi-hive/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
View File

@@ -148,7 +148,6 @@ public class TestHiveSyncTool {
assertEquals("`map_list` ARRAY< MAP< string, int>>", schemaString);
}
@Test
public void testBasicSync() throws Exception {
TestUtil.hiveSyncConfig.useJdbc = this.useJdbc;

3
hudi-hive/src/test/java/org/apache/hudi/hive/util/HiveTestService.java
View File

@@ -218,8 +218,6 @@ public class HiveTestService {
// XXX: From org.apache.hadoop.hive.metastore.HiveMetaStore,
// with changes to support binding to a specified IP address (not only 0.0.0.0)
private static final class ChainedTTransportFactory extends TTransportFactory {
private final TTransportFactory parentTransFactory;
@@ -236,7 +234,6 @@ public class HiveTestService {
}
}
private static final class TServerSocketKeepAlive extends TServerSocket {
public TServerSocketKeepAlive(int port) throws TTransportException {

1
hudi-spark/src/main/java/org/apache/hudi/DataSourceUtils.java
View File

@@ -160,7 +160,6 @@ public class DataSourceUtils {
return new HoodieWriteClient<>(jssc, writeConfig, true);
}
public static JavaRDD<WriteStatus> doWriteOperation(HoodieWriteClient client, JavaRDD<HoodieRecord> hoodieRecords,
String commitTime, String operation) {
if (operation.equals(DataSourceWriteOptions.BULK_INSERT_OPERATION_OPT_VAL())) {

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/HoodieCompactor.java
View File

@@ -35,7 +35,6 @@ import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
public class HoodieCompactor {
private static volatile Logger logger = LogManager.getLogger(HoodieCompactor.class);

2
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java
View File

@@ -75,7 +75,6 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import scala.collection.JavaConversions;
/**
* Sync's one batch of data to hoodie dataset
*/
@@ -155,7 +154,6 @@ public class DeltaSync implements Serializable {
*/
private final HoodieTableType tableType;
public DeltaSync(HoodieDeltaStreamer.Config cfg, SparkSession sparkSession, SchemaProvider schemaProvider,
HoodieTableType tableType, TypedProperties props, JavaSparkContext jssc, FileSystem fs, HiveConf hiveConf,
Function<HoodieWriteClient, Boolean> onInitializingHoodieWriteClient) throws IOException {

3
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
View File

@@ -65,7 +65,6 @@ import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SparkSession;
/**
* An Utility which can incrementally take the output from {@link HiveIncrementalPuller} and apply it to the target
* dataset. Does not maintain any state, queries at runtime to see how far behind the target dataset is from the source
@@ -267,11 +266,9 @@ public class HoodieDeltaStreamer implements Serializable {
@Parameter(names = {"--checkpoint"}, description = "Resume Delta Streamer from this checkpoint.")
public String checkpoint = null;
@Parameter(names = {"--help", "-h"}, help = true)
public Boolean help = false;
public boolean isAsyncCompactionEnabled() {
return continuousMode && !forceDisableCompaction
&& HoodieTableType.MERGE_ON_READ.equals(HoodieTableType.valueOf(storageType));

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
View File

@@ -57,7 +57,6 @@ public class SchedulerConfGenerator {
compactionMinShare.toString());
}
/**
* Helper to set Spark Scheduling Configs dynamically
*

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SourceFormatAdapter.java
View File

@@ -44,7 +44,6 @@ public final class SourceFormatAdapter {
private final Source source;
public SourceFormatAdapter(Source source) {
this.source = source;
}

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/keygen/TimestampBasedKeyGenerator.java
View File

@@ -48,7 +48,6 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
private final String outputDateFormat;
/**
* Supported configs
*/

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/perf/TimelineServerPerf.java
View File

@@ -209,7 +209,6 @@ public class TimelineServerPerf implements Serializable {
}
}
private static class PerfStats implements Serializable {
private final String partition;

1
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/HiveIncrPullSource.java
View File

@@ -63,7 +63,6 @@ public class HiveIncrPullSource extends AvroSource {
private final String incrPullRootPath;
/**
* Configs supported
*/

3
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/AvroConvertor.java
View File

@@ -50,7 +50,6 @@ public class AvroConvertor implements Serializable {
*/
private transient Injection<GenericRecord, byte[]> recordInjection;
public AvroConvertor(String schemaStr) {
this.schemaStr = schemaStr;
}
@@ -79,7 +78,6 @@ public class AvroConvertor implements Serializable {
}
}
public GenericRecord fromJson(String json) throws IOException {
initSchema();
initJsonConvertor();
@@ -90,7 +88,6 @@ public class AvroConvertor implements Serializable {
return new Schema.Parser().parse(schemaStr);
}
public GenericRecord fromAvroBinary(byte[] avroBinary) {
initSchema();
initInjection();

2
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/helpers/KafkaOffsetGen.java
View File

@@ -43,7 +43,6 @@ import scala.collection.mutable.ArrayBuffer;
import scala.collection.mutable.StringBuilder;
import scala.util.Either;
/**
* Source to read data from Kafka, incrementally
*/
@@ -250,7 +249,6 @@ public class KafkaOffsetGen {
return checkpointOffsetReseter ? earliestOffsets : checkpointOffsets;
}
public String getTopicName() {
return topicName;
}

1
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHDFSParquetImporter.java
View File

@@ -61,7 +61,6 @@ public class TestHDFSParquetImporter implements Serializable {
private static MiniDFSCluster dfsCluster;
private static DistributedFileSystem dfs;
@BeforeClass
public static void initClass() throws Exception {
hdfsTestService = new HdfsTestService();

2
hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestKafkaSource.java
View File

@@ -78,7 +78,6 @@ public class TestKafkaSource extends UtilitiesTestBase {
testUtils.teardown();
}
@Test
public void testJsonKafkaSource() throws IOException {
@@ -132,7 +131,6 @@ public class TestKafkaSource extends UtilitiesTestBase {
assertEquals(Option.empty(), fetch4AsRows.getBatch());
}
private static HashMap<TopicAndPartition, LeaderOffset> makeOffsetMap(int[] partitions, long[] offsets) {
HashMap<TopicAndPartition, LeaderOffset> map = new HashMap<>();
for (int i = 0; i < partitions.length; i++) {

3
style/checkstyle.xml
View File

@@ -101,6 +101,9 @@
<module name="ModifierOrder"/>
<module name="EmptyLineSeparator">
<property name="allowNoEmptyLineBetweenFields" value="true"/>
<property name="allowMultipleEmptyLines" value="false"/>
<property name="tokens" value="PACKAGE_DEF, IMPORT, CLASS_DEF, INTERFACE_DEF, ENUM_DEF,
STATIC_INIT, INSTANCE_INIT, METHOD_DEF, CTOR_DEF"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapDot"/>