lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-346] Set allowMultipleEmptyLines to false for EmptyLineSeparator rule (#1025)

Browse Source
This commit is contained in:
谢磊
2019-11-19 18:44:42 +08:00
committed by leesf
parent 66492498f7
commit 804e348d0e
82 changed files with 5 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
View File

@@ -132,7 +132,6 @@ public class HoodieCompactionConfig extends DefaultHoodieConfig {
return this;
}
public Builder withAutoClean(Boolean autoClean) {
props.setProperty(AUTO_CLEAN_PROP, String.valueOf(autoClean));
return this;

1
hudi-client/src/main/java/org/apache/hudi/config/HoodieMetricsConfig.java
View File

@@ -74,7 +74,6 @@ public class HoodieMetricsConfig extends DefaultHoodieConfig {
return this;
}
public Builder on(boolean metricsOn) {
props.setProperty(METRICS_ON, String.valueOf(metricsOn));
return this;

1
hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -147,7 +147,6 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
return Integer.parseInt(props.getProperty(ROLLBACK_PARALLELISM));
}
public int getWriteBufferLimitBytes() {
return Integer.parseInt(props.getProperty(WRITE_BUFFER_LIMIT_BYTES, DEFAULT_WRITE_BUFFER_LIMIT_BYTES));
}

1
hudi-client/src/main/java/org/apache/hudi/exception/HoodieDependentSystemUnavailableException.java
View File

@@ -18,7 +18,6 @@
package org.apache.hudi.exception;
/**
* <p>
* Exception thrown when dependent system is not available

1
hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java
View File

@@ -27,7 +27,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
import org.apache.spark.api.java.function.Function2;
/**
* Map function that handles a sorted stream of HoodieRecords
*/

1
hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java
View File

@@ -52,7 +52,6 @@ public abstract class LazyIterableIterator<I, O> implements Iterable<O>, Iterato
*/
protected abstract O computeNext();
/**
* Called once, after all elements are processed.
*/

2
hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java
View File

@@ -47,7 +47,6 @@ public abstract class HoodieIndex<T extends HoodieRecordPayload> implements Seri
this.config = config;
}
public static <T extends HoodieRecordPayload> HoodieIndex<T> createIndex(HoodieWriteConfig config,
JavaSparkContext jsc) throws HoodieIndexException {
switch (config.getIndexType()) {
@@ -108,7 +107,6 @@ public abstract class HoodieIndex<T extends HoodieRecordPayload> implements Seri
*/
public abstract boolean canIndexLogFiles();
/**
* An index is "implicit" with respect to storage, if just writing new data to a file slice, updates the index as
* well. This is used by storage, to save memory footprint in certain cases.

1
hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java
View File

@@ -38,7 +38,6 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.Function2;
/**
* Hoodie Index implementation backed by an in-memory Hash map.
* <p>

1
hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
View File

@@ -268,7 +268,6 @@ public class HoodieBloomIndex<T extends HoodieRecordPayload> extends HoodieIndex
}
}
@Override
public boolean rollbackCommit(String commitTime) {
// Nope, don't need to do anything.

1
hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java
View File

@@ -170,7 +170,6 @@ public class HoodieCleanHelper<T extends HoodieRecordPayload<T>> implements Seri
return deletePaths;
}
/**
* Selects the versions for file for cleaning, such that it
* <p>

1
hudi-client/src/main/java/org/apache/hudi/io/HoodieIOHandle.java
View File

@@ -23,7 +23,6 @@ import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.HoodieTable;
public abstract class HoodieIOHandle<T extends HoodieRecordPayload> {
protected final String instantTime;

1
hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
View File

@@ -86,7 +86,6 @@ public class HoodieMergeHandle<T extends HoodieRecordPayload> extends HoodieWrit
dataFileToBeMerged);
}
public static Schema createHoodieWriteSchema(Schema originalSchema) {
return HoodieAvroUtils.addMetadataFields(originalSchema);
}

1
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java
View File

@@ -16,7 +16,6 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
import com.google.common.annotations.VisibleForTesting;

1
hudi-client/src/main/java/org/apache/hudi/io/storage/HoodieParquetWriter.java
View File

@@ -51,7 +51,6 @@ public class HoodieParquetWriter<T extends HoodieRecordPayload, R extends Indexe
private final String commitTime;
private final Schema schema;
public HoodieParquetWriter(String commitTime, Path file, HoodieParquetConfig parquetConfig, Schema schema)
throws IOException {
super(HoodieWrapperFileSystem.convertToHoodiePath(file, parquetConfig.getHadoopConf()),

1
hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
View File

@@ -80,7 +80,6 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.PairFlatMapFunction;
import scala.Tuple2;
/**
* Implementation of a very heavily read-optimized Hoodie Table where
* <p>

1
hudi-client/src/main/java/org/apache/hudi/table/RollbackExecutor.java
View File

@@ -212,7 +212,6 @@ public class RollbackExecutor implements Serializable {
return results;
}
private Map<HeaderMetadataType, String> generateHeader(String commit) {
// generate metadata
Map<HeaderMetadataType, String> header = Maps.newHashMap();

1
hudi-client/src/main/java/org/apache/hudi/table/WorkloadProfile.java
View File

@@ -49,7 +49,6 @@ public class WorkloadProfile<T extends HoodieRecordPayload> implements Serializa
private final WorkloadStat globalStat;
public WorkloadProfile(JavaRDD<HoodieRecord<T>> taggedRecords) {
this.taggedRecords = taggedRecords;
this.partitionPathStatMap = new HashMap<>();

1
hudi-client/src/test/java/HoodieClientExample.java
View File

@@ -67,7 +67,6 @@ public class HoodieClientExample {
cli.run();
}
public void run() throws Exception {
SparkConf sparkConf = new SparkConf().setAppName("hoodie-client-example");

1
hudi-client/src/test/java/org/apache/hudi/TestConsistencyGuard.java
View File

@@ -70,7 +70,6 @@ public class TestConsistencyGuard extends HoodieClientTestHarness {
.asList(basePath + "/partition/path/f1_1-0-2_000.parquet", basePath + "/partition/path/f2_1-0-2_000.parquet"));
}
@Test(expected = TimeoutException.class)
public void testCheckFailingAppears() throws Exception {
HoodieClientTestUtils.fakeDataFile(basePath, "partition/path", "000", "f1");

1
hudi-client/src/test/java/org/apache/hudi/common/HoodieClientTestUtils.java
View File

@@ -92,7 +92,6 @@ public class HoodieClientTestUtils {
new File(parentPath + "/" + commitTime + suffix).createNewFile();
}
public static void fakeCommitFile(String basePath, String commitTime) throws IOException {
fakeMetaFile(basePath, commitTime, HoodieTimeline.COMMIT_EXTENSION);
}

2
hudi-client/src/test/java/org/apache/hudi/common/TestRawTripPayload.java
View File

@@ -79,7 +79,6 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa
return partitionPath;
}
@Override
public TestRawTripPayload preCombine(TestRawTripPayload another) {
return another;
@@ -129,7 +128,6 @@ public class TestRawTripPayload implements HoodieRecordPayload<TestRawTripPayloa
return baos.toByteArray();
}
private String unCompressData(byte[] data) throws IOException {
try (InflaterInputStream iis = new InflaterInputStream(new ByteArrayInputStream(data))) {
return FileIOUtils.readAsUTFString(iis, dataSize);

2
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieBloomIndex.java
View File

@@ -291,7 +291,6 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
}
}
@Test
public void testTagLocation() throws Exception {
// We have some records to be tagged (two different partitions)
@@ -433,7 +432,6 @@ public class TestHoodieBloomIndex extends HoodieClientTestHarness {
}
}
@Test
public void testBloomFilterFalseError() throws IOException, InterruptedException {
// We have two hoodie records

1
hudi-client/src/test/java/org/apache/hudi/index/bloom/TestHoodieGlobalBloomIndex.java
View File

@@ -196,7 +196,6 @@ public class TestHoodieGlobalBloomIndex extends HoodieClientTestHarness {
assertEquals(new HashSet<>(Arrays.asList("f4", "f1")), new HashSet<>(recordKeyToFileComps.get("005")));
}
@Test
public void testTagLocation() throws Exception {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).build();

4
hudi-client/src/test/java/org/apache/hudi/table/TestCopyOnWriteTable.java
View File

@@ -250,7 +250,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
assertEquals(4, writeStatus.getStat().getNumWrites());// 3 rewritten records + 1 new record
}
private List<HoodieRecord> newHoodieRecords(int n, String time) throws Exception {
List<HoodieRecord> records = new ArrayList<>();
for (int i = 0; i < n; i++) {
@@ -387,7 +386,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
assertEquals("If the number of records are more than 1150, then there should be a new file", 3, counts);
}
private UpsertPartitioner getUpsertPartitioner(int smallFileSize, int numInserts, int numUpdates, int fileSize,
String testPartitionPath, boolean autoSplitInserts) throws Exception {
HoodieWriteConfig config = makeHoodieClientConfigBuilder()
@@ -419,7 +417,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
return partitioner;
}
@Test
public void testUpsertPartitioner() throws Exception {
final String testPartitionPath = "2016/09/26";
@@ -429,7 +426,6 @@ public class TestCopyOnWriteTable extends HoodieClientTestHarness {
assertEquals("Total of 2 insert buckets", 2, insertBuckets.size());
}
@Test
public void testUpsertPartitionerWithSmallInsertHandling() throws Exception {
final String testPartitionPath = "2016/09/26";