lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Preparation for Avro update (#2650)

Browse Source
This commit is contained in:
Sebastian Bernauer
2021-03-31 06:50:17 +02:00
committed by GitHub
parent 8bc65b9318
commit aa0da72c59
8 changed files with 14 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/index/simple/FlinkHoodieSimpleIndex.java
View File

@@ -34,8 +34,6 @@ import org.apache.hudi.index.HoodieIndexUtils;
import org.apache.hudi.io.HoodieKeyLocationFetchHandle;
import org.apache.hudi.table.HoodieTable;
import avro.shaded.com.google.common.collect.Lists;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
@@ -135,8 +133,8 @@ public class FlinkHoodieSimpleIndex<T extends HoodieRecordPayload> extends Flink
context.map(latestBaseFiles, partitionPathBaseFile -> new HoodieKeyLocationFetchHandle<>(config, hoodieTable, partitionPathBaseFile), parallelism);
Map<HoodieKey, HoodieRecordLocation> recordLocations = new HashMap<>();
hoodieKeyLocationFetchHandles.stream()
.flatMap(handle -> Lists.newArrayList(handle.locations()).stream())
.forEach(x -> x.forEach(y -> recordLocations.put(y.getKey(), y.getRight())));
.flatMap(handle -> handle.locations())
.forEach(x -> recordLocations.put(x.getKey(), x.getRight()));
return recordLocations;
}
}

4
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndex.java
View File

@@ -18,7 +18,6 @@
package org.apache.hudi.index.hbase;
import avro.shaded.com.google.common.collect.Maps;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
@@ -65,6 +64,7 @@ import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.HashMap;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -469,7 +469,7 @@ public class TestHBaseIndex extends FunctionalTestHarness {
@Test
public void testHbaseTagLocationForArchivedCommits() throws Exception {
// Load to memory
Map<String, String> params = Maps.newHashMap();
Map<String, String> params = new HashMap<String, String>();
params.put(HoodieCompactionConfig.CLEANER_COMMITS_RETAINED_PROP, "1");
params.put(HoodieCompactionConfig.MAX_COMMITS_TO_KEEP_PROP, "3");
params.put(HoodieCompactionConfig.MIN_COMMITS_TO_KEEP_PROP, "2");