lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1349] spark sql support overwrite use insert_overwrite_table (#2196)

Browse Source
This commit is contained in:
lw0090
2020-12-04 04:26:21 +08:00
committed by GitHub
parent 78fd122594
commit 1f0d5c077e
14 changed files with 224 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
hudi-common/src/main/java/org/apache/hudi/common/model/WriteOperationType.java
View File

@@ -38,10 +38,12 @@ public enum WriteOperationType {
// delete
DELETE("delete"),
BOOTSTRAP("bootstrap"),
// insert overwrite
// insert overwrite with static partitioning
INSERT_OVERWRITE("insert_overwrite"),
// cluster
CLUSTER("cluster"),
// insert overwrite with dynamic partitioning
INSERT_OVERWRITE_TABLE("insert_overwrite_table"),
// used for old version
UNKNOWN("unknown");
@@ -72,6 +74,8 @@ public enum WriteOperationType {
return DELETE;
case "insert_overwrite":
return INSERT_OVERWRITE;
case "insert_overwrite_table":
return INSERT_OVERWRITE_TABLE;
default:
throw new HoodieException("Invalid value of Type.");
}
@@ -88,4 +92,4 @@ public enum WriteOperationType {
public static boolean isChangingRecords(WriteOperationType operationType) {
return operationType == UPSERT || operationType == UPSERT_PREPPED || operationType == DELETE;
}
}
}

4
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestDataGenerator.java
View File

@@ -451,6 +451,10 @@ public class HoodieTestDataGenerator {
return generateInsertsStream(instantTime, n, false, TRIP_EXAMPLE_SCHEMA, true).collect(Collectors.toList());
}
public List<HoodieRecord> generateInsertsForPartition(String instantTime, Integer n, String partition) {
return generateInsertsStream(instantTime, n, false, TRIP_EXAMPLE_SCHEMA, false, () -> partition, () -> UUID.randomUUID().toString()).collect(Collectors.toList());
}
public Stream<HoodieRecord> generateInsertsStream(String commitTime, Integer n, boolean isFlattened, String schemaStr, boolean containsAllPartitions) {
return generateInsertsStream(commitTime, n, isFlattened, schemaStr, containsAllPartitions,
() -> partitionPaths[RAND.nextInt(partitionPaths.length)],