lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1218] Introduce BulkInsertSortMode as Independent class (#2021)

Browse Source
This commit is contained in:
Trevor
2020-08-25 19:04:13 +08:00
committed by GitHub
parent 218d4a6836
commit 6a4dc7384c
4 changed files with 34 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -27,7 +27,7 @@ import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.execution.bulkinsert.BulkInsertInternalPartitionerFactory.BulkInsertSortMode;
import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.metrics.MetricsReporterType;
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;

8
hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertInternalPartitionerFactory.java
View File

@@ -36,13 +36,7 @@ public abstract class BulkInsertInternalPartitionerFactory {
case PARTITION_SORT:
return new RDDPartitionSortPartitioner();
default:
throw new HoodieException("The bulk insert mode \"" + sortMode.name() + "\" is not supported.");
throw new HoodieException("The bulk insert sort mode \"" + sortMode.name() + "\" is not supported.");
}
}
public enum BulkInsertSortMode {
NONE,
GLOBAL_SORT,
PARTITION_SORT
}
}

28
hudi-client/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java Normal file
View File

@@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.execution.bulkinsert;
/**
* Bulk insert sort mode.
*/
public enum BulkInsertSortMode {
NONE,
GLOBAL_SORT,
PARTITION_SORT
}

8
hudi-client/src/test/java/org/apache/hudi/execution/bulkinsert/TestBulkInsertInternalPartitioner.java
View File

@@ -62,9 +62,9 @@ public class TestBulkInsertInternalPartitioner extends HoodieClientTestBase {
private static Stream<Arguments> configParams() {
Object[][] data = new Object[][] {
{BulkInsertInternalPartitionerFactory.BulkInsertSortMode.GLOBAL_SORT, true, true},
{BulkInsertInternalPartitionerFactory.BulkInsertSortMode.PARTITION_SORT, false, true},
{BulkInsertInternalPartitionerFactory.BulkInsertSortMode.NONE, false, false}
{BulkInsertSortMode.GLOBAL_SORT, true, true},
{BulkInsertSortMode.PARTITION_SORT, false, true},
{BulkInsertSortMode.NONE, false, false}
};
return Stream.of(data).map(Arguments::of);
}
@@ -108,7 +108,7 @@ public class TestBulkInsertInternalPartitioner extends HoodieClientTestBase {
@ParameterizedTest(name = "[{index}] {0}")
@MethodSource("configParams")
public void testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerFactory.BulkInsertSortMode sortMode,
public void testBulkInsertInternalPartitioner(BulkInsertSortMode sortMode,
boolean isGloballySorted, boolean isLocallySorted)
throws Exception {
JavaRDD<HoodieRecord> records1 = generateTestRecordsForBulkInsert(jsc);