[HUDI-1218] Introduce BulkInsertSortMode as Independent class (#2021)
This commit is contained in:
@@ -27,7 +27,7 @@ import org.apache.hudi.common.model.HoodieCleaningPolicy;
|
|||||||
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
|
import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
|
||||||
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
|
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
|
||||||
import org.apache.hudi.common.util.ReflectionUtils;
|
import org.apache.hudi.common.util.ReflectionUtils;
|
||||||
import org.apache.hudi.execution.bulkinsert.BulkInsertInternalPartitionerFactory.BulkInsertSortMode;
|
import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
|
||||||
import org.apache.hudi.index.HoodieIndex;
|
import org.apache.hudi.index.HoodieIndex;
|
||||||
import org.apache.hudi.metrics.MetricsReporterType;
|
import org.apache.hudi.metrics.MetricsReporterType;
|
||||||
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
|
import org.apache.hudi.metrics.datadog.DatadogHttpClient.ApiSite;
|
||||||
|
|||||||
@@ -36,13 +36,7 @@ public abstract class BulkInsertInternalPartitionerFactory {
|
|||||||
case PARTITION_SORT:
|
case PARTITION_SORT:
|
||||||
return new RDDPartitionSortPartitioner();
|
return new RDDPartitionSortPartitioner();
|
||||||
default:
|
default:
|
||||||
throw new HoodieException("The bulk insert mode \"" + sortMode.name() + "\" is not supported.");
|
throw new HoodieException("The bulk insert sort mode \"" + sortMode.name() + "\" is not supported.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public enum BulkInsertSortMode {
|
|
||||||
NONE,
|
|
||||||
GLOBAL_SORT,
|
|
||||||
PARTITION_SORT
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,28 @@
|
|||||||
|
/*
|
||||||
|
* Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
* or more contributor license agreements. See the NOTICE file
|
||||||
|
* distributed with this work for additional information
|
||||||
|
* regarding copyright ownership. The ASF licenses this file
|
||||||
|
* to you under the Apache License, Version 2.0 (the
|
||||||
|
* "License"); you may not use this file except in compliance
|
||||||
|
* with the License. You may obtain a copy of the License at
|
||||||
|
*
|
||||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
*
|
||||||
|
* Unless required by applicable law or agreed to in writing, software
|
||||||
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
* See the License for the specific language governing permissions and
|
||||||
|
* limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package org.apache.hudi.execution.bulkinsert;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bulk insert sort mode.
|
||||||
|
*/
|
||||||
|
public enum BulkInsertSortMode {
|
||||||
|
NONE,
|
||||||
|
GLOBAL_SORT,
|
||||||
|
PARTITION_SORT
|
||||||
|
}
|
||||||
@@ -62,9 +62,9 @@ public class TestBulkInsertInternalPartitioner extends HoodieClientTestBase {
|
|||||||
|
|
||||||
private static Stream<Arguments> configParams() {
|
private static Stream<Arguments> configParams() {
|
||||||
Object[][] data = new Object[][] {
|
Object[][] data = new Object[][] {
|
||||||
{BulkInsertInternalPartitionerFactory.BulkInsertSortMode.GLOBAL_SORT, true, true},
|
{BulkInsertSortMode.GLOBAL_SORT, true, true},
|
||||||
{BulkInsertInternalPartitionerFactory.BulkInsertSortMode.PARTITION_SORT, false, true},
|
{BulkInsertSortMode.PARTITION_SORT, false, true},
|
||||||
{BulkInsertInternalPartitionerFactory.BulkInsertSortMode.NONE, false, false}
|
{BulkInsertSortMode.NONE, false, false}
|
||||||
};
|
};
|
||||||
return Stream.of(data).map(Arguments::of);
|
return Stream.of(data).map(Arguments::of);
|
||||||
}
|
}
|
||||||
@@ -108,7 +108,7 @@ public class TestBulkInsertInternalPartitioner extends HoodieClientTestBase {
|
|||||||
|
|
||||||
@ParameterizedTest(name = "[{index}] {0}")
|
@ParameterizedTest(name = "[{index}] {0}")
|
||||||
@MethodSource("configParams")
|
@MethodSource("configParams")
|
||||||
public void testBulkInsertInternalPartitioner(BulkInsertInternalPartitionerFactory.BulkInsertSortMode sortMode,
|
public void testBulkInsertInternalPartitioner(BulkInsertSortMode sortMode,
|
||||||
boolean isGloballySorted, boolean isLocallySorted)
|
boolean isGloballySorted, boolean isLocallySorted)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
JavaRDD<HoodieRecord> records1 = generateTestRecordsForBulkInsert(jsc);
|
JavaRDD<HoodieRecord> records1 = generateTestRecordsForBulkInsert(jsc);
|
||||||
|
|||||||
Reference in New Issue
Block a user