lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1845] Exception Throws When Sync Non-Partitioned Table To Hive With MultiPartKeysValueExtractor (#2876)

Browse Source
This commit is contained in:
pengzhiwei
2021-04-29 10:11:46 +08:00
committed by GitHub
parent 3ca9030256
commit c9bcb5e33f
2 changed files with 47 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/MultiPartKeysValueExtractor.java
View File

@@ -18,6 +18,7 @@
package org.apache.hudi.hive;
import java.util.Collections;
import org.apache.hudi.common.util.ValidationUtils;
import java.util.Arrays;
@@ -31,6 +32,11 @@ public class MultiPartKeysValueExtractor implements PartitionValueExtractor {
@Override
public List<String> extractPartitionValuesInPath(String partitionPath) {
// If the partitionPath is empty string( which means none-partition table), the partition values
// should be empty list.
if (partitionPath.isEmpty()) {
return Collections.emptyList();
}
String[] splits = partitionPath.split("/");
return Arrays.stream(splits).map(s -> {
if (s.contains("=")) {

41
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestMultiPartKeysValueExtractor.java Normal file
View File

@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.hive;
import java.util.ArrayList;
import java.util.List;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
public class TestMultiPartKeysValueExtractor {
@Test
public void testMultiPartExtractor() {
MultiPartKeysValueExtractor valueExtractor = new MultiPartKeysValueExtractor();
// Test extract empty partitionPath
assertEquals(new ArrayList<>(), valueExtractor.extractPartitionValuesInPath(""));
List<String> expected = new ArrayList<>();
expected.add("2021-04-25");
expected.add("04");
// Test extract multi-partition path
assertEquals(expected, valueExtractor.extractPartitionValuesInPath("2021-04-25/04"));
// Test extract hive style partition path
assertEquals(expected, valueExtractor.extractPartitionValuesInPath("ds=2021-04-25/hh=04"));
}
}