lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1762] Added HiveStylePartitionExtractor to support Hive style partitions (#2769)

Browse Source
This commit is contained in:
Vinoth Govindarajan
2021-04-08 22:00:11 -07:00
committed by GitHub
parent cf3d2e21eb
commit 08e82c469c
2 changed files with 54 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveStylePartitionValueExtractor.java Normal file
View File

@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.hive;
import java.util.Collections;
import java.util.List;
/**
* Extractor for Hive Style Partitioned tables, when the parition folders are key value pairs.
*
* <p>This implementation extracts the partition value of yyyy-mm-dd from the path of type datestr=yyyy-mm-dd.
*/
public class HiveStylePartitionValueExtractor implements PartitionValueExtractor {
private static final long serialVersionUID = 1L;
@Override
public List<String> extractPartitionValuesInPath(String partitionPath) {
// partition path is expected to be in this format partition_key=partition_value.
String[] splits = partitionPath.split("=");
if (splits.length != 2) {
throw new IllegalArgumentException(
"Partition path " + partitionPath + " is not in the form partition_key=partition_value.");
}
return Collections.singletonList(splits[1]);
}
}

11
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestPartitionValueExtractor.java
View File

@@ -35,4 +35,15 @@ public class TestPartitionValueExtractor {
assertThrows(IllegalArgumentException.class, () -> hourPartition.extractPartitionValuesInPath("2020/12/20")); assertThrows(IllegalArgumentException.class, () -> hourPartition.extractPartitionValuesInPath("2020/12/20"));
assertEquals(hourPartition.extractPartitionValuesInPath("update_time=2020/12/20/01"), list); assertEquals(hourPartition.extractPartitionValuesInPath("update_time=2020/12/20/01"), list);
} }
@Test
public void testHiveStylePartition() {
HiveStylePartitionValueExtractor hiveStylePartition = new HiveStylePartitionValueExtractor();
List<String> list = new ArrayList<>();
list.add("2021-04-02");
assertEquals(hiveStylePartition.extractPartitionValuesInPath("datestr=2021-04-02"), list);
assertThrows(
IllegalArgumentException.class,
() -> hiveStylePartition.extractPartitionValuesInPath("2021/04/02"));
}
} }