lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3876] Fixing fetching partitions in GlueSyncClient (#5318)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2022-04-14 00:03:05 -04:00
committed by GitHub
parent 571cbe4c11
commit a081c2b9b5
1 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
View File

@@ -94,13 +94,19 @@ public class AWSGlueCatalogSyncClient extends AbstractHiveSyncHoodieClient {
@Override
public List<Partition> getAllPartitions(String tableName) {
try {
GetPartitionsRequest request = new GetPartitionsRequest();
request.withDatabaseName(databaseName).withTableName(tableName);
GetPartitionsResult result = awsGlue.getPartitions(request);
return result.getPartitions()
.stream()
.map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation()))
.collect(Collectors.toList());
List<Partition> partitions = new ArrayList<>();
String nextToken = null;
do {
GetPartitionsResult result = awsGlue.getPartitions(new GetPartitionsRequest()
.withDatabaseName(databaseName)
.withTableName(tableName)
.withNextToken(nextToken));
partitions.addAll(result.getPartitions().stream()
.map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation()))
.collect(Collectors.toList()));
nextToken = result.getNextToken();
} while (nextToken != null);
return partitions;
} catch (Exception e) {
throw new HoodieGlueSyncException("Failed to get all partitions for table " + tableId(databaseName, tableName), e);
}