lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3876] Fixing fetching partitions in GlueSyncClient (#5318)

Browse Source
This commit is contained in:
Sivabalan Narayanan
2022-04-14 00:03:05 -04:00
committed by GitHub
parent 571cbe4c11
commit a081c2b9b5
1 changed files with 13 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
hudi-aws/src/main/java/org/apache/hudi/aws/sync/AWSGlueCatalogSyncClient.java
View File

@@ -94,13 +94,19 @@ public class AWSGlueCatalogSyncClient extends AbstractHiveSyncHoodieClient {
@Override @Override
public List<Partition> getAllPartitions(String tableName) { public List<Partition> getAllPartitions(String tableName) {
try { try {
GetPartitionsRequest request = new GetPartitionsRequest(); List<Partition> partitions = new ArrayList<>();
request.withDatabaseName(databaseName).withTableName(tableName); String nextToken = null;
GetPartitionsResult result = awsGlue.getPartitions(request); do {
return result.getPartitions() GetPartitionsResult result = awsGlue.getPartitions(new GetPartitionsRequest()
.stream() .withDatabaseName(databaseName)
.withTableName(tableName)
.withNextToken(nextToken));
partitions.addAll(result.getPartitions().stream()
.map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation())) .map(p -> new Partition(p.getValues(), p.getStorageDescriptor().getLocation()))
.collect(Collectors.toList()); .collect(Collectors.toList()));
nextToken = result.getNextToken();
} while (nextToken != null);
return partitions;
} catch (Exception e) { } catch (Exception e) {
throw new HoodieGlueSyncException("Failed to get all partitions for table " + tableId(databaseName, tableName), e); throw new HoodieGlueSyncException("Failed to get all partitions for table " + tableId(databaseName, tableName), e);
} }