lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2013] Removed option to fallback to file listing when Metadata Table is enabled. (#3079)

Browse Source
This commit is contained in:
Prashant Wason
2021-06-22 08:41:52 -07:00
committed by GitHub
parent 69c0d9e2d0
commit 062d5baf84
3 changed files with 4 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/metadata/TestHoodieBackedMetadata.java
View File

@@ -969,8 +969,7 @@ public class TestHoodieBackedMetadata extends HoodieClientTestHarness {
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()) .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build())
.withMetadataConfig(HoodieMetadataConfig.newBuilder() .withMetadataConfig(HoodieMetadataConfig.newBuilder()
.enable(useFileListingMetadata) .enable(useFileListingMetadata)
.enableMetrics(enableMetrics) .enableMetrics(enableMetrics).build())
.enableFallback(false).build())
.withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics) .withMetricsConfig(HoodieMetricsConfig.newBuilder().on(enableMetrics)
.withExecutorMetrics(true).usePrefix("unit-test").build()); .withExecutorMetrics(true).usePrefix("unit-test").build());
} }

15
hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
View File

@@ -67,10 +67,6 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
public static final String CLEANER_COMMITS_RETAINED_PROP = METADATA_PREFIX + ".cleaner.commits.retained"; public static final String CLEANER_COMMITS_RETAINED_PROP = METADATA_PREFIX + ".cleaner.commits.retained";
public static final int DEFAULT_CLEANER_COMMITS_RETAINED = 3; public static final int DEFAULT_CLEANER_COMMITS_RETAINED = 3;
// Controls whether or not, upon failure to fetch from metadata table, should fallback to listing.
public static final String ENABLE_FALLBACK_PROP = METADATA_PREFIX + ".fallback.enable";
public static final String DEFAULT_ENABLE_FALLBACK = "true";
// Regex to filter out matching directories during bootstrap // Regex to filter out matching directories during bootstrap
public static final String DIRECTORY_FILTER_REGEX = METADATA_PREFIX + ".dir.filter.regex"; public static final String DIRECTORY_FILTER_REGEX = METADATA_PREFIX + ".dir.filter.regex";
public static final String DEFAULT_DIRECTORY_FILTER_REGEX = ""; public static final String DEFAULT_DIRECTORY_FILTER_REGEX = "";
@@ -101,10 +97,6 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
return Boolean.parseBoolean(props.getProperty(METADATA_ENABLE_PROP)); return Boolean.parseBoolean(props.getProperty(METADATA_ENABLE_PROP));
} }
public boolean enableFallback() {
return Boolean.parseBoolean(props.getProperty(ENABLE_FALLBACK_PROP));
}
public boolean validateFileListingMetadata() { public boolean validateFileListingMetadata() {
return Boolean.parseBoolean(props.getProperty(METADATA_VALIDATE_PROP)); return Boolean.parseBoolean(props.getProperty(METADATA_VALIDATE_PROP));
} }
@@ -143,11 +135,6 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
return this; return this;
} }
public Builder enableFallback(boolean fallback) {
props.setProperty(ENABLE_FALLBACK_PROP, String.valueOf(fallback));
return this;
}
public Builder validate(boolean validate) { public Builder validate(boolean validate) {
props.setProperty(METADATA_VALIDATE_PROP, String.valueOf(validate)); props.setProperty(METADATA_VALIDATE_PROP, String.valueOf(validate));
return this; return this;
@@ -218,8 +205,6 @@ public final class HoodieMetadataConfig extends DefaultHoodieConfig {
String.valueOf(DEFAULT_FILE_LISTING_PARALLELISM)); String.valueOf(DEFAULT_FILE_LISTING_PARALLELISM));
setDefaultOnCondition(props, !props.containsKey(HOODIE_ASSUME_DATE_PARTITIONING_PROP), setDefaultOnCondition(props, !props.containsKey(HOODIE_ASSUME_DATE_PARTITIONING_PROP),
HOODIE_ASSUME_DATE_PARTITIONING_PROP, DEFAULT_ASSUME_DATE_PARTITIONING); HOODIE_ASSUME_DATE_PARTITIONING_PROP, DEFAULT_ASSUME_DATE_PARTITIONING);
setDefaultOnCondition(props, !props.containsKey(ENABLE_FALLBACK_PROP), ENABLE_FALLBACK_PROP,
DEFAULT_ENABLE_FALLBACK);
setDefaultOnCondition(props, !props.containsKey(DIRECTORY_FILTER_REGEX), DIRECTORY_FILTER_REGEX, setDefaultOnCondition(props, !props.containsKey(DIRECTORY_FILTER_REGEX), DIRECTORY_FILTER_REGEX,
DEFAULT_DIRECTORY_FILTER_REGEX); DEFAULT_DIRECTORY_FILTER_REGEX);
return config; return config;

13
hudi-common/src/main/java/org/apache/hudi/metadata/BaseTableMetadata.java
View File

@@ -101,11 +101,7 @@ public abstract class BaseTableMetadata implements HoodieTableMetadata {
try { try {
return fetchAllPartitionPaths(); return fetchAllPartitionPaths();
} catch (Exception e) { } catch (Exception e) {
if (metadataConfig.enableFallback()) { throw new HoodieMetadataException("Failed to retrieve list of partition from metadata", e);
LOG.error("Failed to retrieve list of partition from metadata", e);
} else {
throw new HoodieMetadataException("Failed to retrieve list of partition from metadata", e);
}
} }
} }
return new FileSystemBackedTableMetadata(getEngineContext(), hadoopConf, datasetBasePath, return new FileSystemBackedTableMetadata(getEngineContext(), hadoopConf, datasetBasePath,
@@ -129,11 +125,7 @@ public abstract class BaseTableMetadata implements HoodieTableMetadata {
try { try {
return fetchAllFilesInPartition(partitionPath); return fetchAllFilesInPartition(partitionPath);
} catch (Exception e) { } catch (Exception e) {
if (metadataConfig.enableFallback()) { throw new HoodieMetadataException("Failed to retrieve files in partition " + partitionPath + " from metadata", e);
LOG.error("Failed to retrieve files in partition " + partitionPath + " from metadata", e);
} else {
throw new HoodieMetadataException("Failed to retrieve files in partition " + partitionPath + " from metadata", e);
}
} }
} }
@@ -293,6 +285,7 @@ public abstract class BaseTableMetadata implements HoodieTableMetadata {
protected abstract List<HoodieInstant> findInstantsToSync(); protected abstract List<HoodieInstant> findInstantsToSync();
@Override
public boolean isInSync() { public boolean isInSync() {
return enabled && findInstantsToSync().isEmpty(); return enabled && findInstantsToSync().isEmpty();
} }