diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java index ed14a64fc..ca8cb19f3 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestRepairsCommand.java @@ -25,6 +25,7 @@ import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; import org.apache.hudi.cli.testutils.HoodieTestCommitMetadataGenerator; import org.apache.hudi.common.fs.FSUtils; import org.apache.hudi.common.model.HoodieTableType; +import org.apache.hudi.common.table.HoodieTableConfig; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.testutils.HoodieTestDataGenerator; @@ -66,7 +67,7 @@ public class TestRepairsCommand extends AbstractShellIntegrationTest { // Create table and connect new TableCommand().createTable( tablePath, tableName, HoodieTableType.COPY_ON_WRITE.name(), - "", TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); + HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER, TimelineLayoutVersion.VERSION_1, "org.apache.hudi.common.model.HoodieAvroPayload"); } /** diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 301d9bd1d..38a044799 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -67,6 +67,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import static org.apache.hudi.common.table.HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER; import static org.apache.hudi.metadata.HoodieTableMetadata.METADATA_TABLE_NAME_SUFFIX; import static org.apache.hudi.metadata.HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP; @@ -294,7 +295,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta HoodieTableMetaClient.withPropertyBuilder() .setTableType(HoodieTableType.MERGE_ON_READ) .setTableName(tableName) - .setArchiveLogFolder("archived") + .setArchiveLogFolder(DEFAULT_ARCHIVELOG_FOLDER) .setPayloadClassName(HoodieMetadataPayload.class.getName()) .setBaseFileFormat(HoodieFileFormat.HFILE.toString()) .initTable(hadoopConf.get(), metadataWriteConfig.getBasePath()); diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java index 6d5b248b1..35506cb23 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java @@ -81,7 +81,7 @@ public class HoodieTableConfig implements Serializable { public static final String DEFAULT_PAYLOAD_CLASS = OverwriteWithLatestAvroPayload.class.getName(); public static final String NO_OP_BOOTSTRAP_INDEX_CLASS = NoOpBootstrapIndex.class.getName(); public static final String DEFAULT_BOOTSTRAP_INDEX_CLASS = HFileBootstrapIndex.class.getName(); - public static final String DEFAULT_ARCHIVELOG_FOLDER = ""; + public static final String DEFAULT_ARCHIVELOG_FOLDER = "archived"; private Properties props; diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index f4edeb81b..14f337f4f 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -34,6 +34,7 @@ import org.apache.hudi.common.table.timeline.TimelineLayout; import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion; import org.apache.hudi.common.util.CommitUtils; import org.apache.hudi.common.util.Option; +import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.exception.HoodieException; import org.apache.hudi.exception.TableNotFoundException; @@ -331,7 +332,7 @@ public class HoodieTableMetaClient implements Serializable { // if anything other than default archive log folder is specified, create that too String archiveLogPropVal = props.getProperty(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER); - if (!archiveLogPropVal.equals(HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER)) { + if (!StringUtils.isNullOrEmpty(archiveLogPropVal)) { Path archiveLogDir = new Path(metaPathDir, archiveLogPropVal); if (!fs.exists(archiveLogDir)) { fs.mkdirs(archiveLogDir); @@ -721,8 +722,10 @@ public class HoodieTableMetaClient implements Serializable { properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, payloadClassName); } - if (null != archiveLogFolder) { + if (!StringUtils.isNullOrEmpty(archiveLogFolder)) { properties.put(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, archiveLogFolder); + } else { + properties.setProperty(HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER, HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER); } if (null != timelineLayoutVersion) { diff --git a/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java b/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java index e585c72c6..0a1b0a5bf 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java +++ b/hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java @@ -68,11 +68,12 @@ import java.util.Locale; import java.util.Objects; import java.util.Properties; +import static org.apache.hudi.common.table.HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER; + /** * Utilities for Flink stream read and write. */ public class StreamerUtil { - private static final String DEFAULT_ARCHIVE_LOG_FOLDER = "archived"; private static final Logger LOG = LoggerFactory.getLogger(StreamerUtil.class); @@ -266,7 +267,7 @@ public class StreamerUtil { .setTableType(conf.getString(FlinkOptions.TABLE_TYPE)) .setTableName(conf.getString(FlinkOptions.TABLE_NAME)) .setPayloadClassName(conf.getString(FlinkOptions.PAYLOAD_CLASS)) - .setArchiveLogFolder(DEFAULT_ARCHIVE_LOG_FOLDER) + .setArchiveLogFolder(DEFAULT_ARCHIVELOG_FOLDER) .setTimelineLayoutVersion(1) .initTable(hadoopConf, basePath); LOG.info("Table initialized under base path {}", basePath); diff --git a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java index 42bbfbfa6..c69607ca8 100644 --- a/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java +++ b/hudi-integ-test/src/main/java/org/apache/hudi/integ/testsuite/HoodieTestSuiteJob.java @@ -61,6 +61,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import static org.apache.hudi.common.table.HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER; + /** * This is the entry point for running a Hudi Test Suite. Although this class has similarities with {@link HoodieDeltaStreamer} this class does not extend it since do not want to create a dependency * on the changes in DeltaStreamer. @@ -109,7 +111,7 @@ public class HoodieTestSuiteJob { metaClient = HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder("archived") + .setArchiveLogFolder(DEFAULT_ARCHIVELOG_FOLDER) .initTable(jsc.hadoopConfiguration(), cfg.targetBasePath); if (cfg.cleanInput) { diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala index 93442d5cd..abb5f7644 100644 --- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala +++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala @@ -52,6 +52,7 @@ import org.apache.spark.sql.{DataFrame, SQLContext, SaveMode, SparkSession} import scala.collection.JavaConversions._ import scala.collection.mutable.ListBuffer +import org.apache.hudi.common.table.HoodieTableConfig.{DEFAULT_ARCHIVELOG_FOLDER, HOODIE_ARCHIVELOG_FOLDER_PROP_NAME} private[hudi] object HoodieSparkSqlWriter { @@ -114,7 +115,7 @@ private[hudi] object HoodieSparkSqlWriter { // Create the table if not present if (!tableExists) { val archiveLogFolder = parameters.getOrElse( - HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, "archived") + HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, DEFAULT_ARCHIVELOG_FOLDER) val partitionColumns = HoodieWriterUtils.getPartitionColumns(keyGenerator) @@ -288,7 +289,7 @@ private[hudi] object HoodieSparkSqlWriter { if (!tableExists) { val archiveLogFolder = parameters.getOrElse( - HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, "archived") + HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, DEFAULT_ARCHIVELOG_FOLDER) val partitionColumns = HoodieWriterUtils.getPartitionColumns(parameters) HoodieTableMetaClient.withPropertyBuilder() .setTableType(HoodieTableType.valueOf(tableType)) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java index bcf40259f..ecc1769fd 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/BootstrapExecutor.java @@ -49,6 +49,8 @@ import java.io.IOException; import java.io.Serializable; import java.util.HashMap; +import static org.apache.hudi.common.table.HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER; + /** * Performs bootstrap from a non-hudi source. */ @@ -172,7 +174,7 @@ public class BootstrapExecutor implements Serializable { HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder("archived") + .setArchiveLogFolder(DEFAULT_ARCHIVELOG_FOLDER) .setPayloadClassName(cfg.payloadClassName) .setBaseFileFormat(cfg.baseFileFormat) .setBootstrapIndexClass(cfg.bootstrapIndexClass) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java index 37586ac2c..fbe1c14e9 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/DeltaSync.java @@ -89,6 +89,7 @@ import java.util.stream.Collectors; import scala.collection.JavaConversions; +import static org.apache.hudi.common.table.HoodieTableConfig.DEFAULT_ARCHIVELOG_FOLDER; import static org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.CHECKPOINT_KEY; import static org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.CHECKPOINT_RESET_KEY; import static org.apache.hudi.config.HoodieCompactionConfig.INLINE_COMPACT_PROP; @@ -241,7 +242,7 @@ public class DeltaSync implements Serializable { HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder("archived") + .setArchiveLogFolder(DEFAULT_ARCHIVELOG_FOLDER) .setPayloadClassName(cfg.payloadClassName) .setBaseFileFormat(cfg.baseFileFormat) .setPartitionColumns(partitionColumns) @@ -334,7 +335,7 @@ public class DeltaSync implements Serializable { HoodieTableMetaClient.withPropertyBuilder() .setTableType(cfg.tableType) .setTableName(cfg.targetTableName) - .setArchiveLogFolder("archived") + .setArchiveLogFolder(DEFAULT_ARCHIVELOG_FOLDER) .setPayloadClassName(cfg.payloadClassName) .setBaseFileFormat(cfg.baseFileFormat) .setPartitionColumns(partitionColumns)