From aa546554ff23e62c374e323ec47d0081fb200227 Mon Sep 17 00:00:00 2001 From: Carl-Zhou-CN <67902676+Carl-Zhou-CN@users.noreply.github.com> Date: Sun, 26 Sep 2021 21:51:27 +0800 Subject: [PATCH] [HUDI-2451] On windows client with hdfs server for wrong file separator (#3687) Co-authored-by: yao.zhou --- .../org/apache/hudi/cli/commands/ExportCommand.java | 2 +- .../hudi/cli/commands/TestHoodieLogFileCommand.java | 3 +-- .../apache/hudi/cli/commands/TestTableCommand.java | 4 ++-- .../hudi/cli/integ/ITTestBootstrapCommand.java | 8 ++++---- .../cli/integ/ITTestHDFSParquetImportCommand.java | 5 ++--- .../hudi/cli/integ/ITTestSavepointsCommand.java | 4 ++-- .../hudi/client/heartbeat/HeartbeatUtils.java | 3 +-- .../client/heartbeat/HoodieHeartbeatClient.java | 7 +++---- .../hudi/common/table/HoodieTableMetaClient.java | 13 ++++++------- .../main/java/org/apache/hudi/source/FileIndex.java | 3 +-- .../org/apache/hudi/table/format/FilePathUtils.java | 7 +++---- .../test/java/org/apache/hudi/utils/TestUtils.java | 4 +--- 12 files changed, 27 insertions(+), 36 deletions(-) diff --git a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java index 8bd842c82..119ccb0dc 100644 --- a/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java +++ b/hudi-cli/src/main/java/org/apache/hudi/cli/commands/ExportCommand.java @@ -181,7 +181,7 @@ public class ExportCommand implements CommandMarker { final HoodieTableMetaClient metaClient = HoodieCLI.getTableMetaClient(); final HoodieActiveTimeline timeline = metaClient.getActiveTimeline(); for (HoodieInstant instant : instants) { - String localPath = localFolder + File.separator + instant.getFileName(); + String localPath = localFolder + Path.SEPARATOR + instant.getFileName(); byte[] data = null; switch (instant.getAction()) { diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java index 6b2bec4ef..f2571ce35 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestHoodieLogFileCommand.java @@ -53,7 +53,6 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; -import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; @@ -174,7 +173,7 @@ public class TestHoodieLogFileCommand extends CLIFunctionalTestHarness { // write to path '2015/03/16'. Schema schema = HoodieAvroUtils.addMetadataFields(getSimpleSchema()); - partitionPath = tablePath + File.separator + HoodieTestCommitMetadataGenerator.DEFAULT_SECOND_PARTITION_PATH; + partitionPath = tablePath + Path.SEPARATOR + HoodieTestCommitMetadataGenerator.DEFAULT_SECOND_PARTITION_PATH; Files.createDirectories(Paths.get(partitionPath)); HoodieLogFormat.Writer writer = null; diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java index 83deb34bf..08cdb7dc4 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestTableCommand.java @@ -142,7 +142,7 @@ public class TestTableCommand extends CLIFunctionalTestHarness { assertTrue(cr.isSuccess()); assertEquals("Metadata for table " + tableName + " loaded", cr.getResult().toString()); HoodieTableMetaClient client = HoodieCLI.getTableMetaClient(); - assertEquals(metaPath + File.separator + "archive", client.getArchivePath()); + assertEquals(metaPath + Path.SEPARATOR + "archive", client.getArchivePath()); assertEquals(tablePath, client.getBasePath()); assertEquals(metaPath, client.getMetaPath()); assertEquals(HoodieTableType.MERGE_ON_READ, client.getTableType()); @@ -181,7 +181,7 @@ public class TestTableCommand extends CLIFunctionalTestHarness { private void testRefreshCommand(String command) throws IOException { // clean table matedata FileSystem fs = FileSystem.get(hadoopConf()); - fs.delete(new Path(tablePath + File.separator + HoodieTableMetaClient.METAFOLDER_NAME), true); + fs.delete(new Path(tablePath + Path.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME), true); // Create table assertTrue(prepareTable()); diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestBootstrapCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestBootstrapCommand.java index c9ddd741a..6b3e30455 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestBootstrapCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestBootstrapCommand.java @@ -18,6 +18,7 @@ package org.apache.hudi.cli.integ; +import org.apache.hadoop.fs.Path; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.HoodiePrintHelper; import org.apache.hudi.cli.commands.TableCommand; @@ -32,7 +33,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; -import java.io.File; import java.io.IOException; import java.time.Instant; import java.util.Arrays; @@ -59,8 +59,8 @@ public class ITTestBootstrapCommand extends AbstractShellIntegrationTest { public void init() { String srcName = "source"; tableName = "test-table"; - sourcePath = basePath + File.separator + srcName; - tablePath = basePath + File.separator + tableName; + sourcePath = basePath + Path.SEPARATOR + srcName; + tablePath = basePath + Path.SEPARATOR + tableName; // generate test data partitions = Arrays.asList("2018", "2019", "2020"); @@ -68,7 +68,7 @@ public class ITTestBootstrapCommand extends AbstractShellIntegrationTest { for (int i = 0; i < partitions.size(); i++) { Dataset df = TestBootstrap.generateTestRawTripDataset(timestamp, i * NUM_OF_RECORDS, i * NUM_OF_RECORDS + NUM_OF_RECORDS, null, jsc, sqlContext); - df.write().parquet(sourcePath + File.separator + PARTITION_FIELD + "=" + partitions.get(i)); + df.write().parquet(sourcePath + Path.SEPARATOR + PARTITION_FIELD + "=" + partitions.get(i)); } } diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java index 53e543a5d..8cdc4c891 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestHDFSParquetImportCommand.java @@ -40,7 +40,6 @@ import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -70,7 +69,7 @@ public class ITTestHDFSParquetImportCommand extends AbstractShellIntegrationTest @BeforeEach public void init() throws IOException, ParseException { tableName = "test_table"; - tablePath = basePath + File.separator + tableName; + tablePath = basePath + Path.SEPARATOR + tableName; sourcePath = new Path(basePath, "source"); targetPath = new Path(tablePath); schemaFile = new Path(basePath, "file.schema").toString(); @@ -101,7 +100,7 @@ public class ITTestHDFSParquetImportCommand extends AbstractShellIntegrationTest () -> assertEquals("Table imported to hoodie format", cr.getResult().toString())); // Check hudi table exist - String metaPath = targetPath + File.separator + HoodieTableMetaClient.METAFOLDER_NAME; + String metaPath = targetPath + Path.SEPARATOR + HoodieTableMetaClient.METAFOLDER_NAME; assertTrue(Files.exists(Paths.get(metaPath)), "Hoodie table not exist."); // Load meta data diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java index e93323942..5f8021ab5 100644 --- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java +++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestSavepointsCommand.java @@ -18,6 +18,7 @@ package org.apache.hudi.cli.integ; +import org.apache.hadoop.fs.Path; import org.apache.hudi.cli.HoodieCLI; import org.apache.hudi.cli.commands.TableCommand; import org.apache.hudi.cli.testutils.AbstractShellIntegrationTest; @@ -32,7 +33,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.springframework.shell.core.CommandResult; -import java.io.File; import java.io.IOException; import static org.junit.jupiter.api.Assertions.assertAll; @@ -53,7 +53,7 @@ public class ITTestSavepointsCommand extends AbstractShellIntegrationTest { @BeforeEach public void init() throws IOException { String tableName = "test_table"; - tablePath = basePath + File.separator + tableName; + tablePath = basePath + Path.SEPARATOR + tableName; // Create table and connect new TableCommand().createTable( diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java index 2fe0fef8b..80191d4c3 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HeartbeatUtils.java @@ -29,7 +29,6 @@ import org.apache.hudi.table.HoodieTable; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import java.io.File; import java.io.IOException; import java.util.List; import java.util.Set; @@ -53,7 +52,7 @@ public class HeartbeatUtils { boolean deleted = false; try { String heartbeatFolderPath = HoodieTableMetaClient.getHeartbeatFolderPath(basePath); - deleted = fs.delete(new Path(heartbeatFolderPath + File.separator + instantTime), false); + deleted = fs.delete(new Path(heartbeatFolderPath + Path.SEPARATOR + instantTime), false); if (!deleted) { LOG.error("Failed to delete heartbeat for instant " + instantTime); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java index cb855c25d..341d72c75 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java @@ -29,7 +29,6 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import javax.annotation.concurrent.NotThreadSafe; -import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.Serializable; @@ -207,7 +206,7 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable { } public static Long getLastHeartbeatTime(FileSystem fs, String basePath, String instantTime) throws IOException { - Path heartbeatFilePath = new Path(HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + File.separator + instantTime); + Path heartbeatFilePath = new Path(HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + Path.SEPARATOR + instantTime); if (fs.exists(heartbeatFilePath)) { return fs.getFileStatus(heartbeatFilePath).getModificationTime(); } else { @@ -217,7 +216,7 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable { } public static Boolean heartbeatExists(FileSystem fs, String basePath, String instantTime) throws IOException { - Path heartbeatFilePath = new Path(HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + File.separator + instantTime); + Path heartbeatFilePath = new Path(HoodieTableMetaClient.getHeartbeatFolderPath(basePath) + Path.SEPARATOR + instantTime); if (fs.exists(heartbeatFilePath)) { return true; } @@ -255,7 +254,7 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable { try { Long newHeartbeatTime = System.currentTimeMillis(); OutputStream outputStream = - this.fs.create(new Path(heartbeatFolderPath + File.separator + instantTime), true); + this.fs.create(new Path(heartbeatFolderPath + Path.SEPARATOR + instantTime), true); outputStream.close(); Heartbeat heartbeat = instantToHeartbeatMap.get(instantTime); if (heartbeat.getLastHeartbeatTime() != null && isHeartbeatExpired(instantTime)) { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java index d5a3988fe..2b94d7ff0 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java @@ -49,7 +49,6 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; -import java.io.File; import java.io.IOException; import java.io.Serializable; import java.util.Arrays; @@ -76,10 +75,10 @@ public class HoodieTableMetaClient implements Serializable { private static final long serialVersionUID = 1L; private static final Logger LOG = LogManager.getLogger(HoodieTableMetaClient.class); public static final String METAFOLDER_NAME = ".hoodie"; - public static final String TEMPFOLDER_NAME = METAFOLDER_NAME + File.separator + ".temp"; - public static final String AUXILIARYFOLDER_NAME = METAFOLDER_NAME + File.separator + ".aux"; - public static final String BOOTSTRAP_INDEX_ROOT_FOLDER_PATH = AUXILIARYFOLDER_NAME + File.separator + ".bootstrap"; - public static final String HEARTBEAT_FOLDER_NAME = METAFOLDER_NAME + File.separator + ".heartbeat"; + public static final String TEMPFOLDER_NAME = METAFOLDER_NAME + Path.SEPARATOR + ".temp"; + public static final String AUXILIARYFOLDER_NAME = METAFOLDER_NAME + Path.SEPARATOR + ".aux"; + public static final String BOOTSTRAP_INDEX_ROOT_FOLDER_PATH = AUXILIARYFOLDER_NAME + Path.SEPARATOR + ".bootstrap"; + public static final String HEARTBEAT_FOLDER_NAME = METAFOLDER_NAME + Path.SEPARATOR + ".heartbeat"; public static final String BOOTSTRAP_INDEX_BY_PARTITION_FOLDER_PATH = BOOTSTRAP_INDEX_ROOT_FOLDER_PATH + Path.SEPARATOR + ".partitions"; public static final String BOOTSTRAP_INDEX_BY_FILE_ID_FOLDER_PATH = BOOTSTRAP_INDEX_ROOT_FOLDER_PATH + Path.SEPARATOR @@ -205,7 +204,7 @@ public class HoodieTableMetaClient implements Serializable { * @return Heartbeat folder path. */ public static String getHeartbeatFolderPath(String basePath) { - return String.format("%s%s%s", basePath, File.separator, HEARTBEAT_FOLDER_NAME); + return String.format("%s%s%s", basePath, Path.SEPARATOR, HEARTBEAT_FOLDER_NAME); } /** @@ -227,7 +226,7 @@ public class HoodieTableMetaClient implements Serializable { */ public String getArchivePath() { String archiveFolder = tableConfig.getArchivelogFolder(); - return getMetaPath() + "/" + archiveFolder; + return getMetaPath() + Path.SEPARATOR + archiveFolder; } /** diff --git a/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java b/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java index f1abf4b75..fc8ce6ac6 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java +++ b/hudi-flink/src/main/java/org/apache/hudi/source/FileIndex.java @@ -30,7 +30,6 @@ import org.apache.hadoop.fs.Path; import javax.annotation.Nullable; -import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -89,7 +88,7 @@ public class FileIndex { } List> partitions = new ArrayList<>(); for (String partitionPath : partitionPaths) { - String[] paths = partitionPath.split(File.separator); + String[] paths = partitionPath.split(Path.SEPARATOR); Map partitionMapping = new LinkedHashMap<>(); if (hivePartition) { Arrays.stream(paths).forEach(p -> { diff --git a/hudi-flink/src/main/java/org/apache/hudi/table/format/FilePathUtils.java b/hudi-flink/src/main/java/org/apache/hudi/table/format/FilePathUtils.java index 1eb7e2db3..523062590 100644 --- a/hudi-flink/src/main/java/org/apache/hudi/table/format/FilePathUtils.java +++ b/hudi-flink/src/main/java/org/apache/hudi/table/format/FilePathUtils.java @@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -83,7 +82,7 @@ public class FilePathUtils { * @param partitionKVs The partition key value mapping * @param hivePartition Whether the partition path is with Hive style, * e.g. {partition key} = {partition value} - * @param sepSuffix Whether to append the file separator as suffix + * @param sepSuffix Whether to append the path separator as suffix * @return an escaped, valid partition name */ public static String generatePartitionPath( @@ -97,7 +96,7 @@ public class FilePathUtils { int i = 0; for (Map.Entry e : partitionKVs.entrySet()) { if (i > 0) { - suffixBuf.append(File.separator); + suffixBuf.append(Path.SEPARATOR); } if (hivePartition) { suffixBuf.append(escapePathName(e.getKey())); @@ -107,7 +106,7 @@ public class FilePathUtils { i++; } if (sepSuffix) { - suffixBuf.append(File.separator); + suffixBuf.append(Path.SEPARATOR); } return suffixBuf.toString(); } diff --git a/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java b/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java index 3719705d6..14c9ac10f 100644 --- a/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java +++ b/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java @@ -27,8 +27,6 @@ import org.apache.hudi.util.StreamerUtil; import org.apache.flink.configuration.Configuration; import org.apache.flink.core.fs.Path; -import java.io.File; - import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -51,7 +49,7 @@ public class TestUtils { public static String getSplitPartitionPath(MergeOnReadInputSplit split) { assertTrue(split.getLogPaths().isPresent()); final String logPath = split.getLogPaths().get().get(0); - String[] paths = logPath.split(File.separator); + String[] paths = logPath.split(Path.SEPARATOR); return paths[paths.length - 2]; }