lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1420] HoodieTableMetaClient.getMarkerFolderPath works incorrectly on windows client with hdfs server for wrong file seperator (#2526)

Browse Source 
* Fix HUDI-1420

FIX https://issues.apache.org/jira/browse/HUDI-1420

* fix(hudi-common): fix HUDI-1420 HoodieTableMetaClient.getMarkerFolderPath works incorrectly on windows client with hdfs server for wrong file seperator

Co-authored-by: 谢波 <xiebo1@yonghui.cn>
This commit is contained in:
hiscat
2021-02-05 16:24:35 +08:00
committed by GitHub
parent 4a5683d54a
commit b51b3a39a8
1 changed files with 12 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
View File

@@ -45,7 +45,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.log4j.LogManager; import org.apache.log4j.LogManager;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.util.Arrays; import java.util.Arrays;
@@ -72,13 +71,13 @@ public class HoodieTableMetaClient implements Serializable {
private static final long serialVersionUID = 1L; private static final long serialVersionUID = 1L;
private static final Logger LOG = LogManager.getLogger(HoodieTableMetaClient.class); private static final Logger LOG = LogManager.getLogger(HoodieTableMetaClient.class);
public static final String METAFOLDER_NAME = ".hoodie"; public static final String METAFOLDER_NAME = ".hoodie";
public static final String TEMPFOLDER_NAME = METAFOLDER_NAME + File.separator + ".temp"; public static final String TEMPFOLDER_NAME = METAFOLDER_NAME + Path.SEPARATOR + ".temp";
public static final String AUXILIARYFOLDER_NAME = METAFOLDER_NAME + File.separator + ".aux"; public static final String AUXILIARYFOLDER_NAME = METAFOLDER_NAME + Path.SEPARATOR + ".aux";
public static final String BOOTSTRAP_INDEX_ROOT_FOLDER_PATH = AUXILIARYFOLDER_NAME + File.separator + ".bootstrap"; public static final String BOOTSTRAP_INDEX_ROOT_FOLDER_PATH = AUXILIARYFOLDER_NAME + Path.SEPARATOR + ".bootstrap";
public static final String BOOTSTRAP_INDEX_BY_PARTITION_FOLDER_PATH = BOOTSTRAP_INDEX_ROOT_FOLDER_PATH public static final String BOOTSTRAP_INDEX_BY_PARTITION_FOLDER_PATH = BOOTSTRAP_INDEX_ROOT_FOLDER_PATH
+ File.separator + ".partitions"; + Path.SEPARATOR + ".partitions";
public static final String BOOTSTRAP_INDEX_BY_FILE_ID_FOLDER_PATH = BOOTSTRAP_INDEX_ROOT_FOLDER_PATH + File.separator public static final String BOOTSTRAP_INDEX_BY_FILE_ID_FOLDER_PATH = BOOTSTRAP_INDEX_ROOT_FOLDER_PATH + Path.SEPARATOR
+ ".fileids"; + ".fileids";
public static final String MARKER_EXTN = ".marker"; public static final String MARKER_EXTN = ".marker";
@@ -118,10 +117,10 @@ public class HoodieTableMetaClient implements Serializable {
ConsistencyGuardConfig consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion, ConsistencyGuardConfig consistencyGuardConfig, Option<TimelineLayoutVersion> layoutVersion,
String payloadClassName) { String payloadClassName) {
LOG.info("Loading HoodieTableMetaClient from " + basePath); LOG.info("Loading HoodieTableMetaClient from " + basePath);
this.basePath = basePath;
this.consistencyGuardConfig = consistencyGuardConfig; this.consistencyGuardConfig = consistencyGuardConfig;
this.hadoopConf = new SerializableConfiguration(conf); this.hadoopConf = new SerializableConfiguration(conf);
Path basePathDir = new Path(this.basePath); Path basePathDir = new Path(basePath);
this.basePath = basePathDir.toString();
this.metaPath = new Path(basePath, METAFOLDER_NAME).toString(); this.metaPath = new Path(basePath, METAFOLDER_NAME).toString();
Path metaPathDir = new Path(this.metaPath); Path metaPathDir = new Path(this.metaPath);
this.fs = getFs(); this.fs = getFs();
@@ -197,7 +196,7 @@ public class HoodieTableMetaClient implements Serializable {
* @return Temp Folder path * @return Temp Folder path
*/ */
public String getTempFolderPath() { public String getTempFolderPath() {
return basePath + File.separator + TEMPFOLDER_NAME; return basePath + Path.SEPARATOR + TEMPFOLDER_NAME;
} }
/** /**
@@ -207,28 +206,28 @@ public class HoodieTableMetaClient implements Serializable {
* @return * @return
*/ */
public String getMarkerFolderPath(String instantTs) { public String getMarkerFolderPath(String instantTs) {
return String.format("%s%s%s", getTempFolderPath(), File.separator, instantTs); return String.format("%s%s%s", getTempFolderPath(), Path.SEPARATOR, instantTs);
} }
/** /**
* @return Auxiliary Meta path * @return Auxiliary Meta path
*/ */
public String getMetaAuxiliaryPath() { public String getMetaAuxiliaryPath() {
return basePath + File.separator + AUXILIARYFOLDER_NAME; return basePath + Path.SEPARATOR + AUXILIARYFOLDER_NAME;
} }
/** /**
* @return Bootstrap Index By Partition Folder * @return Bootstrap Index By Partition Folder
*/ */
public String getBootstrapIndexByPartitionFolderPath() { public String getBootstrapIndexByPartitionFolderPath() {
return basePath + File.separator + BOOTSTRAP_INDEX_BY_PARTITION_FOLDER_PATH; return basePath + Path.SEPARATOR + BOOTSTRAP_INDEX_BY_PARTITION_FOLDER_PATH;
} }
/** /**
* @return Bootstrap Index By Hudi File Id Folder * @return Bootstrap Index By Hudi File Id Folder
*/ */
public String getBootstrapIndexByFileIdFolderNameFolderPath() { public String getBootstrapIndexByFileIdFolderNameFolderPath() {
return basePath + File.separator + BOOTSTRAP_INDEX_BY_FILE_ID_FOLDER_PATH; return basePath + Path.SEPARATOR + BOOTSTRAP_INDEX_BY_FILE_ID_FOLDER_PATH;
} }
/** /**