lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4495] Fix handling of S3 paths incompatible with java URI standards (#6237)

Browse Source
This commit is contained in:
Udit Mehrotra
2022-07-28 20:04:14 -07:00
committed by GitHub
parent cfd0c1ee34
commit c39e88dcf0
2 changed files with 22 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
hudi-common/src/main/java/org/apache/hudi/common/fs/HoodieWrapperFileSystem.java
View File

@@ -145,8 +145,11 @@ public class HoodieWrapperFileSystem extends FileSystem {
URI oldURI = oldPath.toUri(); URI oldURI = oldPath.toUri();
URI newURI; URI newURI;
try { try {
newURI = new URI(newScheme, oldURI.getUserInfo(), oldURI.getHost(), oldURI.getPort(), oldURI.getPath(), newURI = new URI(newScheme,
oldURI.getQuery(), oldURI.getFragment()); oldURI.getAuthority(),
oldURI.getPath(),
oldURI.getQuery(),
oldURI.getFragment());
return new CachingPath(newURI); return new CachingPath(newURI);
} catch (URISyntaxException e) { } catch (URISyntaxException e) {
// TODO - Better Exception handling // TODO - Better Exception handling

17
hudi-common/src/test/java/org/apache/hudi/common/fs/TestStorageSchemes.java
View File

@@ -18,8 +18,10 @@
package org.apache.hudi.common.fs; package org.apache.hudi.common.fs;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -54,4 +56,19 @@ public class TestStorageSchemes {
StorageSchemes.isAppendSupported("s2"); StorageSchemes.isAppendSupported("s2");
}, "Should throw exception for unsupported schemes"); }, "Should throw exception for unsupported schemes");
} }
@Test
public void testConversionToNewSchema() {
Path s3TablePath1 = new Path("s3://test.1234/table1");
assertEquals(s3TablePath1, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath1, "s3"));
Path s3TablePath2 = new Path("s3://1234.test/table1");
assertEquals(s3TablePath2, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath2, "s3"));
Path s3TablePath3 = new Path("s3://test1234/table1");
assertEquals(s3TablePath3, HoodieWrapperFileSystem.convertPathWithScheme(s3TablePath3, "s3"));
Path hdfsTablePath = new Path("hdfs://sandbox.foo.com:8020/test.1234/table1");
System.out.println(HoodieWrapperFileSystem.convertPathWithScheme(hdfsTablePath, "hdfs"));
}
} }