diff --git a/hoodie-client/src/test/java/HoodieClientExample.java b/hoodie-client/src/test/java/HoodieClientExample.java index 14c1e6eac..166bd22de 100644 --- a/hoodie-client/src/test/java/HoodieClientExample.java +++ b/hoodie-client/src/test/java/HoodieClientExample.java @@ -85,7 +85,7 @@ public class HoodieClientExample { FileSystem fs = FSUtils.getFs(tablePath, jsc.hadoopConfiguration()); if (!fs.exists(path)) { HoodieTableMetaClient - .initTableType(fs, tablePath, HoodieTableType.valueOf(tableType), tableName, + .initTableType(jsc.hadoopConfiguration(), tablePath, HoodieTableType.valueOf(tableType), tableName, HoodieAvroPayload.class.getName()); } diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java index 1bf9ca506..60a6a1db2 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java @@ -105,7 +105,7 @@ public class TestHoodieClientOnCopyOnWriteStorage implements Serializable { folder.create(); basePath = folder.getRoot().getAbsolutePath(); fs = FSUtils.getFs(basePath.toString(), jsc.hadoopConfiguration()); - HoodieTestUtils.init(fs, basePath); + HoodieTestUtils.init(jsc.hadoopConfiguration(), basePath); dataGen = new HoodieTestDataGenerator(); } @@ -1258,7 +1258,7 @@ public class TestHoodieClientOnCopyOnWriteStorage implements Serializable { .retainFileVersions(1).build()).build(); HoodieTableMetaClient metaClient = HoodieTestUtils - .initTableType(fs, basePath, HoodieTableType.MERGE_ON_READ); + .initTableType(jsc.hadoopConfiguration(), basePath, HoodieTableType.MERGE_ON_READ); // Make 3 files, one base file and 2 log files associated with base file String file1P0 = HoodieTestUtils.createNewDataFile(basePath, partitionPaths[0], "000"); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java index 676b970d9..0dc7a816f 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java @@ -103,9 +103,8 @@ public class TestMultiFS implements Serializable { HoodieTestDataGenerator dataGen = new HoodieTestDataGenerator(); // Initialize table and filesystem - FileSystem hdfs = FSUtils.getFs(dfsBasePath, jsc.hadoopConfiguration()); HoodieTableMetaClient - .initTableType(hdfs, dfsBasePath, HoodieTableType.valueOf(tableType), tableName, + .initTableType(jsc.hadoopConfiguration(), dfsBasePath, HoodieTableType.valueOf(tableType), tableName, HoodieAvroPayload.class.getName()); //Create write client to write some records in @@ -133,9 +132,8 @@ public class TestMultiFS implements Serializable { assertEquals("Should contain 100 records", readRecords.count(), records.size()); // Write to local - FileSystem local = FSUtils.getFs(tablePath, jsc.hadoopConfiguration()); HoodieTableMetaClient - .initTableType(local, tablePath, HoodieTableType.valueOf(tableType), tableName, + .initTableType(jsc.hadoopConfiguration(), tablePath, HoodieTableType.valueOf(tableType), tableName, HoodieAvroPayload.class.getName()); HoodieWriteConfig localConfig = HoodieWriteConfig.newBuilder().withPath(tablePath) .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) diff --git a/hoodie-client/src/test/java/com/uber/hoodie/func/TestUpdateMapFunction.java b/hoodie-client/src/test/java/com/uber/hoodie/func/TestUpdateMapFunction.java index 352a0036b..0316372ca 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/func/TestUpdateMapFunction.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/func/TestUpdateMapFunction.java @@ -48,7 +48,7 @@ public class TestUpdateMapFunction { TemporaryFolder folder = new TemporaryFolder(); folder.create(); this.basePath = folder.getRoot().getAbsolutePath(); - HoodieTestUtils.init(FSUtils.getFs(basePath, HoodieTestUtils.getDefaultHadoopConf()), basePath); + HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath); } @Test diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java b/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java index 990e62d61..89f31fa2e 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/index/TestHbaseIndex.java @@ -112,7 +112,7 @@ public class TestHbaseIndex { basePath = folder.getRoot().getAbsolutePath(); // Initialize table metaClient = HoodieTableMetaClient - .initTableType(utility.getTestFileSystem(), basePath, HoodieTableType.COPY_ON_WRITE, + .initTableType(utility.getConfiguration(), basePath, HoodieTableType.COPY_ON_WRITE, tableName, HoodieTableConfig.DEFAULT_PAYLOAD_CLASS); } diff --git a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieBloomIndex.java b/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieBloomIndex.java index c3ace401e..c0b7f4a27 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieBloomIndex.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/index/bloom/TestHoodieBloomIndex.java @@ -88,7 +88,7 @@ public class TestHoodieBloomIndex { folder.create(); basePath = folder.getRoot().getAbsolutePath(); fs = FSUtils.getFs(basePath, jsc.hadoopConfiguration()); - HoodieTestUtils.init(fs, basePath); + HoodieTestUtils.init(jsc.hadoopConfiguration(), basePath); // We have some records to be tagged (two different partitions) schemaStr = IOUtils.toString(getClass().getResourceAsStream("/exampleSchema.txt"), "UTF-8"); schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(schemaStr)); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java index 1e5d015d5..d171a80d9 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCommitArchiveLog.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Before; @@ -49,14 +50,16 @@ public class TestHoodieCommitArchiveLog { private String basePath; private FileSystem fs; + private Configuration hadoopConf; @Before public void init() throws Exception { TemporaryFolder folder = new TemporaryFolder(); folder.create(); basePath = folder.getRoot().getAbsolutePath(); - fs = FSUtils.getFs(basePath, HoodieTestUtils.getDefaultHadoopConf()); - HoodieTestUtils.init(fs, basePath); + hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); + fs = FSUtils.getFs(basePath, hadoopConf); + HoodieTestUtils.init(hadoopConf, basePath); } @Test @@ -76,7 +79,7 @@ public class TestHoodieCommitArchiveLog { .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) .withCompactionConfig(HoodieCompactionConfig.newBuilder().archiveCommitsWith(2, 4).build()) .forTable("test-trip-table").build(); - HoodieTestUtils.init(fs, basePath); + HoodieTestUtils.init(hadoopConf, basePath); HoodieTestDataGenerator.createCommitFile(basePath, "100"); HoodieTestDataGenerator.createCommitFile(basePath, "101"); HoodieTestDataGenerator.createCommitFile(basePath, "102"); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java index c1dd84034..5a1ff7ba8 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/io/TestHoodieCompactor.java @@ -43,6 +43,7 @@ import java.io.File; import java.io.IOException; import java.util.List; import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; @@ -58,6 +59,7 @@ public class TestHoodieCompactor { private HoodieCompactor compactor; private transient HoodieTestDataGenerator dataGen = null; private transient FileSystem fs; + private Configuration hadoopConf; @Before public void init() throws IOException { @@ -68,8 +70,9 @@ public class TestHoodieCompactor { TemporaryFolder folder = new TemporaryFolder(); folder.create(); basePath = folder.getRoot().getAbsolutePath(); - fs = FSUtils.getFs(basePath, HoodieTestUtils.getDefaultHadoopConf()); - HoodieTestUtils.initTableType(fs, basePath, HoodieTableType.MERGE_ON_READ); + hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); + fs = FSUtils.getFs(basePath, hadoopConf); + HoodieTestUtils.initTableType(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ); dataGen = new HoodieTestDataGenerator(); compactor = new HoodieRealtimeTableCompactor(); @@ -102,7 +105,7 @@ public class TestHoodieCompactor { @Test(expected = IllegalArgumentException.class) public void testCompactionOnCopyOnWriteFail() throws Exception { - HoodieTestUtils.initTableType(fs, basePath, HoodieTableType.COPY_ON_WRITE); + HoodieTestUtils.initTableType(hadoopConf, basePath, HoodieTableType.COPY_ON_WRITE); HoodieTableMetaClient metaClient = new HoodieTableMetaClient(jsc.hadoopConfiguration(), basePath); HoodieTable table = HoodieTable.getHoodieTable(metaClient, getConfig()); diff --git a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java b/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java index 9432be72d..ab9ba8be4 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/table/TestCopyOnWriteTable.java @@ -76,7 +76,7 @@ public class TestCopyOnWriteTable { TemporaryFolder folder = new TemporaryFolder(); folder.create(); this.basePath = folder.getRoot().getAbsolutePath(); - HoodieTestUtils.init(FSUtils.getFs(basePath, jsc.hadoopConfiguration()), basePath); + HoodieTestUtils.init(jsc.hadoopConfiguration(), basePath); } @Test diff --git a/hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java b/hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java index 3de5654d8..23f201b45 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/table/TestMergeOnReadTable.java @@ -121,7 +121,7 @@ public class TestMergeOnReadTable { jsc.hadoopConfiguration().addResource(dfs.getConf()); dfs.mkdirs(new Path(basePath)); - HoodieTestUtils.initTableType(dfs, basePath, HoodieTableType.MERGE_ON_READ); + HoodieTestUtils.initTableType(jsc.hadoopConfiguration(), basePath, HoodieTableType.MERGE_ON_READ); sqlContext = new SQLContext(jsc); // SQLContext stuff } @@ -346,7 +346,7 @@ public class TestMergeOnReadTable { public void testCOWToMORConvertedDatasetRollback() throws Exception { //Set TableType to COW - HoodieTestUtils.initTableType(dfs, basePath, HoodieTableType.COPY_ON_WRITE); + HoodieTestUtils.initTableType(jsc.hadoopConfiguration(), basePath, HoodieTableType.COPY_ON_WRITE); HoodieWriteConfig cfg = getConfig(true); HoodieWriteClient client = new HoodieWriteClient(jsc, cfg); @@ -385,7 +385,7 @@ public class TestMergeOnReadTable { assertNoWriteErrors(statuses); //Set TableType to MOR - HoodieTestUtils.initTableType(dfs, basePath, HoodieTableType.MERGE_ON_READ); + HoodieTestUtils.initTableType(jsc.hadoopConfiguration(), basePath, HoodieTableType.MERGE_ON_READ); //rollback a COW commit when TableType is MOR client.rollback(newCommitTime); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java index 4c5168f60..1388823af 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTableMetaClient.java @@ -193,7 +193,7 @@ public class HoodieTableMetaClient implements Serializable { /** * Helper method to initialize a given path, as a given storage type and table name */ - public static HoodieTableMetaClient initTableType(FileSystem fs, String basePath, + public static HoodieTableMetaClient initTableType(Configuration hadoopConf, String basePath, HoodieTableType tableType, String tableName, String payloadClassName) throws IOException { Properties properties = new Properties(); properties.setProperty(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tableName); @@ -201,7 +201,7 @@ public class HoodieTableMetaClient implements Serializable { if (tableType == HoodieTableType.MERGE_ON_READ) { properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, payloadClassName); } - return HoodieTableMetaClient.initializePathAsHoodieDataset(fs, basePath, properties); + return HoodieTableMetaClient.initializePathAsHoodieDataset(hadoopConf, basePath, properties); } /** @@ -210,10 +210,11 @@ public class HoodieTableMetaClient implements Serializable { * * @return Instance of HoodieTableMetaClient */ - public static HoodieTableMetaClient initializePathAsHoodieDataset(FileSystem fs, + public static HoodieTableMetaClient initializePathAsHoodieDataset(Configuration hadoopConf, String basePath, Properties props) throws IOException { log.info("Initializing " + basePath + " as hoodie dataset " + basePath); Path basePathDir = new Path(basePath); + final FileSystem fs = FSUtils.getFs(basePath, hadoopConf); if (!fs.exists(basePathDir)) { fs.mkdirs(basePathDir); } @@ -239,7 +240,9 @@ public class HoodieTableMetaClient implements Serializable { fs.mkdirs(temporaryFolder); } HoodieTableConfig.createHoodieProperties(fs, metaPathDir, props); - HoodieTableMetaClient metaClient = new HoodieTableMetaClient(fs.getConf(), basePath); + // We should not use fs.getConf as this might be different from the original configuration + // used to create the fs in unit tests + HoodieTableMetaClient metaClient = new HoodieTableMetaClient(hadoopConf, basePath); log.info("Finished initializing Table of type " + metaClient.getTableConfig().getTableType() + " from " + basePath); return metaClient; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java b/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java index 4db0a58b7..bd698e7c8 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java +++ b/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java @@ -80,11 +80,12 @@ public class HoodieTestUtils { return new Configuration(); } - public static HoodieTableMetaClient init(FileSystem fs, String basePath) throws IOException { - return initTableType(fs, basePath, HoodieTableType.COPY_ON_WRITE); + public static HoodieTableMetaClient init(Configuration hadoopConf, String basePath) + throws IOException { + return initTableType(hadoopConf, basePath, HoodieTableType.COPY_ON_WRITE); } - public static HoodieTableMetaClient initTableType(FileSystem fs, String basePath, + public static HoodieTableMetaClient initTableType(Configuration hadoopConf, String basePath, HoodieTableType tableType) throws IOException { Properties properties = new Properties(); @@ -92,7 +93,7 @@ public class HoodieTestUtils { properties.setProperty(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, tableType.name()); properties.setProperty(HoodieTableConfig.HOODIE_PAYLOAD_CLASS_PROP_NAME, HoodieAvroPayload.class.getName()); - return HoodieTableMetaClient.initializePathAsHoodieDataset(fs, basePath, properties); + return HoodieTableMetaClient.initializePathAsHoodieDataset(hadoopConf, basePath, properties); } public static HoodieTableMetaClient initOnTemp() throws IOException { @@ -100,8 +101,7 @@ public class HoodieTestUtils { TemporaryFolder folder = new TemporaryFolder(); folder.create(); String basePath = folder.getRoot().getAbsolutePath(); - return HoodieTestUtils - .init(FSUtils.getFs(basePath, HoodieTestUtils.getDefaultHadoopConf()), basePath); + return HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath); } public static String makeNewCommitTime() { diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatTest.java b/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatTest.java index 0acd77fb7..3ef9f23bf 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatTest.java +++ b/hoodie-common/src/test/java/com/uber/hoodie/common/table/log/HoodieLogFormatTest.java @@ -109,7 +109,7 @@ public class HoodieLogFormatTest { assertTrue(fs.mkdirs(new Path(folder.getRoot().getPath()))); this.partitionPath = new Path(folder.getRoot().getPath()); this.basePath = folder.getRoot().getParent(); - HoodieTestUtils.initTableType(fs, basePath, HoodieTableType.MERGE_ON_READ); + HoodieTestUtils.initTableType(MiniClusterUtil.configuration, basePath, HoodieTableType.MERGE_ON_READ); } @After diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/InputFormatTestUtil.java b/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/InputFormatTestUtil.java index 4b060a132..c8305e3a3 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/InputFormatTestUtil.java +++ b/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/InputFormatTestUtil.java @@ -39,9 +39,7 @@ public class InputFormatTestUtil { public static File prepareDataset(TemporaryFolder basePath, int numberOfFiles, String commitNumber) throws IOException { basePath.create(); - HoodieTestUtils - .init(FSUtils.getFs(basePath.getRoot().toString(), HoodieTestUtils.getDefaultHadoopConf()), - basePath.getRoot().toString()); + HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.getRoot().toString()); File partitionPath = basePath.newFolder("2016", "05", "01"); for (int i = 0; i < numberOfFiles; i++) { File dataFile = @@ -101,9 +99,7 @@ public class InputFormatTestUtil { int numberOfFiles, int numberOfRecords, String commitNumber) throws IOException { basePath.create(); - HoodieTestUtils - .init(FSUtils.getFs(basePath.getRoot().toString(), HoodieTestUtils.getDefaultHadoopConf()), - basePath.getRoot().toString()); + HoodieTestUtils.init(HoodieTestUtils.getDefaultHadoopConf(), basePath.getRoot().toString()); File partitionPath = basePath.newFolder("2016", "05", "01"); AvroParquetWriter parquetWriter; for (int i = 0; i < numberOfFiles; i++) { diff --git a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReaderTest.java b/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReaderTest.java index 12c010e98..653e122d3 100644 --- a/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReaderTest.java +++ b/hoodie-hadoop-mr/src/test/java/com/uber/hoodie/hadoop/realtime/HoodieRealtimeRecordReaderTest.java @@ -41,6 +41,7 @@ import java.util.Map; import java.util.stream.Collectors; import org.apache.avro.Schema; import org.apache.avro.generic.IndexedRecord; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat; @@ -66,12 +67,13 @@ public class HoodieRealtimeRecordReaderTest { private JobConf jobConf; private FileSystem fs; + private Configuration hadoopConf; @Before public void setUp() { jobConf = new JobConf(); - fs = FSUtils - .getFs(basePath.getRoot().getAbsolutePath(), HoodieTestUtils.getDefaultHadoopConf()); + hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); + fs = FSUtils.getFs(basePath.getRoot().getAbsolutePath(), hadoopConf); } @Rule @@ -105,7 +107,7 @@ public class HoodieRealtimeRecordReaderTest { // initial commit Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getEvolvedSchema()); HoodieTestUtils - .initTableType(fs, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); + .initTableType(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); String commitTime = "100"; File partitionDir = InputFormatTestUtil .prepareParquetDataset(basePath, schema, 1, 100, commitTime); @@ -163,7 +165,7 @@ public class HoodieRealtimeRecordReaderTest { // initial commit Schema schema = HoodieAvroUtils.addMetadataFields(SchemaTestUtil.getComplexEvolvedSchema()); HoodieTestUtils - .initTableType(fs, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); + .initTableType(hadoopConf, basePath.getRoot().getAbsolutePath(), HoodieTableType.MERGE_ON_READ); String commitTime = "100"; int numberOfRecords = 100; int numberOfLogRecords = numberOfRecords / 2; diff --git a/hoodie-hive/src/test/java/com/uber/hoodie/hive/TestUtil.java b/hoodie-hive/src/test/java/com/uber/hoodie/hive/TestUtil.java index 10b8c11f8..ff260f893 100644 --- a/hoodie-hive/src/test/java/com/uber/hoodie/hive/TestUtil.java +++ b/hoodie-hive/src/test/java/com/uber/hoodie/hive/TestUtil.java @@ -118,7 +118,7 @@ public class TestUtil { static void clear() throws IOException { fileSystem.delete(new Path(hiveSyncConfig.basePath), true); HoodieTableMetaClient - .initTableType(fileSystem, hiveSyncConfig.basePath, HoodieTableType.COPY_ON_WRITE, + .initTableType(configuration, hiveSyncConfig.basePath, HoodieTableType.COPY_ON_WRITE, hiveSyncConfig.tableName, HoodieAvroPayload.class.getName()); HoodieHiveClient client = new HoodieHiveClient(hiveSyncConfig, hiveServer.getHiveConf(), @@ -154,7 +154,7 @@ public class TestUtil { Path path = new Path(hiveSyncConfig.basePath); FileUtils.deleteDirectory(new File(hiveSyncConfig.basePath)); HoodieTableMetaClient - .initTableType(fileSystem, hiveSyncConfig.basePath, HoodieTableType.COPY_ON_WRITE, + .initTableType(configuration, hiveSyncConfig.basePath, HoodieTableType.COPY_ON_WRITE, hiveSyncConfig.tableName, HoodieAvroPayload.class.getName()); boolean result = fileSystem.mkdirs(path); checkResult(result); @@ -170,7 +170,7 @@ public class TestUtil { Path path = new Path(hiveSyncConfig.basePath); FileUtils.deleteDirectory(new File(hiveSyncConfig.basePath)); HoodieTableMetaClient - .initTableType(fileSystem, hiveSyncConfig.basePath, HoodieTableType.MERGE_ON_READ, + .initTableType(configuration, hiveSyncConfig.basePath, HoodieTableType.MERGE_ON_READ, hiveSyncConfig.tableName, HoodieAvroPayload.class.getName()); boolean result = fileSystem.mkdirs(path); diff --git a/hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala b/hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala index 285ba87b1..dec9160e2 100644 --- a/hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala +++ b/hoodie-spark/src/main/scala/com/uber/hoodie/DefaultSource.scala @@ -190,7 +190,8 @@ class DefaultSource extends RelationProvider properties.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, tblName.get); properties.put(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, storageType); properties.put(HoodieTableConfig.HOODIE_ARCHIVELOG_FOLDER_PROP_NAME, "archived"); - HoodieTableMetaClient.initializePathAsHoodieDataset(fs, path.get, properties); + HoodieTableMetaClient.initializePathAsHoodieDataset( + sqlContext.sparkContext.hadoopConfiguration, path.get, properties); } // Create a HoodieWriteClient & issue the write. diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HDFSParquetImporter.java b/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HDFSParquetImporter.java index 1d3a2b1c1..77184971c 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HDFSParquetImporter.java +++ b/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/HDFSParquetImporter.java @@ -232,7 +232,7 @@ public class HDFSParquetImporter implements Serializable { Properties properties = new Properties(); properties.put(HoodieTableConfig.HOODIE_TABLE_NAME_PROP_NAME, cfg.tableName); properties.put(HoodieTableConfig.HOODIE_TABLE_TYPE_PROP_NAME, cfg.tableType); - HoodieTableMetaClient.initializePathAsHoodieDataset(fs, cfg.targetPath, properties); + HoodieTableMetaClient.initializePathAsHoodieDataset(jsc.hadoopConfiguration(), cfg.targetPath, properties); HoodieWriteClient client = createHoodieClient(jsc, cfg.targetPath, schemaStr, cfg.parallelism); diff --git a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java b/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java index 6a63c33fc..eadb5cd49 100644 --- a/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java +++ b/hoodie-utilities/src/main/java/com/uber/hoodie/utilities/deltastreamer/HoodieDeltaStreamer.java @@ -202,8 +202,7 @@ public class HoodieDeltaStreamer implements Serializable { Properties properties = new Properties(); properties.put(HoodieWriteConfig.TABLE_NAME, cfg.targetTableName); HoodieTableMetaClient - .initializePathAsHoodieDataset( - FSUtils.getFs(cfg.targetBasePath, jssc.hadoopConfiguration()), cfg.targetBasePath, + .initializePathAsHoodieDataset(jssc.hadoopConfiguration(), cfg.targetBasePath, properties); } log.info("Checkpoint to resume from : " + resumeCheckpointStr); diff --git a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieSnapshotCopier.java b/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieSnapshotCopier.java index 326894dfe..f52233339 100644 --- a/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieSnapshotCopier.java +++ b/hoodie-utilities/src/test/java/com/uber/hoodie/utilities/TestHoodieSnapshotCopier.java @@ -25,6 +25,7 @@ import com.uber.hoodie.common.model.HoodieTestUtils; import com.uber.hoodie.common.util.FSUtils; import java.io.File; import java.io.IOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.spark.SparkConf; @@ -52,8 +53,9 @@ public class TestHoodieSnapshotCopier { basePath = rootPath + "/" + HoodieTestUtils.RAW_TRIPS_TEST_NAME; outputPath = rootPath + "/output"; - fs = FSUtils.getFs(basePath, HoodieTestUtils.getDefaultHadoopConf()); - HoodieTestUtils.init(fs, basePath); + final Configuration hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); + fs = FSUtils.getFs(basePath, hadoopConf); + HoodieTestUtils.init(hadoopConf, basePath); // Start a local Spark job SparkConf conf = new SparkConf().setAppName("snapshot-test-job").setMaster("local[2]"); jsc = new JavaSparkContext(conf);