diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java index 038e1ac27..a86edc4b7 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java @@ -37,6 +37,8 @@ import com.uber.hoodie.common.table.HoodieTableMetaClient; import com.uber.hoodie.common.table.HoodieTimeline; import com.uber.hoodie.common.table.timeline.HoodieInstant; import com.uber.hoodie.common.table.timeline.HoodieInstant.State; +import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; +import com.uber.hoodie.common.table.view.FileSystemViewStorageType; import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; import com.uber.hoodie.common.util.AvroUtils; import com.uber.hoodie.common.util.CompactionUtils; @@ -75,7 +77,10 @@ public class TestAsyncCompaction extends TestHoodieClientBase { .withMaxNumDeltaCommitsBeforeCompaction(1).build()) .withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024 * 1024 * 1024).build()) .forTable("test-trip-table") - .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()); + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()) + .withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig( + FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE) + .build()); } @Override @@ -572,8 +577,4 @@ public class TestAsyncCompaction extends TestHoodieClientBase { protected HoodieTableType getTableType() { return HoodieTableType.MERGE_ON_READ; } - - protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWriteConfig config) { - return HoodieTable.getHoodieTable(metaClient, config, jsc); - } } diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompactionWithEmbeddedTimelineServer.java b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompactionWithEmbeddedTimelineServer.java deleted file mode 100644 index fe5f1951b..000000000 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompactionWithEmbeddedTimelineServer.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.uber.hoodie; - -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.table.HoodieTable; -import java.io.IOException; -import org.junit.After; - -public class TestAsyncCompactionWithEmbeddedTimelineServer extends TestAsyncCompaction { - - @Override - protected HoodieWriteConfig.Builder getConfigBuilder(Boolean autoCommit) { - HoodieWriteConfig.Builder builder = super.getConfigBuilder(autoCommit); - try { - return builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig( - FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE) - .build()); - } catch (Exception e) { - throw new HoodieException(e); - } - } - - @Override - protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWriteConfig config) { - HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc); - ((SyncableFileSystemView) (table.getRTFileSystemView())).reset(); - return table; - } - - @After - public void tearDown() throws IOException { - super.tearDown(); - } -} diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestCleanerWithEmbeddedTimelineServer.java b/hoodie-client/src/test/java/com/uber/hoodie/TestCleanerWithEmbeddedTimelineServer.java deleted file mode 100644 index 357854488..000000000 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestCleanerWithEmbeddedTimelineServer.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.uber.hoodie; - -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import java.io.IOException; -import org.junit.After; - -public class TestCleanerWithEmbeddedTimelineServer extends TestCleaner { - - @Override - HoodieWriteConfig.Builder getConfigBuilder() { - HoodieWriteConfig.Builder builder = super.getConfigBuilder(); - try { - builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig( - FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE) - .build()); - } catch (Exception e) { - throw new HoodieException(e); - } - return builder; - } - - @After - public void tearDown() throws IOException { - super.tearDown(); - } -} diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java index ae2e57ceb..850f45fbb 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientBase.java @@ -33,7 +33,10 @@ import com.uber.hoodie.common.model.HoodieTableType; import com.uber.hoodie.common.model.HoodieTestUtils; import com.uber.hoodie.common.table.HoodieTableMetaClient; import com.uber.hoodie.common.table.HoodieTimeline; +import com.uber.hoodie.common.table.SyncableFileSystemView; import com.uber.hoodie.common.table.timeline.HoodieActiveTimeline; +import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; +import com.uber.hoodie.common.table.view.FileSystemViewStorageType; import com.uber.hoodie.common.util.FSUtils; import com.uber.hoodie.config.HoodieCompactionConfig; import com.uber.hoodie.config.HoodieIndexConfig; @@ -176,7 +179,16 @@ public class TestHoodieClientBase implements Serializable { .withCompactionConfig(HoodieCompactionConfig.newBuilder().compactionSmallFileSize(1024 * 1024).build()) .withStorageConfig(HoodieStorageConfig.newBuilder().limitFileSize(1024 * 1024).build()) .forTable("test-trip-table") - .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()); + .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()) + .withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig( + FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE) + .build()); + } + + protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWriteConfig config) { + HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc); + ((SyncableFileSystemView) (table.getRTFileSystemView())).reset(); + return table; } /** diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java index 0c2fefb4e..6bcfcd974 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieClientOnCopyOnWriteStorage.java @@ -446,10 +446,6 @@ public class TestHoodieClientOnCopyOnWriteStorage extends TestHoodieClientBase { assertEquals("Total inserts in commit3 must add up", keys3.size(), numTotalInsertsInCommit3); } - protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWriteConfig config) { - return HoodieTable.getHoodieTable(metaClient, config, jsc); - } - /** * Test scenario of new file-group getting added during insert() */ diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClientWithEmbeddedServer.java b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClientWithEmbeddedServer.java deleted file mode 100644 index a7936cd45..000000000 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieReadClientWithEmbeddedServer.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.uber.hoodie; - -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import java.io.IOException; -import org.junit.After; - -public class TestHoodieReadClientWithEmbeddedServer extends TestHoodieReadClient { - - @Override - HoodieWriteConfig.Builder getConfigBuilder() { - HoodieWriteConfig.Builder builder = super.getConfigBuilder(); - try { - builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig( - FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE) - .build()); - } catch (Exception e) { - throw new HoodieException(e); - } - return builder; - } - - @After - public void tearDown() throws IOException { - super.tearDown(); - } -} diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieWriteClientWithEmbeddedServer.java b/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieWriteClientWithEmbeddedServer.java deleted file mode 100644 index 178941341..000000000 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestHoodieWriteClientWithEmbeddedServer.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.uber.hoodie; - -import com.uber.hoodie.common.table.HoodieTableMetaClient; -import com.uber.hoodie.common.table.SyncableFileSystemView; -import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig; -import com.uber.hoodie.common.table.view.FileSystemViewStorageType; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.exception.HoodieException; -import com.uber.hoodie.table.HoodieTable; -import java.io.IOException; -import org.junit.After; - -public class TestHoodieWriteClientWithEmbeddedServer extends TestHoodieClientOnCopyOnWriteStorage { - - @Override - HoodieWriteConfig.Builder getConfigBuilder() { - HoodieWriteConfig.Builder builder = super.getConfigBuilder(); - try { - builder.withEmbeddedTimelineServerEnabled(true).withFileSystemViewConfig( - FileSystemViewStorageConfig.newBuilder().withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE) - .build()); - } catch (Exception e) { - throw new HoodieException(e); - } - return builder; - } - - @Override - protected HoodieTable getHoodieTable(HoodieTableMetaClient metaClient, HoodieWriteConfig config) { - HoodieTable table = HoodieTable.getHoodieTable(metaClient, config, jsc); - ((SyncableFileSystemView) (table.getRTFileSystemView())).reset(); - return table; - } - - @After - public void tearDown() throws IOException { - super.tearDown(); - } -} diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java index 2388c6a65..2ae6c9385 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFS.java @@ -99,10 +99,10 @@ public class TestMultiFS implements Serializable { } protected HoodieWriteConfig getHoodieWriteConfig(String basePath) { - return HoodieWriteConfig.newBuilder().withPath(basePath) + return HoodieWriteConfig.newBuilder().withPath(basePath).withEmbeddedTimelineServerEnabled(true) .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) .forTable(tableName).withIndexConfig( - HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build(); + HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build(); } @Test diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFSWithEmbeddedServer.java b/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFSWithEmbeddedServer.java deleted file mode 100644 index e096fcaa0..000000000 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestMultiFSWithEmbeddedServer.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.uber.hoodie; - -import com.uber.hoodie.common.HoodieTestDataGenerator; -import com.uber.hoodie.config.HoodieIndexConfig; -import com.uber.hoodie.config.HoodieWriteConfig; -import com.uber.hoodie.index.HoodieIndex; - -/** - * Tests MultiFS with embedded timeline server enabled - */ -public class TestMultiFSWithEmbeddedServer extends TestMultiFS { - - @Override - protected HoodieWriteConfig getHoodieWriteConfig(String basePath) { - return HoodieWriteConfig.newBuilder().withPath(basePath).withEmbeddedTimelineServerEnabled(true) - .withSchema(HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA).withParallelism(2, 2) - .forTable(tableName).withIndexConfig( - HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BLOOM).build()).build(); - } -}