lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-554] Cleanup package structure in hudi-client (#1346)

Browse Source 
- Just package, class moves and renames with the following intent
 - `client` now has all the various client classes, that do the transaction management
 - `func` renamed to `execution` and some helpers moved to `client/utils`
 - All compaction code under `io` now under `table/compact`
 - Rollback code under `table/rollback` and in general all code for individual operations under `table`
 - `exception` `config`, `metrics` left untouched
 - Moved the tests also accordingly
 - Fixed some flaky tests
This commit is contained in:
vinoth chandar
2020-02-27 08:05:58 -08:00
committed by GitHub
parent cacd9a3322
commit 71170fafe7
90 changed files with 290 additions and 301 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-client/src/main/java/org/apache/hudi/AbstractHoodieClient.java → hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieClient.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi;
package org.apache.hudi.client;
import org.apache.hudi.client.embedded.EmbeddedTimelineService;
import org.apache.hudi.client.utils.ClientUtils;

3
hudi-client/src/main/java/org/apache/hudi/AbstractHoodieWriteClient.java → hudi-client/src/main/java/org/apache/hudi/client/AbstractHoodieWriteClient.java
View File

@@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hudi;
package org.apache.hudi.client;
import java.util.Collections;
import org.apache.hudi.avro.model.HoodieRollbackMetadata;
import org.apache.hudi.client.embedded.EmbeddedTimelineService;
import org.apache.hudi.common.HoodieRollbackStat;

4
hudi-client/src/main/java/org/apache/hudi/CompactionAdminClient.java → hudi-client/src/main/java/org/apache/hudi/client/CompactionAdminClient.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi;
package org.apache.hudi.client;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
@@ -40,7 +40,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.func.OperationResult;
import org.apache.hudi.table.compact.OperationResult;
import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileStatus;

6
hudi-client/src/main/java/org/apache/hudi/HoodieCleanClient.java → hudi-client/src/main/java/org/apache/hudi/client/HoodieCleanClient.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi;
package org.apache.hudi.client;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
@@ -78,7 +78,7 @@ public class HoodieCleanClient<T extends HoodieRecordPayload> extends AbstractHo
* @param startCleanTime Cleaner Instant Timestamp
* @throws HoodieIOException in case of any IOException
*/
protected HoodieCleanMetadata clean(String startCleanTime) throws HoodieIOException {
public HoodieCleanMetadata clean(String startCleanTime) throws HoodieIOException {
// Create a Hoodie table which encapsulated the commits and files visible
final HoodieTable<T> table = HoodieTable.getHoodieTable(createMetaClient(true), config, jsc);
@@ -136,7 +136,7 @@ public class HoodieCleanClient<T extends HoodieRecordPayload> extends AbstractHo
* @param table Hoodie Table
* @param cleanInstant Cleaner Instant
*/
protected HoodieCleanMetadata runClean(HoodieTable<T> table, HoodieInstant cleanInstant) {
public HoodieCleanMetadata runClean(HoodieTable<T> table, HoodieInstant cleanInstant) {
try {
HoodieCleanerPlan cleanerPlan = CleanerUtils.getCleanerPlan(table.getMetaClient(), cleanInstant);
return runClean(table, cleanInstant, cleanerPlan);

2
hudi-client/src/main/java/org/apache/hudi/HoodieReadClient.java → hudi-client/src/main/java/org/apache/hudi/client/HoodieReadClient.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi;
package org.apache.hudi.client;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.client.embedded.EmbeddedTimelineService;

8
hudi-client/src/main/java/org/apache/hudi/HoodieWriteClient.java → hudi-client/src/main/java/org/apache/hudi/client/HoodieWriteClient.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi;
package org.apache.hudi.client;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
@@ -51,9 +51,9 @@ import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.exception.HoodieRollbackException;
import org.apache.hudi.exception.HoodieSavepointException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.func.BulkInsertMapFunction;
import org.apache.hudi.execution.BulkInsertMapFunction;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.io.HoodieCommitArchiveLog;
import org.apache.hudi.table.HoodieCommitArchiveLog;
import org.apache.hudi.metrics.HoodieMetrics;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.UserDefinedBulkInsertPartitioner;
@@ -1111,7 +1111,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
* @param inflightInstant Inflight Compaction Instant
* @param table Hoodie Table
*/
void rollbackInflightCompaction(HoodieInstant inflightInstant, HoodieTable table) throws IOException {
public void rollbackInflightCompaction(HoodieInstant inflightInstant, HoodieTable table) throws IOException {
table.rollback(jsc, inflightInstant, false);
// Revert instant state file
table.getActiveTimeline().revertCompactionInflightToRequested(inflightInstant);

2
hudi-client/src/main/java/org/apache/hudi/WriteStatus.java → hudi-client/src/main/java/org/apache/hudi/client/WriteStatus.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi;
package org.apache.hudi.client;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;

2
hudi-client/src/main/java/org/apache/hudi/func/LazyIterableIterator.java → hudi-client/src/main/java/org/apache/hudi/client/utils/LazyIterableIterator.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.func;
package org.apache.hudi.client.utils;
import java.util.Iterator;

2
hudi-client/src/main/java/org/apache/hudi/func/ParquetReaderIterator.java → hudi-client/src/main/java/org/apache/hudi/client/utils/ParquetReaderIterator.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.func;
package org.apache.hudi.client.utils;
import org.apache.hudi.common.util.queue.BoundedInMemoryQueue;
import org.apache.hudi.exception.HoodieIOException;

4
hudi-client/src/main/java/org/apache/hudi/config/HoodieCompactionConfig.java
View File

@@ -20,8 +20,8 @@ package org.apache.hudi.config;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
import org.apache.hudi.io.compact.strategy.CompactionStrategy;
import org.apache.hudi.io.compact.strategy.LogFileSizeBasedCompactionStrategy;
import org.apache.hudi.table.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.compact.strategy.LogFileSizeBasedCompactionStrategy;
import com.google.common.base.Preconditions;

6
hudi-client/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
View File

@@ -18,15 +18,15 @@
package org.apache.hudi.config;
import org.apache.hudi.HoodieWriteClient;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.HoodieWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieCleaningPolicy;
import org.apache.hudi.common.model.TimelineLayoutVersion;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.util.ConsistencyGuardConfig;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.io.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.compact.strategy.CompactionStrategy;
import org.apache.hudi.metrics.MetricsReporterType;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;

4
hudi-client/src/main/java/org/apache/hudi/func/BulkInsertMapFunction.java → hudi-client/src/main/java/org/apache/hudi/execution/BulkInsertMapFunction.java
View File

@@ -16,9 +16,9 @@
* limitations under the License.
*/
package org.apache.hudi.func;
package org.apache.hudi.execution;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;

5
hudi-client/src/main/java/org/apache/hudi/func/CopyOnWriteLazyInsertIterable.java → hudi-client/src/main/java/org/apache/hudi/execution/CopyOnWriteLazyInsertIterable.java
View File

@@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hudi.func;
package org.apache.hudi.execution;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.utils.LazyIterableIterator;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.Option;

4
hudi-client/src/main/java/org/apache/hudi/func/MergeOnReadLazyInsertIterable.java → hudi-client/src/main/java/org/apache/hudi/execution/MergeOnReadLazyInsertIterable.java
View File

@@ -16,9 +16,9 @@
* limitations under the License.
*/
package org.apache.hudi.func;
package org.apache.hudi.execution;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.config.HoodieWriteConfig;

2
hudi-client/src/main/java/org/apache/hudi/func/SparkBoundedInMemoryExecutor.java → hudi-client/src/main/java/org/apache/hudi/execution/SparkBoundedInMemoryExecutor.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.func;
package org.apache.hudi.execution;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.queue.BoundedInMemoryExecutor;

2
hudi-client/src/main/java/org/apache/hudi/index/HoodieIndex.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.index;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;

2
hudi-client/src/main/java/org/apache/hudi/index/InMemoryHashIndex.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.index;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;

2
hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.index.bloom;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;

2
hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndexCheckFunction.java
View File

@@ -23,7 +23,7 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.func.LazyIterableIterator;
import org.apache.hudi.client.utils.LazyIterableIterator;
import org.apache.hudi.io.HoodieKeyLookupHandle;
import org.apache.hudi.io.HoodieKeyLookupHandle.KeyLookupResult;
import org.apache.hudi.table.HoodieTable;

2
hudi-client/src/main/java/org/apache/hudi/index/hbase/HBaseIndex.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.index.hbase;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;

2
hudi-client/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.io;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieDeltaWriteStat;
import org.apache.hudi.common.model.HoodieKey;

2
hudi-client/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.io;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordLocation;

2
hudi-client/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.io;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieRecord;

2
hudi-client/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.io;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.FSUtils;

9
hudi-client/src/main/java/org/apache/hudi/io/HoodieCleanHelper.java → hudi-client/src/main/java/org/apache/hudi/table/CleanHelper.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io;
package org.apache.hudi.table;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.common.model.CompactionOperation;
@@ -38,7 +38,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.table.HoodieTable;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -60,9 +59,9 @@ import java.util.stream.Collectors;
* <p>
* TODO: Should all cleaning be done based on {@link HoodieCommitMetadata}
*/
public class HoodieCleanHelper<T extends HoodieRecordPayload<T>> implements Serializable {
public class CleanHelper<T extends HoodieRecordPayload<T>> implements Serializable {
private static final Logger LOG = LogManager.getLogger(HoodieCleanHelper.class);
private static final Logger LOG = LogManager.getLogger(CleanHelper.class);
private final SyncableFileSystemView fileSystemView;
private final HoodieTimeline commitTimeline;
@@ -70,7 +69,7 @@ public class HoodieCleanHelper<T extends HoodieRecordPayload<T>> implements Seri
private HoodieTable<T> hoodieTable;
private HoodieWriteConfig config;
public HoodieCleanHelper(HoodieTable<T> hoodieTable, HoodieWriteConfig config) {
public CleanHelper(HoodieTable<T> hoodieTable, HoodieWriteConfig config) {
this.hoodieTable = hoodieTable;
this.fileSystemView = hoodieTable.getHoodieView();
this.commitTimeline = hoodieTable.getCompletedCommitTimeline();

3
hudi-client/src/main/java/org/apache/hudi/io/HoodieCommitArchiveLog.java → hudi-client/src/main/java/org/apache/hudi/table/HoodieCommitArchiveLog.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io;
package org.apache.hudi.table;
import org.apache.hudi.avro.model.HoodieArchivedMetaEntry;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
@@ -45,7 +45,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieCommitException;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.table.HoodieTable;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;

15
hudi-client/src/main/java/org/apache/hudi/table/HoodieCopyOnWriteTable.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.table;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.avro.model.HoodieActionInstant;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
@@ -46,10 +46,9 @@ import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.func.CopyOnWriteLazyInsertIterable;
import org.apache.hudi.func.ParquetReaderIterator;
import org.apache.hudi.func.SparkBoundedInMemoryExecutor;
import org.apache.hudi.io.HoodieCleanHelper;
import org.apache.hudi.execution.CopyOnWriteLazyInsertIterable;
import org.apache.hudi.client.utils.ParquetReaderIterator;
import org.apache.hudi.execution.SparkBoundedInMemoryExecutor;
import org.apache.hudi.io.HoodieCreateHandle;
import org.apache.hudi.io.HoodieMergeHandle;
@@ -58,6 +57,8 @@ import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.table.rollback.RollbackHelper;
import org.apache.hudi.table.rollback.RollbackRequest;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.parquet.avro.AvroParquetReader;
@@ -284,7 +285,7 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
@Override
public HoodieCleanerPlan scheduleClean(JavaSparkContext jsc) {
try {
HoodieCleanHelper cleaner = new HoodieCleanHelper(this, config);
CleanHelper cleaner = new CleanHelper(this, config);
Option<HoodieInstant> earliestInstant = cleaner.getEarliestCommitToRetain();
List<String> partitionsToClean = cleaner.getPartitionPathsToClean(earliestInstant);
@@ -370,7 +371,7 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
List<RollbackRequest> rollbackRequests = generateRollbackRequests(instant);
//TODO: We need to persist this as rollback workload and use it in case of partial failures
stats = new RollbackExecutor(metaClient, config).performRollback(jsc, instant, rollbackRequests);
stats = new RollbackHelper(metaClient, config).performRollback(jsc, instant, rollbackRequests);
}
// Delete Inflight instant if enabled
deleteInflightAndRequestedInstant(deleteInstants, activeTimeline, instant);

10
hudi-client/src/main/java/org/apache/hudi/table/HoodieMergeOnReadTable.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.table;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.HoodieRollbackStat;
import org.apache.hudi.common.model.FileSlice;
@@ -38,11 +38,13 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieCompactionException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieUpsertException;
import org.apache.hudi.func.MergeOnReadLazyInsertIterable;
import org.apache.hudi.execution.MergeOnReadLazyInsertIterable;
import org.apache.hudi.io.HoodieAppendHandle;
import org.apache.hudi.io.compact.HoodieMergeOnReadTableCompactor;
import org.apache.hudi.table.compact.HoodieMergeOnReadTableCompactor;
import com.google.common.base.Preconditions;
import org.apache.hudi.table.rollback.RollbackHelper;
import org.apache.hudi.table.rollback.RollbackRequest;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.Partitioner;
@@ -194,7 +196,7 @@ public class HoodieMergeOnReadTable<T extends HoodieRecordPayload> extends Hoodi
LOG.info("Unpublished " + commit);
List<RollbackRequest> rollbackRequests = generateRollbackRequests(jsc, instant);
// TODO: We need to persist this as rollback workload and use it in case of partial failures
allRollbackStats = new RollbackExecutor(metaClient, config).performRollback(jsc, instant, rollbackRequests);
allRollbackStats = new RollbackHelper(metaClient, config).performRollback(jsc, instant, rollbackRequests);
}
// Delete Inflight instants if enabled

2
hudi-client/src/main/java/org/apache/hudi/table/HoodieTable.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.table;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.avro.model.HoodieSavepointMetadata;

4
hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieCompactor.java → hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieCompactor.java
View File

@@ -16,9 +16,9 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact;
package org.apache.hudi.table.compact;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.model.HoodieFileGroupId;
import org.apache.hudi.config.HoodieWriteConfig;

6
hudi-client/src/main/java/org/apache/hudi/io/compact/HoodieMergeOnReadTableCompactor.java → hudi-client/src/main/java/org/apache/hudi/table/compact/HoodieMergeOnReadTableCompactor.java
View File

@@ -16,9 +16,9 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact;
package org.apache.hudi.table.compact;
import org.apache.hudi.WriteStatus;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.common.model.CompactionOperation;
@@ -37,7 +37,7 @@ import org.apache.hudi.common.util.HoodieAvroUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.io.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.HoodieCopyOnWriteTable;
import org.apache.hudi.table.HoodieTable;

2
hudi-client/src/main/java/org/apache/hudi/func/OperationResult.java → hudi-client/src/main/java/org/apache/hudi/table/compact/OperationResult.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.func;
package org.apache.hudi.table.compact;
import org.apache.hudi.common.util.Option;

2
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedIOCompactionStrategy.java → hudi-client/src/main/java/org/apache/hudi/table/compact/strategy/BoundedIOCompactionStrategy.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
package org.apache.hudi.table.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;

2
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/BoundedPartitionAwareCompactionStrategy.java → hudi-client/src/main/java/org/apache/hudi/table/compact/strategy/BoundedPartitionAwareCompactionStrategy.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
package org.apache.hudi.table.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;

4
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/CompactionStrategy.java → hudi-client/src/main/java/org/apache/hudi/table/compact/strategy/CompactionStrategy.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
package org.apache.hudi.table.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
@@ -26,7 +26,7 @@ import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.io.compact.HoodieMergeOnReadTableCompactor;
import org.apache.hudi.table.compact.HoodieMergeOnReadTableCompactor;
import java.io.Serializable;
import java.util.HashMap;

2
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/DayBasedCompactionStrategy.java → hudi-client/src/main/java/org/apache/hudi/table/compact/strategy/DayBasedCompactionStrategy.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
package org.apache.hudi.table.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;

2
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/LogFileSizeBasedCompactionStrategy.java → hudi-client/src/main/java/org/apache/hudi/table/compact/strategy/LogFileSizeBasedCompactionStrategy.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
package org.apache.hudi.table.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;

2
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedCompactionStrategy.java → hudi-client/src/main/java/org/apache/hudi/table/compact/strategy/UnBoundedCompactionStrategy.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
package org.apache.hudi.table.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;

2
hudi-client/src/main/java/org/apache/hudi/io/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java → hudi-client/src/main/java/org/apache/hudi/table/compact/strategy/UnBoundedPartitionAwareCompactionStrategy.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.io.compact.strategy;
package org.apache.hudi.table.compact.strategy;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
import org.apache.hudi.avro.model.HoodieCompactionPlan;

11
hudi-client/src/main/java/org/apache/hudi/table/RollbackExecutor.java → hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackHelper.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.table;
package org.apache.hudi.table.rollback;
import org.apache.hudi.common.HoodieRollbackStat;
import org.apache.hudi.common.model.HoodieLogFile;
@@ -53,14 +53,14 @@ import scala.Tuple2;
/**
* Performs Rollback of Hoodie Tables.
*/
public class RollbackExecutor implements Serializable {
public class RollbackHelper implements Serializable {
private static final Logger LOG = LogManager.getLogger(RollbackExecutor.class);
private static final Logger LOG = LogManager.getLogger(RollbackHelper.class);
private final HoodieTableMetaClient metaClient;
private final HoodieWriteConfig config;
public RollbackExecutor(HoodieTableMetaClient metaClient, HoodieWriteConfig config) {
public RollbackHelper(HoodieTableMetaClient metaClient, HoodieWriteConfig config) {
this.metaClient = metaClient;
this.config = config;
}
@@ -68,8 +68,7 @@ public class RollbackExecutor implements Serializable {
/**
* Performs all rollback actions that we have collected in parallel.
*/
public List<HoodieRollbackStat> performRollback(JavaSparkContext jsc, HoodieInstant instantToRollback,
List<RollbackRequest> rollbackRequests) {
public List<HoodieRollbackStat> performRollback(JavaSparkContext jsc, HoodieInstant instantToRollback, List<RollbackRequest> rollbackRequests) {
SerializablePathFilter filter = (path) -> {
if (path.toString().contains(".parquet")) {

6
hudi-client/src/main/java/org/apache/hudi/table/RollbackRequest.java → hudi-client/src/main/java/org/apache/hudi/table/rollback/RollbackRequest.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.table;
package org.apache.hudi.table.rollback;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
@@ -30,7 +30,9 @@ public class RollbackRequest {
* Rollback Action Types.
*/
public enum RollbackAction {
DELETE_DATA_FILES_ONLY, DELETE_DATA_AND_LOG_FILES, APPEND_ROLLBACK_BLOCK
DELETE_DATA_FILES_ONLY,
DELETE_DATA_AND_LOG_FILES,
APPEND_ROLLBACK_BLOCK
}
/**