lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1684] Tweak hudi-flink-bundle module pom and reorganize the pacakges for hudi-flink module (#2669)

Browse Source 
* Add required dependencies for hudi-flink-bundle module
* Some packages reorganization of hudi-flink module
This commit is contained in:
Danny Chan
2021-03-15 16:02:05 +08:00
committed by GitHub
parent e93c6a5693
commit fc6c5f4285
72 changed files with 357 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
hudi-flink/src/main/java/org/apache/hudi/operator/FlinkOptions.java → hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
View File

@@ -16,14 +16,14 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.configuration;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.common.model.OverwriteWithLatestAvroPayload;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.keygen.SimpleAvroKeyGenerator;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.configuration.ConfigOption;

2
hudi-flink/src/main/java/org/apache/hudi/schema/FilebasedSchemaProvider.java
View File

@@ -20,8 +20,8 @@ package org.apache.hudi.schema;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.util.StreamerUtil;
import org.apache.avro.Schema;

1
hudi-flink/src/main/java/org/apache/hudi/sink/CommitSink.java
View File

@@ -31,7 +31,6 @@ import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.sink.RichSinkFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

8
hudi-flink/src/main/java/org/apache/hudi/operator/InstantGenerateOperator.java → hudi-flink/src/main/java/org/apache/hudi/sink/InstantGenerateOperator.java
View File

@@ -16,9 +16,8 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.HoodieFlinkWriteClient;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
@@ -30,6 +29,8 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.api.common.state.ListState;
@@ -40,11 +41,10 @@ import org.apache.flink.streaming.api.operators.AbstractStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.StreamingRuntimeContext;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.PathFilter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

6
hudi-flink/src/main/java/org/apache/hudi/operator/KeyedWriteProcessFunction.java → hudi-flink/src/main/java/org/apache/hudi/sink/KeyedWriteProcessFunction.java
View File

@@ -16,10 +16,8 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.HoodieFlinkWriteClient;
import org.apache.hudi.client.WriteStatus;
@@ -29,6 +27,8 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.exception.HoodieFlinkStreamerException;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.table.action.commit.FlinkWriteHelper;
import org.apache.hudi.util.StreamerUtil;

3
hudi-flink/src/main/java/org/apache/hudi/operator/KeyedWriteProcessOperator.java → hudi-flink/src/main/java/org/apache/hudi/sink/KeyedWriteProcessOperator.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieRecord;
@@ -26,7 +26,6 @@ import org.apache.flink.runtime.state.StateSnapshotContext;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.streaming.api.operators.KeyedProcessOperator;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

5
hudi-flink/src/main/java/org/apache/hudi/operator/StreamWriteFunction.java → hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteFunction.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.HoodieFlinkWriteClient;
@@ -26,8 +26,9 @@ import org.apache.hudi.common.config.SerializableConfiguration;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.util.ObjectSizeCalculator;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.operator.event.BatchWriteSuccessEvent;
import org.apache.hudi.sink.event.BatchWriteSuccessEvent;
import org.apache.hudi.table.action.commit.FlinkWriteHelper;
import org.apache.hudi.util.StreamerUtil;

2
hudi-flink/src/main/java/org/apache/hudi/operator/StreamWriteOperator.java → hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperator.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;

5
hudi-flink/src/main/java/org/apache/hudi/operator/StreamWriteOperatorCoordinator.java → hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
View File

@@ -16,15 +16,16 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.HoodieFlinkWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.operator.event.BatchWriteSuccessEvent;
import org.apache.hudi.sink.event.BatchWriteSuccessEvent;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.annotation.VisibleForTesting;

2
hudi-flink/src/main/java/org/apache/hudi/operator/StreamWriteOperatorFactory.java → hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorFactory.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.jobgraph.OperatorID;

2
hudi-flink/src/main/java/org/apache/hudi/operator/compact/CompactFunction.java → hudi-flink/src/main/java/org/apache/hudi/sink/compact/CompactFunction.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.compact;
package org.apache.hudi.sink.compact;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.HoodieFlinkWriteClient;

2
hudi-flink/src/main/java/org/apache/hudi/operator/compact/CompactionCommitEvent.java → hudi-flink/src/main/java/org/apache/hudi/sink/compact/CompactionCommitEvent.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.compact;
package org.apache.hudi.sink.compact;
import org.apache.hudi.client.WriteStatus;

2
hudi-flink/src/main/java/org/apache/hudi/operator/compact/CompactionCommitSink.java → hudi-flink/src/main/java/org/apache/hudi/sink/compact/CompactionCommitSink.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.compact;
package org.apache.hudi.sink.compact;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.client.FlinkTaskContextSupplier;

2
hudi-flink/src/main/java/org/apache/hudi/operator/compact/CompactionPlanEvent.java → hudi-flink/src/main/java/org/apache/hudi/sink/compact/CompactionPlanEvent.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.compact;
package org.apache.hudi.sink.compact;
import org.apache.hudi.common.model.CompactionOperation;

2
hudi-flink/src/main/java/org/apache/hudi/operator/compact/CompactionPlanOperator.java → hudi-flink/src/main/java/org/apache/hudi/sink/compact/CompactionPlanOperator.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.compact;
package org.apache.hudi.sink.compact;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.client.FlinkTaskContextSupplier;

6
hudi-flink/src/main/java/org/apache/hudi/operator/event/BatchWriteSuccessEvent.java → hudi-flink/src/main/java/org/apache/hudi/sink/event/BatchWriteSuccessEvent.java
View File

@@ -16,13 +16,13 @@
* limitations under the License.
*/
package org.apache.hudi.operator.event;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;
package org.apache.hudi.sink.event;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;
import java.util.ArrayList;
import java.util.List;

4
hudi-flink/src/main/java/org/apache/hudi/operator/partitioner/BucketAssignFunction.java → hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssignFunction.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.partitioner;
package org.apache.hudi.sink.partitioner;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
@@ -30,10 +30,10 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.index.HoodieIndexUtils;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.table.action.commit.BucketInfo;
import org.apache.hudi.util.StreamerUtil;

2
hudi-flink/src/main/java/org/apache/hudi/operator/partitioner/BucketAssigner.java → hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssigner.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.partitioner;
package org.apache.hudi.sink.partitioner;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
import org.apache.hudi.common.fs.FSUtils;

4
hudi-flink/src/main/java/org/apache/hudi/operator/partitioner/BucketAssigners.java → hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/BucketAssigners.java
View File

@@ -16,12 +16,12 @@
* limitations under the License.
*/
package org.apache.hudi.operator.partitioner;
package org.apache.hudi.sink.partitioner;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.operator.partitioner.delta.DeltaBucketAssigner;
import org.apache.hudi.sink.partitioner.delta.DeltaBucketAssigner;
/**
* Utilities for {@code BucketAssigner}.

4
hudi-flink/src/main/java/org/apache/hudi/operator/partitioner/delta/DeltaBucketAssigner.java → hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/delta/DeltaBucketAssigner.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.partitioner.delta;
package org.apache.hudi.sink.partitioner.delta;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
import org.apache.hudi.common.fs.FSUtils;
@@ -26,7 +26,7 @@ import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.operator.partitioner.BucketAssigner;
import org.apache.hudi.sink.partitioner.BucketAssigner;
import org.apache.hudi.table.action.commit.SmallFile;
import java.util.ArrayList;

2
hudi-flink/src/main/java/org/apache/hudi/source/JsonStringToHoodieRecordMapFunction.java → hudi-flink/src/main/java/org/apache/hudi/sink/transform/JsonStringToHoodieRecordMapFunction.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source;
package org.apache.hudi.sink.transform;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.config.TypedProperties;

4
hudi-flink/src/main/java/org/apache/hudi/operator/transform/RowDataToHoodieFunction.java → hudi-flink/src/main/java/org/apache/hudi/sink/transform/RowDataToHoodieFunction.java
View File

@@ -16,15 +16,15 @@
* limitations under the License.
*/
package org.apache.hudi.operator.transform;
package org.apache.hudi.sink.transform;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.util.RowDataToAvroConverters;
import org.apache.hudi.util.StreamerUtil;

11
hudi-flink/src/main/java/org/apache/hudi/operator/StreamReadMonitoringFunction.java → hudi-flink/src/main/java/org/apache/hudi/source/StreamReadMonitoringFunction.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.source;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -27,9 +27,10 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.source.format.mor.InstantRange;
import org.apache.hudi.source.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.table.format.mor.InstantRange;
import org.apache.hudi.table.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.annotation.VisibleForTesting;
@@ -341,13 +342,13 @@ public class StreamReadMonitoringFunction
* @return the file statuses array
*/
private FileStatus[] getWritePathsOfInstants(List<HoodieCommitMetadata> metadataList) {
FileSystem fs = FSUtils.getFs(path.getPath(), hadoopConf);
FileSystem fs = FSUtils.getFs(path.toString(), hadoopConf);
return metadataList.stream().map(metadata -> getWritePathsOfInstant(metadata, fs))
.flatMap(Collection::stream).toArray(FileStatus[]::new);
}
private List<FileStatus> getWritePathsOfInstant(HoodieCommitMetadata metadata, FileSystem fs) {
return metadata.getFileIdAndFullPaths(path.getPath()).values().stream()
return metadata.getFileIdAndFullPaths(path.toString()).values().stream()
.map(path -> {
try {
return fs.getFileStatus(new org.apache.hadoop.fs.Path(path));

6
hudi-flink/src/main/java/org/apache/hudi/operator/StreamReadOperator.java → hudi-flink/src/main/java/org/apache/hudi/source/StreamReadOperator.java
View File

@@ -16,10 +16,10 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.source;
import org.apache.hudi.source.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.source.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.table.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.table.format.mor.MergeOnReadInputSplit;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;

12
hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamer.java
View File

@@ -22,13 +22,13 @@ import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.InstantGenerateOperator;
import org.apache.hudi.operator.KeyedWriteProcessFunction;
import org.apache.hudi.operator.KeyedWriteProcessOperator;
import org.apache.hudi.operator.partitioner.BucketAssignFunction;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.sink.CommitSink;
import org.apache.hudi.source.JsonStringToHoodieRecordMapFunction;
import org.apache.hudi.sink.InstantGenerateOperator;
import org.apache.hudi.sink.KeyedWriteProcessFunction;
import org.apache.hudi.sink.KeyedWriteProcessOperator;
import org.apache.hudi.sink.partitioner.BucketAssignFunction;
import org.apache.hudi.sink.transform.JsonStringToHoodieRecordMapFunction;
import org.apache.hudi.util.StreamerUtil;
import com.beust.jcommander.JCommander;

8
hudi-flink/src/main/java/org/apache/hudi/streamer/HoodieFlinkStreamerV2.java
View File

@@ -19,10 +19,10 @@
package org.apache.hudi.streamer;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.StreamWriteOperatorFactory;
import org.apache.hudi.operator.partitioner.BucketAssignFunction;
import org.apache.hudi.operator.transform.RowDataToHoodieFunction;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.sink.StreamWriteOperatorFactory;
import org.apache.hudi.sink.partitioner.BucketAssignFunction;
import org.apache.hudi.sink.transform.RowDataToHoodieFunction;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.hudi.util.StreamerUtil;

6
hudi-flink/src/main/java/org/apache/hudi/factory/HoodieTableFactory.java → hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableFactory.java
View File

@@ -16,11 +16,9 @@
* limitations under the License.
*/
package org.apache.hudi.factory;
package org.apache.hudi.table;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.sink.HoodieTableSink;
import org.apache.hudi.source.HoodieTableSource;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.flink.configuration.Configuration;

20
hudi-flink/src/main/java/org/apache/hudi/sink/HoodieTableSink.java → hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSink.java
View File

@@ -16,18 +16,18 @@
* limitations under the License.
*/
package org.apache.hudi.sink;
package org.apache.hudi.table;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.StreamWriteOperatorFactory;
import org.apache.hudi.operator.compact.CompactFunction;
import org.apache.hudi.operator.compact.CompactionCommitEvent;
import org.apache.hudi.operator.compact.CompactionCommitSink;
import org.apache.hudi.operator.compact.CompactionPlanEvent;
import org.apache.hudi.operator.compact.CompactionPlanOperator;
import org.apache.hudi.operator.partitioner.BucketAssignFunction;
import org.apache.hudi.operator.transform.RowDataToHoodieFunction;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.sink.StreamWriteOperatorFactory;
import org.apache.hudi.sink.compact.CompactFunction;
import org.apache.hudi.sink.compact.CompactionCommitEvent;
import org.apache.hudi.sink.compact.CompactionCommitSink;
import org.apache.hudi.sink.compact.CompactionPlanEvent;
import org.apache.hudi.sink.compact.CompactionPlanOperator;
import org.apache.hudi.sink.partitioner.BucketAssignFunction;
import org.apache.hudi.sink.transform.RowDataToHoodieFunction;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.annotation.VisibleForTesting;

20
hudi-flink/src/main/java/org/apache/hudi/source/HoodieTableSource.java → hudi-flink/src/main/java/org/apache/hudi/table/HoodieTableSource.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source;
package org.apache.hudi.table;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieBaseFile;
@@ -27,17 +27,17 @@ import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.hadoop.HoodieROTablePathFilter;
import org.apache.hudi.hadoop.utils.HoodieRealtimeInputFormatUtils;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.StreamReadMonitoringFunction;
import org.apache.hudi.operator.StreamReadOperator;
import org.apache.hudi.source.format.FilePathUtils;
import org.apache.hudi.source.format.cow.CopyOnWriteInputFormat;
import org.apache.hudi.source.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.source.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.source.format.mor.MergeOnReadTableState;
import org.apache.hudi.source.StreamReadMonitoringFunction;
import org.apache.hudi.source.StreamReadOperator;
import org.apache.hudi.table.format.FilePathUtils;
import org.apache.hudi.table.format.cow.CopyOnWriteInputFormat;
import org.apache.hudi.table.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.table.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.table.format.mor.MergeOnReadTableState;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.hudi.util.StreamerUtil;
@@ -87,7 +87,7 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.apache.hudi.hadoop.utils.HoodieRealtimeRecordReaderUtils.getMaxCompactionMemoryInBytes;
import static org.apache.hudi.source.format.FormatUtils.getParquetConf;
import static org.apache.hudi.table.format.FormatUtils.getParquetConf;
/**
* Hoodie batch table source that always read the latest snapshot of the underneath table.

4
hudi-flink/src/main/java/org/apache/hudi/source/format/FilePathUtils.java → hudi-flink/src/main/java/org/apache/hudi/table/format/FilePathUtils.java
View File

@@ -16,10 +16,10 @@
* limitations under the License.
*/
package org.apache.hudi.source.format;
package org.apache.hudi.table.format;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.table.api.TableException;

6
hudi-flink/src/main/java/org/apache/hudi/source/format/FormatUtils.java → hudi-flink/src/main/java/org/apache/hudi/table/format/FormatUtils.java
View File

@@ -16,13 +16,13 @@
* limitations under the License.
*/
package org.apache.hudi.source.format;
package org.apache.hudi.table.format;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.hadoop.config.HoodieRealtimeConfig;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.source.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.table.format.mor.MergeOnReadInputSplit;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/cow/AbstractColumnReader.java → hudi-flink/src/main/java/org/apache/hudi/table/format/cow/AbstractColumnReader.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.cow;
package org.apache.hudi.table.format.cow;
import org.apache.flink.formats.parquet.vector.ParquetDictionary;
import org.apache.flink.formats.parquet.vector.reader.ColumnReader;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/cow/CopyOnWriteInputFormat.java → hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.cow;
package org.apache.hudi.table.format.cow;
import org.apache.hudi.common.fs.FSUtils;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/cow/Int64TimestampColumnReader.java → hudi-flink/src/main/java/org/apache/hudi/table/format/cow/Int64TimestampColumnReader.java
View File

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.cow;
package org.apache.hudi.table.format.cow;
import org.apache.flink.table.data.TimestampData;
import org.apache.flink.table.data.vector.writable.WritableIntVector;

6
hudi-flink/src/main/java/org/apache/hudi/source/format/cow/ParquetColumnarRowSplitReader.java → hudi-flink/src/main/java/org/apache/hudi/table/format/cow/ParquetColumnarRowSplitReader.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.cow;
package org.apache.hudi.table.format.cow;
import org.apache.flink.formats.parquet.vector.reader.ColumnReader;
import org.apache.flink.table.data.ColumnarRowData;
@@ -47,8 +47,8 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import static org.apache.hudi.source.format.cow.ParquetSplitReaderUtil.createColumnReader;
import static org.apache.hudi.source.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createColumnReader;
import static org.apache.hudi.table.format.cow.ParquetSplitReaderUtil.createWritableColumnVector;
import static org.apache.parquet.filter2.compat.RowGroupFilter.filterRowGroups;
import static org.apache.parquet.format.converter.ParquetMetadataConverter.range;
import static org.apache.parquet.hadoop.ParquetFileReader.readFooter;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/cow/ParquetDecimalVector.java → hudi-flink/src/main/java/org/apache/hudi/table/format/cow/ParquetDecimalVector.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.cow;
package org.apache.hudi.table.format.cow;
import org.apache.flink.table.data.DecimalData;
import org.apache.flink.table.data.DecimalDataUtils;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/cow/ParquetSplitReaderUtil.java → hudi-flink/src/main/java/org/apache/hudi/table/format/cow/ParquetSplitReaderUtil.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.cow;
package org.apache.hudi.table.format.cow;
import org.apache.flink.core.fs.Path;
import org.apache.flink.formats.parquet.vector.reader.BooleanColumnReader;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/cow/RunLengthDecoder.java → hudi-flink/src/main/java/org/apache/hudi/table/format/cow/RunLengthDecoder.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.cow;
package org.apache.hudi.table.format.cow;
import org.apache.flink.table.data.vector.writable.WritableColumnVector;
import org.apache.flink.table.data.vector.writable.WritableIntVector;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/mor/InstantRange.java → hudi-flink/src/main/java/org/apache/hudi/table/format/mor/InstantRange.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.mor;
package org.apache.hudi.table.format.mor;
import org.apache.hudi.common.table.timeline.HoodieTimeline;

14
hudi-flink/src/main/java/org/apache/hudi/source/format/mor/MergeOnReadInputFormat.java → hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputFormat.java
View File

@@ -16,17 +16,17 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.mor;
package org.apache.hudi.table.format.mor;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.source.format.FilePathUtils;
import org.apache.hudi.source.format.FormatUtils;
import org.apache.hudi.source.format.cow.ParquetColumnarRowSplitReader;
import org.apache.hudi.source.format.cow.ParquetSplitReaderUtil;
import org.apache.hudi.table.format.FilePathUtils;
import org.apache.hudi.table.format.FormatUtils;
import org.apache.hudi.table.format.cow.ParquetColumnarRowSplitReader;
import org.apache.hudi.table.format.cow.ParquetSplitReaderUtil;
import org.apache.hudi.util.AvroToRowDataConverters;
import org.apache.hudi.util.RowDataToAvroConverters;
import org.apache.hudi.util.StreamerUtil;
@@ -57,7 +57,7 @@ import java.util.stream.IntStream;
import static org.apache.flink.table.data.vector.VectorizedColumnBatch.DEFAULT_SIZE;
import static org.apache.flink.table.filesystem.RowPartitionComputer.restorePartValueFromType;
import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.HOODIE_RECORD_KEY_COL_POS;
import static org.apache.hudi.source.format.FormatUtils.buildAvroRecordBySchema;
import static org.apache.hudi.table.format.FormatUtils.buildAvroRecordBySchema;
/**
* The base InputFormat class to read from Hoodie data + log files.

2
hudi-flink/src/main/java/org/apache/hudi/source/format/mor/MergeOnReadInputSplit.java → hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadInputSplit.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.mor;
package org.apache.hudi.table.format.mor;
import org.apache.hudi.common.util.Option;

2
hudi-flink/src/main/java/org/apache/hudi/source/format/mor/MergeOnReadTableState.java → hudi-flink/src/main/java/org/apache/hudi/table/format/mor/MergeOnReadTableState.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source.format.mor;
package org.apache.hudi.table.format.mor;
import org.apache.flink.table.types.logical.RowType;

9
hudi-flink/src/main/java/org/apache/hudi/util/RowDataToAvroConverters.java
View File

@@ -18,6 +18,10 @@
package org.apache.hudi.util;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.Utf8;
import org.apache.flink.annotation.Internal;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
@@ -28,11 +32,6 @@ import org.apache.flink.table.types.logical.ArrayType;
import org.apache.flink.table.types.logical.LogicalType;
import org.apache.flink.table.types.logical.RowType;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.util.Utf8;
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.util.ArrayList;

2
hudi-flink/src/main/java/org/apache/hudi/util/StreamerUtil.java
View File

@@ -30,12 +30,12 @@ import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.TablePathUtils;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.SimpleAvroKeyGenerator;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.schema.FilebasedSchemaProvider;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.table.action.compact.CompactionTriggerStrategy;

2
hudi-flink/src/main/resources/META-INF/services/org.apache.flink.table.factories.TableFactory
View File

@@ -14,4 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.hudi.factory.HoodieTableFactory
org.apache.hudi.table.HoodieTableFactory

22
hudi-flink/src/test/java/org/apache/hudi/operator/StreamWriteITCase.java → hudi-flink/src/test/java/org/apache/hudi/sink/StreamWriteITCase.java
View File

@@ -16,24 +16,24 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.operator.compact.CompactFunction;
import org.apache.hudi.operator.compact.CompactionCommitEvent;
import org.apache.hudi.operator.compact.CompactionCommitSink;
import org.apache.hudi.operator.compact.CompactionPlanEvent;
import org.apache.hudi.operator.compact.CompactionPlanOperator;
import org.apache.hudi.operator.partitioner.BucketAssignFunction;
import org.apache.hudi.operator.transform.RowDataToHoodieFunction;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.sink.CommitSink;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.sink.compact.CompactFunction;
import org.apache.hudi.sink.compact.CompactionCommitEvent;
import org.apache.hudi.sink.compact.CompactionCommitSink;
import org.apache.hudi.sink.compact.CompactionPlanEvent;
import org.apache.hudi.sink.compact.CompactionPlanOperator;
import org.apache.hudi.sink.partitioner.BucketAssignFunction;
import org.apache.hudi.sink.transform.RowDataToHoodieFunction;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
import org.apache.hudi.utils.source.ContinuousFileSource;
import org.apache.flink.api.common.JobStatus;

8
hudi-flink/src/test/java/org/apache/hudi/operator/StreamWriteOperatorCoordinatorTest.java → hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
View File

@@ -16,16 +16,16 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.operator.event.BatchWriteSuccessEvent;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.sink.event.BatchWriteSuccessEvent;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;
import org.apache.hadoop.fs.FileSystem;
@@ -49,7 +49,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
/**
* Test cases for StreamingSinkOperatorCoordinator.
*/
public class StreamWriteOperatorCoordinatorTest {
public class TestStreamWriteOperatorCoordinator {
private StreamWriteOperatorCoordinator coordinator;
@TempDir

11
hudi-flink/src/test/java/org/apache/hudi/operator/TestWriteCopyOnWrite.java → hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.client.HoodieFlinkWriteClient;
import org.apache.hudi.client.WriteStatus;
@@ -24,11 +24,12 @@ import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.operator.event.BatchWriteSuccessEvent;
import org.apache.hudi.operator.utils.StreamWriteFunctionWrapper;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.sink.event.BatchWriteSuccessEvent;
import org.apache.hudi.sink.utils.StreamWriteFunctionWrapper;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.operators.coordination.OperatorEvent;

4
hudi-flink/src/test/java/org/apache/hudi/operator/TestWriteMergeOnRead.java → hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnRead.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
@@ -28,9 +28,9 @@ import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.table.HoodieFlinkTable;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestData;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileSystem;

3
hudi-flink/src/test/java/org/apache/hudi/operator/TestWriteMergeOnReadWithCompact.java → hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
View File

@@ -16,9 +16,10 @@
* limitations under the License.
*/
package org.apache.hudi.operator;
package org.apache.hudi.sink;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.flink.configuration.Configuration;
import org.junit.jupiter.api.Disabled;

4
hudi-flink/src/test/java/org/apache/hudi/operator/partitioner/TestBucketAssigner.java → hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java
View File

@@ -16,18 +16,18 @@
* limitations under the License.
*/
package org.apache.hudi.operator.partitioner;
package org.apache.hudi.sink.partitioner;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
import org.apache.hudi.common.config.SerializableConfiguration;
import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.table.action.commit.BucketInfo;
import org.apache.hudi.table.action.commit.BucketType;
import org.apache.hudi.table.action.commit.SmallFile;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.flink.configuration.Configuration;
import org.junit.jupiter.api.BeforeEach;

2
hudi-flink/src/test/java/org/apache/hudi/source/TestJsonStringToHoodieRecordMapFunction.java → hudi-flink/src/test/java/org/apache/hudi/sink/transform/TestJsonStringToHoodieRecordMapFunction.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.source;
package org.apache.hudi.sink.transform;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.common.model.HoodieRecord;

14
hudi-flink/src/test/java/org/apache/hudi/operator/utils/CompactFunctionWrapper.java → hudi-flink/src/test/java/org/apache/hudi/sink/utils/CompactFunctionWrapper.java
View File

@@ -16,14 +16,14 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.sink.utils;
import org.apache.hudi.avro.model.HoodieCompactionPlan;
import org.apache.hudi.operator.compact.CompactFunction;
import org.apache.hudi.operator.compact.CompactionCommitEvent;
import org.apache.hudi.operator.compact.CompactionCommitSink;
import org.apache.hudi.operator.compact.CompactionPlanEvent;
import org.apache.hudi.operator.compact.CompactionPlanOperator;
import org.apache.hudi.sink.compact.CompactFunction;
import org.apache.hudi.sink.compact.CompactionCommitEvent;
import org.apache.hudi.sink.compact.CompactionCommitSink;
import org.apache.hudi.sink.compact.CompactionPlanEvent;
import org.apache.hudi.sink.compact.CompactionPlanOperator;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;
@@ -43,7 +43,7 @@ import java.util.ArrayList;
import java.util.List;
/**
* A wrapper class to manipulate the {@link org.apache.hudi.operator.compact.CompactFunction} instance for testing.
* A wrapper class to manipulate the {@link org.apache.hudi.sink.compact.CompactFunction} instance for testing.
*/
public class CompactFunctionWrapper {
private final Configuration conf;

2
hudi-flink/src/test/java/org/apache/hudi/operator/utils/MockFunctionInitializationContext.java → hudi-flink/src/test/java/org/apache/hudi/sink/utils/MockFunctionInitializationContext.java
View File

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.sink.utils;
import org.apache.flink.api.common.state.KeyedStateStore;
import org.apache.flink.runtime.state.FunctionInitializationContext;

2
hudi-flink/src/test/java/org/apache/hudi/operator/utils/MockMapState.java → hudi-flink/src/test/java/org/apache/hudi/sink/utils/MockMapState.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.sink.utils;
import org.apache.flink.api.common.state.MapState;

2
hudi-flink/src/test/java/org/apache/hudi/operator/utils/MockOperatorStateStore.java → hudi-flink/src/test/java/org/apache/hudi/sink/utils/MockOperatorStateStore.java
View File

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.sink.utils;
import org.apache.flink.api.common.state.AggregatingState;
import org.apache.flink.api.common.state.AggregatingStateDescriptor;

2
hudi-flink/src/test/java/org/apache/hudi/operator/utils/MockStreamingRuntimeContext.java → hudi-flink/src/test/java/org/apache/hudi/sink/utils/MockStreamingRuntimeContext.java
View File

@@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.sink.utils;
import org.apache.flink.api.common.ExecutionConfig;
import org.apache.flink.api.common.state.KeyedStateStore;

15
hudi-flink/src/test/java/org/apache/hudi/operator/utils/StreamWriteFunctionWrapper.java → hudi-flink/src/test/java/org/apache/hudi/sink/utils/StreamWriteFunctionWrapper.java
View File

@@ -16,18 +16,19 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.sink.utils;
import org.apache.hudi.client.HoodieFlinkWriteClient;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.StreamWriteFunction;
import org.apache.hudi.operator.StreamWriteOperatorCoordinator;
import org.apache.hudi.operator.event.BatchWriteSuccessEvent;
import org.apache.hudi.operator.partitioner.BucketAssignFunction;
import org.apache.hudi.operator.transform.RowDataToHoodieFunction;
import org.apache.hudi.sink.StreamWriteFunction;
import org.apache.hudi.sink.StreamWriteOperatorCoordinator;
import org.apache.hudi.sink.event.BatchWriteSuccessEvent;
import org.apache.hudi.sink.partitioner.BucketAssignFunction;
import org.apache.hudi.sink.transform.RowDataToHoodieFunction;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.runtime.io.disk.iomanager.IOManager;

9
hudi-flink/src/test/java/org/apache/hudi/source/TestStreamReadMonitoringFunction.java
View File

@@ -18,12 +18,11 @@
package org.apache.hudi.source;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.StreamReadMonitoringFunction;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.source.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.table.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
import org.apache.hudi.utils.TestUtils;
import org.apache.flink.configuration.Configuration;

16
hudi-flink/src/test/java/org/apache/hudi/source/TestStreamReadOperator.java
View File

@@ -20,18 +20,16 @@ package org.apache.hudi.source;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.StreamReadMonitoringFunction;
import org.apache.hudi.operator.StreamReadOperator;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.source.format.FilePathUtils;
import org.apache.hudi.source.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.source.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.source.format.mor.MergeOnReadTableState;
import org.apache.hudi.table.format.FilePathUtils;
import org.apache.hudi.table.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.table.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.table.format.mor.MergeOnReadTableState;
import org.apache.hudi.util.AvroSchemaConverter;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
import org.apache.hudi.utils.TestUtils;
import org.apache.avro.Schema;

10
hudi-flink/src/test/java/org/apache/hudi/source/HoodieDataSourceITCase.java → hudi-flink/src/test/java/org/apache/hudi/table/HoodieDataSourceITCase.java
View File

@@ -16,11 +16,11 @@
* limitations under the License.
*/
package org.apache.hudi.source;
package org.apache.hudi.table;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
import org.apache.hudi.utils.TestUtils;
import org.apache.hudi.utils.factory.CollectSinkTableFactory;
@@ -46,7 +46,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.apache.hudi.operator.utils.TestData.assertRowsEquals;
import static org.apache.hudi.utils.TestData.assertRowsEquals;
/**
* IT cases for Hoodie table source and sink.

8
hudi-flink/src/test/java/org/apache/hudi/factory/TestHoodieTableFactory.java → hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableFactory.java
View File

@@ -16,13 +16,11 @@
* limitations under the License.
*/
package org.apache.hudi.factory;
package org.apache.hudi.table;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.sink.HoodieTableSink;
import org.apache.hudi.source.HoodieTableSource;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.configuration.ReadableConfig;

10
hudi-flink/src/test/java/org/apache/hudi/source/TestHoodieTableSource.java → hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
View File

@@ -16,14 +16,14 @@
* limitations under the License.
*/
package org.apache.hudi.source;
package org.apache.hudi.table;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.source.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.table.format.mor.MergeOnReadInputFormat;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
import org.apache.flink.api.common.io.FileInputFormat;
import org.apache.flink.api.common.io.InputFormat;

10
hudi-flink/src/test/java/org/apache/hudi/source/format/TestInputFormat.java → hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java
View File

@@ -16,14 +16,14 @@
* limitations under the License.
*/
package org.apache.hudi.source.format;
package org.apache.hudi.table.format;
import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.operator.utils.TestData;
import org.apache.hudi.source.HoodieTableSource;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.table.HoodieTableSource;
import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
import org.apache.flink.api.common.io.InputFormat;
import org.apache.flink.configuration.Configuration;

4
hudi-flink/src/test/java/org/apache/hudi/operator/utils/TestConfigurations.java → hudi-flink/src/test/java/org/apache/hudi/utils/TestConfigurations.java
View File

@@ -16,9 +16,9 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.utils;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.streamer.FlinkStreamerConfig;
import org.apache.hudi.utils.factory.CollectSinkTableFactory;
import org.apache.hudi.utils.factory.ContinuousFileSourceFactory;

5
hudi-flink/src/test/java/org/apache/hudi/operator/utils/TestData.java → hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java
View File

@@ -16,7 +16,7 @@
* limitations under the License.
*/
package org.apache.hudi.operator.utils;
package org.apache.hudi.utils;
import org.apache.hudi.client.FlinkTaskContextSupplier;
import org.apache.hudi.client.common.HoodieFlinkEngineContext;
@@ -25,7 +25,8 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.sink.utils.StreamWriteFunctionWrapper;
import org.apache.hudi.table.HoodieFlinkTable;
import org.apache.avro.Schema;

6
hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java
View File

@@ -19,9 +19,9 @@
package org.apache.hudi.utils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.operator.StreamReadMonitoringFunction;
import org.apache.hudi.source.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.source.StreamReadMonitoringFunction;
import org.apache.hudi.table.format.mor.MergeOnReadInputSplit;
import org.apache.hudi.util.StreamerUtil;
import org.apache.flink.configuration.Configuration;

2
hudi-flink/src/test/java/org/apache/hudi/utils/factory/CollectSinkTableFactory.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.utils.factory;
import org.apache.hudi.operator.utils.TestConfigurations;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.flink.api.common.state.ListState;
import org.apache.flink.api.common.state.ListStateDescriptor;

2
hudi-flink/src/test/java/org/apache/hudi/utils/factory/ContinuousFileSourceFactory.java
View File

@@ -18,7 +18,7 @@
package org.apache.hudi.utils.factory;
import org.apache.hudi.operator.FlinkOptions;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.utils.source.ContinuousFileSource;
import org.apache.flink.configuration.Configuration;

2
hudi-flink/src/test/java/org/apache/hudi/utils/source/ContinuousFileSource.java
View File

@@ -153,7 +153,7 @@ public class ContinuousFileSource implements StreamTableSource<RowData> {
private void loadDataBuffer() {
this.dataBuffer = new ArrayList<>();
try (BufferedReader reader =
new BufferedReader(new FileReader(this.path.getPath()))) {
new BufferedReader(new FileReader(this.path.toString()))) {
String line = reader.readLine();
while (line != null) {
this.dataBuffer.add(line);

2
hudi-flink/src/test/resources/META-INF/services/org.apache.flink.table.factories.TableFactory
View File

@@ -14,5 +14,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.hudi.factory.HoodieTableFactory
org.apache.hudi.table.HoodieTableFactory
org.apache.hudi.utils.factory.ContinuousFileSourceFactory

163
packaging/hudi-flink-bundle/pom.xml
View File

@@ -31,6 +31,8 @@
<properties>
<checkstyle.skip>true</checkstyle.skip>
<main.basedir>${project.parent.basedir}</main.basedir>
<flink.bundle.hive.shade.prefix>org.apache.hudi.</flink.bundle.hive.shade.prefix>
<javax.servlet.version>3.1.0</javax.servlet.version>
</properties>
<build>
@@ -70,6 +72,7 @@
<include>org.apache.hudi:hudi-client-common</include>
<include>org.apache.hudi:hudi-flink-client</include>
<include>org.apache.hudi:hudi-flink_${scala.binary.version}</include>
<include>org.apache.hudi:hudi-hadoop-mr</include>
<include>org.apache.hudi:hudi-timeline-service</include>
<include>com.yammer.metrics:metrics-core</include>
@@ -81,7 +84,14 @@
<include>org.apache.httpcomponents:httpcore</include>
<include>org.apache.httpcomponents:fluent-hc</include>
<include>org.antlr:stringtemplate</include>
<!-- Parquet -->
<include>org.apache.parquet:parquet-avro</include>
<include>org.apache.parquet:parquet-hadoop</include>
<include>org.apache.parquet:parquet-column</include>
<include>org.apache.parquet:parquet-common</include>
<include>org.apache.parquet:parquet-format</include>
<include>org.apache.parquet:parquet-encoding</include>
<include>org.apache.avro:avro</include>
<include>joda-time:joda-time</include>
<include>com.fasterxml.jackson.core:jackson-annotations</include>
@@ -103,11 +113,25 @@
<include>io.prometheus:simpleclient_pushgateway</include>
<include>io.prometheus:simpleclient_common</include>
<include>com.yammer.metrics:metrics-core</include>
<include>org.apache.flink:flink-connector-kafka_${scala.binary.version}</include>
<include>org.apache.flink:flink-connector-kafka-base_${scala.binary.version}</include>
<include>org.apache.kafka:kafka_${scala.binary.version}</include>
<include>com.101tec:zkclient</include>
<include>org.apache.kafka:kafka-clients</include>
<!-- Used for HUDI TimelineService -->
<include>org.eclipse.jetty:*</include>
<include>org.eclipse.jetty.websocket:*</include>
<include>javax.servlet:javax.servlet-api</include>
<include>org.apache.flink:flink-connector-kafka_${scala.binary.version}</include>
<include>org.apache.flink:flink-connector-kafka-base_${scala.binary.version}</include>
<include>org.apache.flink:flink-hadoop-compatibility_${scala.binary.version}</include>
<include>org.apache.flink:flink-avro</include>
<include>org.apache.flink:flink-json</include>
<include>org.apache.flink:flink-parquet_${scala.binary.version}</include>
<include>org.apache.hive:hive-common</include>
<include>org.apache.hive:hive-exec</include>
<include>org.apache.hive:hive-metastore</include>
<include>org.apache.hive:hive-jdbc</include>
<include>org.apache.hbase:hbase-client</include>
<include>org.apache.hbase:hbase-common</include>
@@ -125,6 +149,34 @@
<pattern>com.beust.jcommander.</pattern>
<shadedPattern>org.apache.hudi.com.beust.jcommander.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.jdbc.</pattern>
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hive.jdbc.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.metastore.</pattern>
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.metastore.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.common.</pattern>
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.common.</pattern>
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.common.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.conf.</pattern>
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.conf.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hive.service.</pattern>
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>org.apache.hadoop.hive.service.</pattern>
<shadedPattern>${flink.bundle.hive.shade.prefix}org.apache.hadoop.hive.service.</shadedPattern>
</relocation>
<relocation>
<pattern>com.codahale.metrics.</pattern>
<shadedPattern>org.apache.hudi.com.codahale.metrics.</shadedPattern>
@@ -133,6 +185,10 @@
<pattern>org.apache.commons.codec.</pattern>
<shadedPattern>org.apache.hudi.org.apache.commons.codec.</shadedPattern>
</relocation>
<relocation>
<pattern>org.eclipse.jetty.</pattern>
<shadedPattern>org.apache.hudi.org.apache.jetty.</shadedPattern>
</relocation>
</relocations>
<filters>
<filter>
@@ -142,6 +198,8 @@
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/services/javax.*</exclude>
<!-- exclude the test dynamic factory -->
<exclude>META-INF/services/org.apache.flink.table.factories.Factory</exclude>
</excludes>
</filter>
</filters>
@@ -183,6 +241,21 @@
<artifactId>hudi-flink_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-hadoop-mr</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hudi</groupId>
<artifactId>hudi-timeline-service</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>${javax.servlet.version}</version>
</dependency>
<!-- Flink -->
<dependency>
@@ -196,6 +269,30 @@
<version>${flink.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-hadoop-compatibility_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-avro</artifactId>
<version>${flink.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-parquet_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-json</artifactId>
<version>${flink.version}</version>
<scope>compile</scope>
</dependency>
<!-- Parquet -->
<dependency>
@@ -208,9 +305,69 @@
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<!-- Override the version to be same with Flink avro -->
<version>1.10.0</version>
<scope>compile</scope>
</dependency>
<!-- Hive -->
<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>javax.mail</groupId>
<artifactId>mail</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty.aggregate</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-jdbc</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>javax.servlet</groupId>
<artifactId>*</artifactId>
</exclusion>
<exclusion>
<groupId>javax.servlet.jsp</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>${hive.groupid}</groupId>
<artifactId>hive-common</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.jetty.orbit</groupId>
<artifactId>javax.servlet</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>joda-time</groupId>
<artifactId>joda-time</artifactId>