[HUDI-311] : Support for AWS Database Migration Service in DeltaStreamer
- Add a transformer class, that adds `Op` fiels if not found in input frame - Add a payload implementation, that issues deletes when Op=D - Remove Parquet as a top level source type, consolidate with RowSource - Made delta streamer work without a property file, simply using overridden cli options - Unit tests for transformer/payload classes
This commit is contained in:
committed by
vinoth chandar
parent
313fab5fd1
commit
350b0ecb4d
@@ -60,6 +60,17 @@ public class DFSPropertiesConfiguration {
|
||||
visitFile(rootFile);
|
||||
}
|
||||
|
||||
public DFSPropertiesConfiguration(FileSystem fs, Path rootFile) {
|
||||
this(fs, rootFile, new TypedProperties());
|
||||
}
|
||||
|
||||
public DFSPropertiesConfiguration() {
|
||||
this.fs = null;
|
||||
this.rootFile = null;
|
||||
this.props = new TypedProperties();
|
||||
this.visitedFiles = new HashSet<>();
|
||||
}
|
||||
|
||||
private String[] splitProperty(String line) {
|
||||
int ind = line.indexOf('=');
|
||||
String k = line.substring(0, ind).trim();
|
||||
@@ -106,10 +117,6 @@ public class DFSPropertiesConfiguration {
|
||||
}
|
||||
}
|
||||
|
||||
public DFSPropertiesConfiguration(FileSystem fs, Path rootFile) {
|
||||
this(fs, rootFile, new TypedProperties());
|
||||
}
|
||||
|
||||
public TypedProperties getConfig() {
|
||||
return props;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user