1
0

[HUDI-311] : Support for AWS Database Migration Service in DeltaStreamer

- Add a transformer class, that adds `Op` fiels if not found in input frame
 - Add a payload implementation, that issues deletes when Op=D
 - Remove Parquet as a top level source type, consolidate with RowSource
 - Made delta streamer work without a property file, simply using overridden cli options
 - Unit tests for transformer/payload classes
This commit is contained in:
vinoth chandar
2019-12-22 23:33:35 -08:00
committed by vinoth chandar
parent 313fab5fd1
commit 350b0ecb4d
11 changed files with 239 additions and 47 deletions

View File

@@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hudi.payload;
import org.apache.hudi.OverwriteWithLatestAvroPayload;
import org.apache.hudi.common.util.Option;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import java.io.IOException;
/**
* Provides support for seamlessly applying changes captured via Amazon Database Migration Service onto S3.
*
* Typically, we get the following pattern of full change records corresponding to DML against the
* source database
*
* - Full load records with no `Op` field
* - For inserts against the source table, records contain full after image with `Op=I`
* - For updates against the source table, records contain full after image with `Op=U`
* - For deletes against the source table, records contain full before image with `Op=D`
*
* This payload implementation will issue matching insert, delete, updates against the hudi dataset
*
*/
public class AWSDmsAvroPayload extends OverwriteWithLatestAvroPayload {
public static final String OP_FIELD = "Op";
public AWSDmsAvroPayload(GenericRecord record, Comparable orderingVal) {
super(record, orderingVal);
}
public AWSDmsAvroPayload(Option<GenericRecord> record) {
this(record.get(), (record1) -> 0); // natural order
}
@Override
public Option<IndexedRecord> combineAndGetUpdateValue(IndexedRecord currentValue, Schema schema)
throws IOException {
IndexedRecord insertValue = getInsertValue(schema).get();
boolean delete = false;
if (insertValue instanceof GenericRecord) {
GenericRecord record = (GenericRecord) insertValue;
delete = record.get(OP_FIELD) != null && record.get(OP_FIELD).toString().equalsIgnoreCase("D");
}
return delete ? Option.empty() : Option.of(insertValue);
}
}