1
0

[HUDI-394] Provide a basic implementation of test suite

This commit is contained in:
Nishith Agarwal
2019-11-01 12:40:09 -07:00
committed by n3nash
parent d5b593b7d9
commit 2fc2b01d86
102 changed files with 8633 additions and 64 deletions

View File

@@ -95,7 +95,7 @@ public class QuickstartUtils {
}
public static GenericRecord generateGenericRecord(String rowKey, String riderName, String driverName,
double timestamp) {
double timestamp) {
GenericRecord rec = new GenericData.Record(avroSchema);
rec.put("uuid", rowKey);
rec.put("ts", timestamp);
@@ -221,4 +221,4 @@ public class QuickstartUtils {
demoConfigs.put("hoodie.upsert.shuffle.parallelism", "2");
return demoConfigs;
}
}
}

View File

@@ -37,6 +37,7 @@ public class ComplexKeyGenerator extends KeyGenerator {
private static final String DEFAULT_PARTITION_PATH = "default";
private static final String DEFAULT_PARTITION_PATH_SEPARATOR = "/";
public static final String DEFAULT_RECORD_KEY_SEPARATOR = ":";
protected static final String NULL_RECORDKEY_PLACEHOLDER = "__null__";
protected static final String EMPTY_RECORDKEY_PLACEHOLDER = "__empty__";
@@ -98,8 +99,16 @@ public class ComplexKeyGenerator extends KeyGenerator {
recordKey.deleteCharAt(recordKey.length() - 1);
if (keyIsNullEmpty) {
throw new HoodieKeyException("recordKey values: \"" + recordKey + "\" for fields: "
+ recordKeyFields.toString() + " cannot be entirely null or empty.");
+ recordKeyFields.toString() + " cannot be entirely null or empty.");
}
return recordKey.toString();
}
}
public List<String> getRecordKeyFields() {
return recordKeyFields;
}
public List<String> getPartitionPathFields() {
return partitionPathFields;
}
}

View File

@@ -40,4 +40,4 @@ public abstract class KeyGenerator implements Serializable {
* Generate a Hoodie Key out of provided generic record.
*/
public abstract HoodieKey getKey(GenericRecord record);
}
}

View File

@@ -48,4 +48,4 @@ public class NonpartitionedKeyGenerator extends KeyGenerator {
}
return new HoodieKey(recordKey, EMPTY_PARTITION);
}
}
}

View File

@@ -73,4 +73,12 @@ public class SimpleKeyGenerator extends KeyGenerator {
}
return recordKey;
}
}
public String getRecordKeyField() {
return recordKeyField;
}
public String getPartitionPathField() {
return partitionPathField;
}
}

View File

@@ -149,7 +149,6 @@ public class TimestampBasedKeyGenerator extends SimpleKeyGenerator {
"Unexpected type for partition field: " + partitionVal.getClass().getName());
}
DateTime timestamp = new DateTime(timeMs, outputDateTimeZone);
return hiveStylePartitioning ? partitionPathField + "=" + timestamp.toString(partitionFormatter)
: timestamp.toString(partitionFormatter);
} catch (Exception e) {