CR feedback
This commit is contained in:
committed by
vinoth chandar
parent
e0fc4ec38e
commit
2b6322318c
@@ -13,4 +13,5 @@ issues or pull requests against this repo. Before you do so, please sign the
|
||||
Also, be sure to write unit tests for your bug fix or feature to show that it works as expected.
|
||||
|
||||
If you want to participate in day-day conversations, please join our [slack group](https://hoodielib.slack.com/x-147852474016-157730502112/signup).
|
||||
If you are from select pre-listed email domains, you can self signup. Others, please add your email onto this [issue](https://github.com/uber/hoodie/issues/143)
|
||||
|
||||
|
||||
@@ -50,11 +50,11 @@ public class RepairsCommand implements CommandMarker {
|
||||
@CliCommand(value = "repair deduplicate", help = "De-duplicate a partition path contains duplicates & produce repaired files to replace with")
|
||||
public String deduplicate(
|
||||
@CliOption(key = {
|
||||
"duplicatedPartitionPath"}, help = "Partition Path containing the duplicates")
|
||||
"duplicatedPartitionPath"}, help = "Partition Path containing the duplicates", mandatory = true)
|
||||
final String duplicatedPartitionPath,
|
||||
@CliOption(key = {"repairedOutputPath"}, help = "Location to place the repaired files")
|
||||
@CliOption(key = {"repairedOutputPath"}, help = "Location to place the repaired files", mandatory = true)
|
||||
final String repairedOutputPath,
|
||||
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path")
|
||||
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path", mandatory = true)
|
||||
final String sparkPropertiesPath) throws Exception {
|
||||
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
|
||||
sparkLauncher
|
||||
|
||||
@@ -88,7 +88,7 @@ public class HoodieUpdateHandle <T extends HoodieRecordPayload> extends HoodieIO
|
||||
commitTime,
|
||||
new Path(config.getBasePath()),
|
||||
new Path(config.getBasePath(), record.getPartitionPath()));
|
||||
partitionMetadata.trySave(TaskContext.getPartitionId());
|
||||
partitionMetadata.trySave(TaskContext.getPartitionId());
|
||||
|
||||
oldFilePath = new Path(
|
||||
config.getBasePath() + "/" + record.getPartitionPath() + "/"
|
||||
|
||||
@@ -78,7 +78,9 @@ public class HoodiePartitionMetadata {
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the metadata safely into partition
|
||||
* Write the metadata safely into partition atomically.
|
||||
*
|
||||
* @param taskPartitionId
|
||||
*/
|
||||
public void trySave(int taskPartitionId) {
|
||||
Path tmpMetaPath = new Path(partitionPath, HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE + "_" + taskPartitionId);
|
||||
|
||||
Reference in New Issue
Block a user