1
0

CR feedback

This commit is contained in:
Vinoth Chandar
2017-04-03 18:17:56 -07:00
committed by vinoth chandar
parent e0fc4ec38e
commit 2b6322318c
4 changed files with 8 additions and 5 deletions

View File

@@ -13,4 +13,5 @@ issues or pull requests against this repo. Before you do so, please sign the
Also, be sure to write unit tests for your bug fix or feature to show that it works as expected.
If you want to participate in day-day conversations, please join our [slack group](https://hoodielib.slack.com/x-147852474016-157730502112/signup).
If you are from select pre-listed email domains, you can self signup. Others, please add your email onto this [issue](https://github.com/uber/hoodie/issues/143)

View File

@@ -50,11 +50,11 @@ public class RepairsCommand implements CommandMarker {
@CliCommand(value = "repair deduplicate", help = "De-duplicate a partition path contains duplicates & produce repaired files to replace with")
public String deduplicate(
@CliOption(key = {
"duplicatedPartitionPath"}, help = "Partition Path containing the duplicates")
"duplicatedPartitionPath"}, help = "Partition Path containing the duplicates", mandatory = true)
final String duplicatedPartitionPath,
@CliOption(key = {"repairedOutputPath"}, help = "Location to place the repaired files")
@CliOption(key = {"repairedOutputPath"}, help = "Location to place the repaired files", mandatory = true)
final String repairedOutputPath,
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path")
@CliOption(key = {"sparkProperties"}, help = "Spark Properites File Path", mandatory = true)
final String sparkPropertiesPath) throws Exception {
SparkLauncher sparkLauncher = SparkUtil.initLauncher(sparkPropertiesPath);
sparkLauncher

View File

@@ -88,7 +88,7 @@ public class HoodieUpdateHandle <T extends HoodieRecordPayload> extends HoodieIO
commitTime,
new Path(config.getBasePath()),
new Path(config.getBasePath(), record.getPartitionPath()));
partitionMetadata.trySave(TaskContext.getPartitionId());
partitionMetadata.trySave(TaskContext.getPartitionId());
oldFilePath = new Path(
config.getBasePath() + "/" + record.getPartitionPath() + "/"

View File

@@ -78,7 +78,9 @@ public class HoodiePartitionMetadata {
}
/**
* Write the metadata safely into partition
* Write the metadata safely into partition atomically.
*
* @param taskPartitionId
*/
public void trySave(int taskPartitionId) {
Path tmpMetaPath = new Path(partitionPath, HoodiePartitionMetadata.HOODIE_PARTITION_METAFILE + "_" + taskPartitionId);