lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2795] Add mechanism to safely update,delete and recover table properties (#4038)

Browse Source 
* [HUDI-2795] Add mechanism to safely update,delete and recover table properties

  - Fail safe mechanism, that lets queries succeed off a backup file
  - Readers who are not upgraded to this version of code will just fail until recovery is done.
  - Added unit tests that exercises all these scenarios.
  - Adding CLI for recovery, updation to table command.
  - [Pending] Add some hash based verfication to ensure any rare partial writes for HDFS

* Fixing upgrade/downgrade infrastructure to use new updation method
This commit is contained in:
vinoth chandar
2021-11-20 08:07:40 -08:00
committed by GitHub
parent f4b974ac7b
commit ae0c67d9fc
7 changed files with 337 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hudi-cli/src/main/java/org/apache/hudi/cli/commands/RepairsCommand.java
View File

@@ -156,7 +156,7 @@ public class RepairsCommand implements CommandMarker {
newProps.load(new FileInputStream(new File(overwriteFilePath)));
Map<String, String> oldProps = client.getTableConfig().propsMap();
Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
HoodieTableConfig.createHoodieProperties(client.getFs(), metaPathDir, newProps);
HoodieTableConfig.create(client.getFs(), metaPathDir, newProps);
TreeSet<String> allPropKeys = new TreeSet<>();
allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet()));

73
hudi-cli/src/main/java/org/apache/hudi/cli/commands/TableCommand.java
View File

@@ -20,13 +20,16 @@ package org.apache.hudi.cli.commands;
import org.apache.hudi.cli.HoodieCLI;
import org.apache.hudi.cli.HoodiePrintHelper;
import org.apache.hudi.cli.HoodieTableHeaderFields;
import org.apache.hudi.cli.TableHeader;
import org.apache.hudi.common.fs.ConsistencyGuardConfig;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.springframework.shell.core.CommandMarker;
@@ -35,12 +38,21 @@ import org.springframework.shell.core.annotation.CliOption;
import org.springframework.stereotype.Component;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.stream.Collectors;
import static org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME;
/**
* CLI command to display hudi table options.
@@ -170,6 +182,67 @@ public class TableCommand implements CommandMarker {
}
}
@CliCommand(value = "table recover-configs", help = "Recover table configs, from update/delete that failed midway.")
public String recoverTableConfig() throws IOException {
HoodieCLI.refreshTableMetadata();
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
HoodieTableConfig.recover(client.getFs(), metaPathDir);
return descTable();
}
@CliCommand(value = "table update-configs", help = "Update the table configs with configs with provided file.")
public String updateTableConfig(
@CliOption(key = {"props-file"}, mandatory = true, help = "Path to a properties file on local filesystem")
final String updatePropsFilePath) throws IOException {
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
Map<String, String> oldProps = client.getTableConfig().propsMap();
Properties updatedProps = new Properties();
updatedProps.load(new FileInputStream(updatePropsFilePath));
Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
HoodieTableConfig.update(client.getFs(), metaPathDir, updatedProps);
HoodieCLI.refreshTableMetadata();
Map<String, String> newProps = HoodieCLI.getTableMetaClient().getTableConfig().propsMap();
return renderOldNewProps(newProps, oldProps);
}
@CliCommand(value = "table delete-configs", help = "Delete the supplied table configs from the table.")
public String deleteTableConfig(
@CliOption(key = {"comma-separated-configs"}, mandatory = true, help = "Comma separated list of configs to delete.")
final String csConfigs) {
HoodieTableMetaClient client = HoodieCLI.getTableMetaClient();
Map<String, String> oldProps = client.getTableConfig().propsMap();
Set<String> deleteConfigs = Arrays.stream(csConfigs.split(",")).collect(Collectors.toSet());
Path metaPathDir = new Path(client.getBasePath(), METAFOLDER_NAME);
HoodieTableConfig.delete(client.getFs(), metaPathDir, deleteConfigs);
HoodieCLI.refreshTableMetadata();
Map<String, String> newProps = HoodieCLI.getTableMetaClient().getTableConfig().propsMap();
return renderOldNewProps(newProps, oldProps);
}
private static String renderOldNewProps(Map<String, String> newProps, Map<String, String> oldProps) {
TreeSet<String> allPropKeys = new TreeSet<>();
allPropKeys.addAll(newProps.keySet().stream().map(Object::toString).collect(Collectors.toSet()));
allPropKeys.addAll(oldProps.keySet());
String[][] rows = new String[allPropKeys.size()][];
int ind = 0;
for (String propKey : allPropKeys) {
String[] row = new String[]{
propKey,
oldProps.getOrDefault(propKey, "null"),
newProps.getOrDefault(propKey, "null")
};
rows[ind++] = row;
}
return HoodiePrintHelper.print(new String[] {HoodieTableHeaderFields.HEADER_HOODIE_PROPERTY,
HoodieTableHeaderFields.HEADER_OLD_VALUE, HoodieTableHeaderFields.HEADER_NEW_VALUE}, rows);
}
/**
* Use Streams when you are dealing with raw data.
* @param filePath output file path.