lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2809] Introduce a checksum mechanism for validating hoodie.properties (#4712)

Browse Source 
Fix dependency conflict

Fix repairs command

Implement putIfAbsent for DDB lock provider

Add upgrade step and validate while fetching configs

Validate checksum for latest table version only while fetching config

Move generateChecksum to BinaryUtil

Rebase and resolve conflict

Fix table version check
This commit is contained in:
Sagar Sumit
2022-02-18 10:17:06 +05:30
committed by GitHub
parent 2844a77b43
commit ed106f671e
20 changed files with 320 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
hudi-common/src/test/java/org/apache/hudi/common/properties/TestTypedProperties.java
View File

@@ -19,8 +19,10 @@
package org.apache.hudi.common.properties;
import org.apache.hudi.common.config.TypedProperties;
import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.util.Properties;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -81,4 +83,25 @@ public class TestTypedProperties {
assertEquals(true, typedProperties.getBoolean("key1", false));
assertEquals(false, typedProperties.getBoolean("key2", false));
}
@Test
public void testPropertiesOrder() throws IOException {
Properties properties = new TypedProperties();
properties.put("key0", "true");
properties.put("key1", "false");
properties.put("key2", "true");
properties.put("key3", "false");
properties.put("key4", "true");
properties.put("key5", "true");
properties.put("key6", "false");
properties.put("key7", "true");
properties.put("key8", "false");
properties.put("key9", "true");
TypedProperties typedProperties = new TypedProperties(properties);
String[] props = typedProperties.stringPropertyNames().toArray(new String[0]);
for (int i = 0; i < props.length; i++) {
assertEquals(String.format("key%d", i), props[i]);
}
}
}

10
hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
View File

@@ -64,7 +64,7 @@ public class TestHoodieTableConfig extends HoodieCommonTestHarness {
public void testCreate() throws IOException {
assertTrue(fs.exists(new Path(metaPath, HoodieTableConfig.HOODIE_PROPERTIES_FILE)));
HoodieTableConfig config = new HoodieTableConfig(fs, metaPath.toString(), null);
assertEquals(4, config.getProps().size());
assertEquals(5, config.getProps().size());
}
@Test
@@ -77,7 +77,7 @@ public class TestHoodieTableConfig extends HoodieCommonTestHarness {
assertTrue(fs.exists(cfgPath));
assertFalse(fs.exists(backupCfgPath));
HoodieTableConfig config = new HoodieTableConfig(fs, metaPath.toString(), null);
assertEquals(5, config.getProps().size());
assertEquals(6, config.getProps().size());
assertEquals("test-table2", config.getTableName());
assertEquals("new_field", config.getPreCombineField());
}
@@ -90,7 +90,7 @@ public class TestHoodieTableConfig extends HoodieCommonTestHarness {
assertTrue(fs.exists(cfgPath));
assertFalse(fs.exists(backupCfgPath));
HoodieTableConfig config = new HoodieTableConfig(fs, metaPath.toString(), null);
assertEquals(3, config.getProps().size());
assertEquals(4, config.getProps().size());
assertNull(config.getProps().getProperty("hoodie.invalid.config"));
assertFalse(config.getProps().contains(HoodieTableConfig.ARCHIVELOG_FOLDER.key()));
}
@@ -114,7 +114,7 @@ public class TestHoodieTableConfig extends HoodieCommonTestHarness {
assertFalse(fs.exists(cfgPath));
assertTrue(fs.exists(backupCfgPath));
config = new HoodieTableConfig(fs, metaPath.toString(), null);
assertEquals(4, config.getProps().size());
assertEquals(5, config.getProps().size());
}
@ParameterizedTest
@@ -132,6 +132,6 @@ public class TestHoodieTableConfig extends HoodieCommonTestHarness {
assertTrue(fs.exists(cfgPath));
assertFalse(fs.exists(backupCfgPath));
config = new HoodieTableConfig(fs, metaPath.toString(), null);
assertEquals(4, config.getProps().size());
assertEquals(5, config.getProps().size());
}
}

2
hudi-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableMetaClient.java
View File

@@ -54,6 +54,8 @@ public class TestHoodieTableMetaClient extends HoodieCommonTestHarness {
assertEquals(basePath, metaClient.getBasePath(), "Basepath should be the one assigned");
assertEquals(basePath + "/.hoodie", metaClient.getMetaPath(),
"Metapath should be ${basepath}/.hoodie");
assertTrue(metaClient.getTableConfig().getProps().containsKey(HoodieTableConfig.TABLE_CHECKSUM.key()));
assertTrue(HoodieTableConfig.validateChecksum(metaClient.getTableConfig().getProps()));
}
@Test