lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Enabled deletes in merge_on_read

Browse Source
This commit is contained in:
Nishith Agarwal
2017-07-07 12:50:01 -07:00
committed by vinoth chandar
parent cf1dde0323
commit 616c9a68c3
6 changed files with 275 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hoodie-common/src/main/java/com/uber/hoodie/common/util/ParquetUtils.java
View File

@@ -40,6 +40,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import static com.uber.hoodie.common.util.FSUtils.getFs;
/**
* Utility functions involving with parquet.
*/
@@ -52,6 +54,7 @@ public class ParquetUtils {
*/
public static Set<String> readRowKeysFromParquet(Path filePath) {
Configuration conf = new Configuration();
conf.addResource(getFs().getConf());
Schema readSchema = HoodieAvroUtils.getRecordKeySchema();
AvroReadSupport.setAvroReadSchema(conf, readSchema);
AvroReadSupport.setRequestedProjection(conf, readSchema);
@@ -97,7 +100,7 @@ public class ParquetUtils {
ParquetMetadata footer;
try {
// TODO(vc): Should we use the parallel reading version here?
footer = ParquetFileReader.readFooter(conf, parquetFilePath);
footer = ParquetFileReader.readFooter(getFs().getConf(), parquetFilePath);
} catch (IOException e) {
throw new HoodieIOException("Failed to read footer for parquet " + parquetFilePath,
e);

4
hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java
View File

@@ -64,6 +64,10 @@ public class HoodieTestUtils {
public static final String RAW_TRIPS_TEST_NAME = "raw_trips";
public static final int DEFAULT_TASK_PARTITIONID = 1;
public static void resetFS() {
HoodieTestUtils.fs = FSUtils.getFs();
}
public static HoodieTableMetaClient init(String basePath) throws IOException {
return initTableType(basePath, HoodieTableType.COPY_ON_WRITE);
}