lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-2941] Show _hoodie_operation in spark sql results (#4649)

Browse Source
This commit is contained in:
ForwardXu
2022-02-07 22:28:13 +08:00
committed by GitHub
parent 24f738fe68
commit 773b317983
10 changed files with 85 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/AbstractSyncHoodieClient.java
View File

@@ -18,6 +18,8 @@
package org.apache.hudi.sync.common;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hudi.common.engine.HoodieLocalEngineContext;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -29,9 +31,6 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.TimelineUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.parquet.schema.MessageType;
@@ -149,11 +148,7 @@ public abstract class AbstractSyncHoodieClient {
*/
public MessageType getDataSchema() {
try {
if (withOperationField) {
return new TableSchemaResolver(metaClient, true).getTableParquetSchema();
} else {
return new TableSchemaResolver(metaClient).getTableParquetSchema();
}
return new TableSchemaResolver(metaClient).getTableParquetSchema();
} catch (Exception e) {
throw new HoodieSyncException("Failed to read data schema", e);
}
@@ -162,11 +157,7 @@ public abstract class AbstractSyncHoodieClient {
public boolean isDropPartition() {
try {
Option<HoodieCommitMetadata> hoodieCommitMetadata;
if (withOperationField) {
hoodieCommitMetadata = new TableSchemaResolver(metaClient, true).getLatestCommitMetadata();
} else {
hoodieCommitMetadata = new TableSchemaResolver(metaClient).getLatestCommitMetadata();
}
hoodieCommitMetadata = new TableSchemaResolver(metaClient).getLatestCommitMetadata();
if (hoodieCommitMetadata.isPresent()
&& WriteOperationType.DELETE_PARTITION.equals(hoodieCommitMetadata.get().getOperationType())) {