[MINOR] Fix some code style issues based on check-style plugin (#4532)
Co-authored-by: yuezhang <yuezhang@freewheel.tv>
This commit is contained in:
@@ -40,7 +40,8 @@ public class HoodieBucketLayout extends HoodieStorageLayout {
|
|||||||
add(WriteOperationType.DELETE);
|
add(WriteOperationType.DELETE);
|
||||||
add(WriteOperationType.COMPACT);
|
add(WriteOperationType.COMPACT);
|
||||||
add(WriteOperationType.DELETE_PARTITION);
|
add(WriteOperationType.DELETE_PARTITION);
|
||||||
}};
|
}
|
||||||
|
};
|
||||||
|
|
||||||
public HoodieBucketLayout(HoodieWriteConfig config) {
|
public HoodieBucketLayout(HoodieWriteConfig config) {
|
||||||
super(config);
|
super(config);
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
|
|||||||
import org.apache.hudi.exception.HoodieIOException;
|
import org.apache.hudi.exception.HoodieIOException;
|
||||||
import org.apache.hudi.table.HoodieTable;
|
import org.apache.hudi.table.HoodieTable;
|
||||||
import org.apache.hudi.avro.model.HoodieFileStatus;
|
import org.apache.hudi.avro.model.HoodieFileStatus;
|
||||||
|
|
||||||
import static org.apache.hudi.common.model.HoodieFileFormat.ORC;
|
import static org.apache.hudi.common.model.HoodieFileFormat.ORC;
|
||||||
import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;
|
import static org.apache.hudi.common.model.HoodieFileFormat.PARQUET;
|
||||||
|
|
||||||
|
|||||||
@@ -63,7 +63,8 @@ public class DataTypeUtils {
|
|||||||
// String types
|
// String types
|
||||||
put(StringType$.class,
|
put(StringType$.class,
|
||||||
newHashSet(VarcharType$.class, StringType$.class));
|
newHashSet(VarcharType$.class, StringType$.class));
|
||||||
}};
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validates whether one {@link StructType} is compatible w/ the other one.
|
* Validates whether one {@link StructType} is compatible w/ the other one.
|
||||||
|
|||||||
@@ -52,6 +52,7 @@ import org.apache.orc.storage.ql.exec.vector.UnionColumnVector;
|
|||||||
import org.apache.orc.storage.serde2.io.DateWritable;
|
import org.apache.orc.storage.serde2.io.DateWritable;
|
||||||
import org.apache.hudi.exception.HoodieIOException;
|
import org.apache.hudi.exception.HoodieIOException;
|
||||||
import org.apache.orc.TypeDescription;
|
import org.apache.orc.TypeDescription;
|
||||||
|
|
||||||
import static org.apache.avro.JsonProperties.NULL_VALUE;
|
import static org.apache.avro.JsonProperties.NULL_VALUE;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ package org.apache.hudi.common.util;
|
|||||||
|
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||||
|
|
||||||
|
|||||||
@@ -31,13 +31,10 @@ import org.apache.spark.sql.SaveMode;
|
|||||||
import org.apache.spark.sql.SparkSession;
|
import org.apache.spark.sql.SparkSession;
|
||||||
import org.apache.spark.sql.Dataset;
|
import org.apache.spark.sql.Dataset;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public class HoodieSparkBootstrapExample {
|
public class HoodieSparkBootstrapExample {
|
||||||
|
|
||||||
private static String tableType = HoodieTableType.MERGE_ON_READ.name();
|
private static String tableType = HoodieTableType.MERGE_ON_READ.name();
|
||||||
|
|
||||||
|
|
||||||
public static void main(String[] args) throws Exception {
|
public static void main(String[] args) throws Exception {
|
||||||
if (args.length < 5) {
|
if (args.length < 5) {
|
||||||
System.err.println("Usage: HoodieSparkBootstrapExample <recordKey> <tableName> <partitionPath> <preCombineField> <basePath>");
|
System.err.println("Usage: HoodieSparkBootstrapExample <recordKey> <tableName> <partitionPath> <preCombineField> <basePath>");
|
||||||
@@ -68,7 +65,7 @@ public class HoodieSparkBootstrapExample {
|
|||||||
.option(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieFileFormat.ORC.name())
|
.option(HoodieTableConfig.BASE_FILE_FORMAT.key(), HoodieFileFormat.ORC.name())
|
||||||
.option(HoodieBootstrapConfig.BASE_PATH.key(), basePath)
|
.option(HoodieBootstrapConfig.BASE_PATH.key(), basePath)
|
||||||
.option(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key(), NonpartitionedKeyGenerator.class.getCanonicalName())
|
.option(HoodieBootstrapConfig.KEYGEN_CLASS_NAME.key(), NonpartitionedKeyGenerator.class.getCanonicalName())
|
||||||
.mode(SaveMode.Overwrite).save("/hudi/"+tableName);
|
.mode(SaveMode.Overwrite).save("/hudi/" + tableName);
|
||||||
|
|
||||||
df.count();
|
df.count();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -106,6 +106,7 @@ public class DagUtils {
|
|||||||
Entry<String, JsonNode> dagContentNode = contentItr.next();
|
Entry<String, JsonNode> dagContentNode = contentItr.next();
|
||||||
allNodes.put(dagContentNode.getKey(), convertJsonToDagNode(allNodes, dagContentNode.getKey(), dagContentNode.getValue()));
|
allNodes.put(dagContentNode.getKey(), convertJsonToDagNode(allNodes, dagContentNode.getKey(), dagContentNode.getValue()));
|
||||||
}
|
}
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -161,7 +161,6 @@ public abstract class BaseValidateDatasetNode extends DagNode<Boolean> {
|
|||||||
.filter("_hoodie_is_deleted != true");
|
.filter("_hoodie_is_deleted != true");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private ExpressionEncoder getEncoder(StructType schema) {
|
private ExpressionEncoder getEncoder(StructType schema) {
|
||||||
List<Attribute> attributes = JavaConversions.asJavaCollection(schema.toAttributes()).stream()
|
List<Attribute> attributes = JavaConversions.asJavaCollection(schema.toAttributes()).stream()
|
||||||
.map(Attribute::toAttribute).collect(Collectors.toList());
|
.map(Attribute::toAttribute).collect(Collectors.toList());
|
||||||
|
|||||||
@@ -38,7 +38,6 @@ import java.io.IOException;
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -74,10 +73,10 @@ public class DFSTestSuitePathSelector extends DFSPathSelector {
|
|||||||
// Say input data is as follow input/1, input/2, input/5 since 3,4 was rolled back and 5 is new generated data
|
// Say input data is as follow input/1, input/2, input/5 since 3,4 was rolled back and 5 is new generated data
|
||||||
// checkpoint from the latest commit metadata will be 2 since 3,4 has been rolled back. We need to set the
|
// checkpoint from the latest commit metadata will be 2 since 3,4 has been rolled back. We need to set the
|
||||||
// next batch id correctly as 5 instead of 3
|
// next batch id correctly as 5 instead of 3
|
||||||
Optional<String> correctBatchIdDueToRollback = Arrays.stream(fileStatuses)
|
Option<String> correctBatchIdDueToRollback = Option.fromJavaOptional(Arrays.stream(fileStatuses)
|
||||||
.map(f -> f.getPath().toString().split("/")[f.getPath().toString().split("/").length - 1])
|
.map(f -> f.getPath().toString().split("/")[f.getPath().toString().split("/").length - 1])
|
||||||
.filter(bid1 -> Integer.parseInt(bid1) > lastBatchId)
|
.filter(bid1 -> Integer.parseInt(bid1) > lastBatchId)
|
||||||
.min((bid1, bid2) -> Integer.min(Integer.parseInt(bid1), Integer.parseInt(bid2)));
|
.min((bid1, bid2) -> Integer.min(Integer.parseInt(bid1), Integer.parseInt(bid2))));
|
||||||
if (correctBatchIdDueToRollback.isPresent() && Integer.parseInt(correctBatchIdDueToRollback.get()) > nextBatchId) {
|
if (correctBatchIdDueToRollback.isPresent() && Integer.parseInt(correctBatchIdDueToRollback.get()) > nextBatchId) {
|
||||||
nextBatchId = Integer.parseInt(correctBatchIdDueToRollback.get());
|
nextBatchId = Integer.parseInt(correctBatchIdDueToRollback.get());
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,7 +35,6 @@ import org.apache.hudi.integ.testsuite.schema.SchemaUtils;
|
|||||||
import org.apache.hudi.integ.testsuite.schema.TestSuiteFileBasedSchemaProvider;
|
import org.apache.hudi.integ.testsuite.schema.TestSuiteFileBasedSchemaProvider;
|
||||||
import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
|
import org.apache.hudi.integ.testsuite.writer.DeltaOutputMode;
|
||||||
import org.apache.hudi.keygen.TimestampBasedKeyGenerator;
|
import org.apache.hudi.keygen.TimestampBasedKeyGenerator;
|
||||||
import org.apache.hudi.utilities.schema.FilebasedSchemaProvider;
|
|
||||||
import org.apache.hudi.utilities.sources.AvroDFSSource;
|
import org.apache.hudi.utilities.sources.AvroDFSSource;
|
||||||
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
|
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
|
||||||
|
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ package org.apache.hudi.hive;
|
|||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
|
||||||
public class TestMultiPartKeysValueExtractor {
|
public class TestMultiPartKeysValueExtractor {
|
||||||
|
|||||||
Reference in New Issue
Block a user