1
0

HUDI-479: Eliminate or Minimize use of Guava if possible (#1159)

This commit is contained in:
Suneel Marthi
2020-03-28 03:11:32 -04:00
committed by GitHub
parent 1713f686f8
commit 8c3001363d
46 changed files with 429 additions and 217 deletions

View File

@@ -64,7 +64,6 @@ import org.apache.hudi.table.WorkloadProfile;
import org.apache.hudi.table.WorkloadStat;
import com.codahale.metrics.Timer;
import com.google.common.collect.ImmutableMap;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.spark.Partitioner;
@@ -77,6 +76,7 @@ import org.apache.spark.storage.StorageLevel;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.text.ParseException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -746,7 +746,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
String startRollbackInstant = HoodieActiveTimeline.createNewInstantTime();
// Start the timer
final Timer.Context context = startContext();
ImmutableMap.Builder<String, List<HoodieRollbackStat>> instantsToStats = ImmutableMap.builder();
Map<String, List<HoodieRollbackStat>> instantsToStats = new HashMap<>();
table.getActiveTimeline().createNewInstant(
new HoodieInstant(true, HoodieTimeline.RESTORE_ACTION, startRollbackInstant));
instantsToRollback.forEach(instant -> {
@@ -773,7 +773,7 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> extends AbstractHo
}
});
try {
finishRestore(context, instantsToStats.build(),
finishRestore(context, Collections.unmodifiableMap(instantsToStats),
instantsToRollback.stream().map(HoodieInstant::getTimestamp).collect(Collectors.toList()),
startRollbackInstant, instantTime);
} catch (IOException io) {

View File

@@ -64,9 +64,8 @@ public class BloomIndexFileInfo implements Serializable {
* Does the given key fall within the range (inclusive).
*/
public boolean isKeyInRange(String recordKey) {
assert minRecordKey != null;
assert maxRecordKey != null;
return minRecordKey.compareTo(recordKey) <= 0 && maxRecordKey.compareTo(recordKey) >= 0;
return Objects.requireNonNull(minRecordKey).compareTo(recordKey) <= 0
&& Objects.requireNonNull(maxRecordKey).compareTo(recordKey) >= 0;
}
@Override

View File

@@ -54,8 +54,8 @@ import org.apache.spark.util.SizeEstimator;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;

View File

@@ -18,19 +18,19 @@
package org.apache.hudi.metrics;
import com.google.common.base.Preconditions;
import java.lang.management.ManagementFactory;
import javax.management.MBeanServer;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import org.apache.log4j.LogManager;
import com.codahale.metrics.MetricRegistry;
import java.io.Closeable;
import java.util.Objects;
import java.util.stream.IntStream;
import java.lang.management.ManagementFactory;
import javax.management.MBeanServer;
/**
* Implementation of Jmx reporter, which used to report jmx metric.
*/
@@ -92,7 +92,7 @@ public class JmxMetricsReporter extends MetricsReporter {
@Override
public void stop() {
Preconditions.checkNotNull(jmxReporterServer, "jmxReporterServer is not running.");
Objects.requireNonNull(jmxReporterServer, "jmxReporterServer is not running.");
try {
jmxReporterServer.stop();
} catch (Exception e) {

View File

@@ -18,20 +18,23 @@
package org.apache.hudi.metrics;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieException;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.jmx.JmxReporter;
import com.google.common.base.Preconditions;
import java.io.IOException;
import java.rmi.NoSuchObjectException;
import java.rmi.registry.LocateRegistry;
import java.rmi.registry.Registry;
import java.rmi.server.UnicastRemoteObject;
import java.util.Objects;
import javax.management.MBeanServer;
import javax.management.remote.JMXConnectorServer;
import javax.management.remote.JMXConnectorServerFactory;
import javax.management.remote.JMXServiceURL;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieException;
/**
* A reporter which publishes metric values to a JMX server.
@@ -78,10 +81,9 @@ public class JmxReporterServer {
}
public JmxReporterServer build() {
Preconditions.checkNotNull(registry, "registry cannot be null!");
Preconditions.checkNotNull(mBeanServer, "mBeanServer cannot be null!");
Preconditions
.checkArgument(!StringUtils.isNullOrEmpty(host), "host cannot be null or empty!");
Objects.requireNonNull(registry, "registry cannot be null!");
Objects.requireNonNull(mBeanServer, "mBeanServer cannot be null!");
ValidationUtils.checkArgument(!StringUtils.isNullOrEmpty(host), "host cannot be null or empty!");
return new JmxReporterServer(registry, host, port, mBeanServer);
}
}
@@ -110,7 +112,7 @@ public class JmxReporterServer {
}
public void start() {
Preconditions.checkArgument(reporter != null && connector != null,
ValidationUtils.checkArgument(reporter != null && connector != null,
"reporter or connector cannot be null!");
try {
connector.start();

View File

@@ -18,12 +18,12 @@
package org.apache.hudi.metrics;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
import com.codahale.metrics.Gauge;
import com.codahale.metrics.MetricRegistry;
import com.google.common.io.Closeables;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -53,8 +53,7 @@ public class Metrics {
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
try {
reporter.report();
reporter.stop();
Closeables.close(reporter.getReporter(), true);
FileIOUtils.close(reporter.getReporter(), true);
} catch (Exception e) {
e.printStackTrace();
}

View File

@@ -32,6 +32,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.HoodieTimeline;
import org.apache.hudi.common.table.TableFileSystemView.SliceView;
import org.apache.hudi.common.table.log.HoodieMergedLogRecordScanner;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.CompactionUtils;
import org.apache.hudi.common.util.FSUtils;
import org.apache.hudi.common.util.HoodieAvroUtils;
@@ -43,7 +44,6 @@ import org.apache.hudi.table.compact.strategy.CompactionStrategy;
import org.apache.hudi.table.HoodieCopyOnWriteTable;
import org.apache.hudi.table.HoodieTable;
import com.google.common.collect.Sets;
import org.apache.avro.Schema;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -113,7 +113,7 @@ public class HoodieMergeOnReadTableCompactor implements HoodieCompactor {
// loaded and load it using CompositeAvroLogReader
// Since a DeltaCommit is not defined yet, reading all the records. revisit this soon.
String maxInstantTime = metaClient
.getActiveTimeline().getTimelineOfActions(Sets.newHashSet(HoodieTimeline.COMMIT_ACTION,
.getActiveTimeline().getTimelineOfActions(CollectionUtils.createSet(HoodieTimeline.COMMIT_ACTION,
HoodieTimeline.ROLLBACK_ACTION, HoodieTimeline.DELTA_COMMIT_ACTION))
.filterCompletedInstants().lastInstant().get().getTimestamp();
LOG.info("MaxMemoryPerCompaction => " + SparkConfigUtils.getMaxMemoryPerCompaction(config.getProps()));

View File

@@ -214,7 +214,7 @@ public class RollbackHelper implements Serializable {
private Map<HeaderMetadataType, String> generateHeader(String commit) {
// generate metadata
Map<HeaderMetadataType, String> header = new HashMap<>();
Map<HeaderMetadataType, String> header = new HashMap<>(3);
header.put(HeaderMetadataType.INSTANT_TIME, metaClient.getActiveTimeline().lastInstant().get().getTimestamp());
header.put(HeaderMetadataType.TARGET_INSTANT_TIME, commit);
header.put(HeaderMetadataType.COMMAND_BLOCK_TYPE,