lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-1358] Fix Memory Leak in HoodieLogFormatWriter (#2217)

Browse Source
This commit is contained in:
Balaji Varadarajan
2020-11-09 19:26:13 -08:00
committed by GitHub
parent 0364498ae3
commit 42b6aeca28
3 changed files with 30 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/SparkHoodieHBaseIndex.java
View File

@@ -97,6 +97,7 @@ public class SparkHoodieHBaseIndex<T extends HoodieRecordPayload> extends SparkH
private int maxQpsPerRegionServer; private int maxQpsPerRegionServer;
private long totalNumInserts; private long totalNumInserts;
private int numWriteStatusWithInserts; private int numWriteStatusWithInserts;
private static transient Thread shutdownThread;
/** /**
* multiPutBatchSize will be computed and re-set in updateLocation if * multiPutBatchSize will be computed and re-set in updateLocation if
@@ -155,13 +156,16 @@ public class SparkHoodieHBaseIndex<T extends HoodieRecordPayload> extends SparkH
* exits. * exits.
*/ */
private void addShutDownHook() { private void addShutDownHook() {
Runtime.getRuntime().addShutdownHook(new Thread(() -> { if (null == shutdownThread) {
try { shutdownThread = new Thread(() -> {
hbaseConnection.close(); try {
} catch (Exception e) { hbaseConnection.close();
// fail silently for any sort of exception } catch (Exception e) {
} // fail silently for any sort of exception
})); }
});
Runtime.getRuntime().addShutdownHook(shutdownThread);
}
} }
/** /**

9
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFileReader.java
View File

@@ -67,6 +67,7 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
private long lastReverseLogFilePosition; private long lastReverseLogFilePosition;
private boolean reverseReader; private boolean reverseReader;
private boolean closed = false; private boolean closed = false;
private transient Thread shutdownThread = null;
public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize, public HoodieLogFileReader(FileSystem fs, HoodieLogFile logFile, Schema readerSchema, int bufferSize,
boolean readBlockLazily, boolean reverseReader) throws IOException { boolean readBlockLazily, boolean reverseReader) throws IOException {
@@ -108,14 +109,15 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
* Close the inputstream if not closed when the JVM exits. * Close the inputstream if not closed when the JVM exits.
*/ */
private void addShutDownHook() { private void addShutDownHook() {
Runtime.getRuntime().addShutdownHook(new Thread(() -> { shutdownThread = new Thread(() -> {
try { try {
close(); close();
} catch (Exception e) { } catch (Exception e) {
LOG.warn("unable to close input stream for log file " + logFile, e); LOG.warn("unable to close input stream for log file " + logFile, e);
// fail silently for any sort of exception // fail silently for any sort of exception
} }
})); });
Runtime.getRuntime().addShutdownHook(shutdownThread);
} }
// TODO : convert content and block length to long by using ByteBuffer, raw byte [] allows // TODO : convert content and block length to long by using ByteBuffer, raw byte [] allows
@@ -291,6 +293,9 @@ public class HoodieLogFileReader implements HoodieLogFormat.Reader {
public void close() throws IOException { public void close() throws IOException {
if (!closed) { if (!closed) {
this.inputStream.close(); this.inputStream.close();
if (null != shutdownThread) {
Runtime.getRuntime().removeShutdownHook(shutdownThread);
}
closed = true; closed = true;
} }
} }

14
hudi-common/src/main/java/org/apache/hudi/common/table/log/HoodieLogFormatWriter.java
View File

@@ -55,6 +55,8 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
private final String rolloverLogWriteToken; private final String rolloverLogWriteToken;
private FSDataOutputStream output; private FSDataOutputStream output;
private boolean closed = false; private boolean closed = false;
private transient Thread shutdownThread = null;
private static final String APPEND_UNAVAILABLE_EXCEPTION_MESSAGE = "not sufficiently replicated yet"; private static final String APPEND_UNAVAILABLE_EXCEPTION_MESSAGE = "not sufficiently replicated yet";
/** /**
@@ -222,6 +224,13 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
@Override @Override
public void close() throws IOException { public void close() throws IOException {
if (null != shutdownThread) {
Runtime.getRuntime().removeShutdownHook(shutdownThread);
}
closeStream();
}
private void closeStream() throws IOException {
if (output != null) { if (output != null) {
flush(); flush();
output.close(); output.close();
@@ -256,7 +265,7 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
* Close the output stream when the JVM exits. * Close the output stream when the JVM exits.
*/ */
private void addShutDownHook() { private void addShutDownHook() {
Runtime.getRuntime().addShutdownHook(new Thread() { shutdownThread = new Thread() {
public void run() { public void run() {
try { try {
if (output != null) { if (output != null) {
@@ -267,7 +276,8 @@ public class HoodieLogFormatWriter implements HoodieLogFormat.Writer {
// fail silently for any sort of exception // fail silently for any sort of exception
} }
} }
}); };
Runtime.getRuntime().addShutdownHook(shutdownThread);
} }
private void handleAppendExceptionOrRecoverLease(Path path, RemoteException e) private void handleAppendExceptionOrRecoverLease(Path path, RemoteException e)