lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4456] Close FileSystem in SparkClientFunctionalTestHarness (#6201)

Browse Source
This commit is contained in:
Tim Brown
2022-07-24 21:42:15 -07:00
committed by GitHub
parent 2a08a65f71
commit 76a28daeb0
1 changed files with 14 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
View File

@@ -67,6 +67,7 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext; import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.api.io.TempDir;
@@ -96,6 +97,7 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
private static transient JavaSparkContext jsc; private static transient JavaSparkContext jsc;
private static transient HoodieSparkEngineContext context; private static transient HoodieSparkEngineContext context;
private static transient TimelineService timelineService; private static transient TimelineService timelineService;
private FileSystem fileSystem;
/** /**
* An indicator of the initialization status. * An indicator of the initialization status.
@@ -128,7 +130,10 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
} }
public FileSystem fs() { public FileSystem fs() {
return FSUtils.getFs(basePath(), hadoopConf()); if (fileSystem == null) {
fileSystem = FSUtils.getFs(basePath(), hadoopConf());
}
return fileSystem;
} }
@Override @Override
@@ -208,6 +213,14 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
} }
} }
@AfterEach
public void closeFileSystem() throws IOException {
if (fileSystem != null) {
fileSystem.close();
fileSystem = null;
}
}
protected JavaRDD<HoodieRecord> tagLocation( protected JavaRDD<HoodieRecord> tagLocation(
HoodieIndex index, JavaRDD<HoodieRecord> records, HoodieTable table) { HoodieIndex index, JavaRDD<HoodieRecord> records, HoodieTable table) {
return HoodieJavaRDD.getJavaRDD( return HoodieJavaRDD.getJavaRDD(