lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-4456] Close FileSystem in SparkClientFunctionalTestHarness (#6201)

Browse Source
This commit is contained in:
Tim Brown
2022-07-24 21:42:15 -07:00
committed by GitHub
parent 2a08a65f71
commit 76a28daeb0
1 changed files with 14 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
View File

@@ -67,6 +67,7 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir;
@@ -96,6 +97,7 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
private static transient JavaSparkContext jsc;
private static transient HoodieSparkEngineContext context;
private static transient TimelineService timelineService;
private FileSystem fileSystem;
/**
* An indicator of the initialization status.
@@ -128,7 +130,10 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
}
public FileSystem fs() {
return FSUtils.getFs(basePath(), hadoopConf());
if (fileSystem == null) {
fileSystem = FSUtils.getFs(basePath(), hadoopConf());
}
return fileSystem;
}
@Override
@@ -208,6 +213,14 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
}
}
@AfterEach
public void closeFileSystem() throws IOException {
if (fileSystem != null) {
fileSystem.close();
fileSystem = null;
}
}
protected JavaRDD<HoodieRecord> tagLocation(
HoodieIndex index, JavaRDD<HoodieRecord> records, HoodieTable table) {
return HoodieJavaRDD.getJavaRDD(