lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-3011] Adding ability to read entire data with HoodieIncrSource with empty checkpoint (#4334)

Browse Source 
* Adding ability to read entire data with HoodieIncrSource with empty checkpoint

* Addressing comments
This commit is contained in:
Sivabalan Narayanan
2021-12-22 02:13:06 -08:00
committed by GitHub
parent b5890cd17d
commit 1a5f8693aa
5 changed files with 184 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestHarness.java
View File

@@ -71,6 +71,7 @@ import org.apache.log4j.Logger;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -109,6 +110,7 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im
private String testMethodName;
protected transient JavaSparkContext jsc = null;
protected transient HoodieSparkEngineContext context = null;
protected transient SparkSession sparkSession = null;
protected transient Configuration hadoopConf = null;
protected transient SQLContext sqlContext;
protected transient FileSystem fs;
@@ -182,6 +184,7 @@ public abstract class HoodieClientTestHarness extends HoodieCommonTestHarness im
sqlContext = new SQLContext(jsc);
context = new HoodieSparkEngineContext(jsc);
hadoopConf = context.getHadoopConf().get();
sparkSession = SparkSession.builder().config(jsc.getConf()).getOrCreate();
}
/**