[HUDI-372] Support the shortName for Hudi DataSource (#1054)
- Ability to do `spark.write.format("hudi")...`
This commit is contained in:
committed by
vinoth chandar
parent
0b52ae3ac2
commit
024230fbd2
@@ -0,0 +1,20 @@
|
|||||||
|
################################################################################
|
||||||
|
# Licensed to the Apache Software Foundation (ASF) under one
|
||||||
|
# or more contributor license agreements. See the NOTICE file
|
||||||
|
# distributed with this work for additional information
|
||||||
|
# regarding copyright ownership. The ASF licenses this file
|
||||||
|
# to you under the Apache License, Version 2.0 (the
|
||||||
|
# "License"); you may not use this file except in compliance
|
||||||
|
# with the License. You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
|
||||||
|
org.apache.hudi.DefaultSource
|
||||||
@@ -104,5 +104,5 @@ class DefaultSource extends RelationProvider
|
|||||||
outputMode)
|
outputMode)
|
||||||
}
|
}
|
||||||
|
|
||||||
override def shortName(): String = "hoodie"
|
override def shortName(): String = "hudi"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,6 +63,19 @@ class TestDataSource extends AssertionsForJUnit {
|
|||||||
fs = FSUtils.getFs(basePath, spark.sparkContext.hadoopConfiguration)
|
fs = FSUtils.getFs(basePath, spark.sparkContext.hadoopConfiguration)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test def testShortNameStorage() {
|
||||||
|
// Insert Operation
|
||||||
|
val records = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("000", 100)).toList
|
||||||
|
val inputDF: Dataset[Row] = spark.read.json(spark.sparkContext.parallelize(records, 2))
|
||||||
|
inputDF.write.format("hudi")
|
||||||
|
.options(commonOpts)
|
||||||
|
.option(DataSourceWriteOptions.OPERATION_OPT_KEY, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL)
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.save(basePath)
|
||||||
|
|
||||||
|
assertTrue(HoodieDataSourceHelpers.hasNewCommits(fs, basePath, "000"))
|
||||||
|
}
|
||||||
|
|
||||||
@Test def testCopyOnWriteStorage() {
|
@Test def testCopyOnWriteStorage() {
|
||||||
// Insert Operation
|
// Insert Operation
|
||||||
val records1 = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("000", 100)).toList
|
val records1 = DataSourceTestUtils.convertToStringList(dataGen.generateInserts("000", 100)).toList
|
||||||
|
|||||||
Reference in New Issue
Block a user