1
0

Initial impl of HoodieRealtimeInputFormat

- Works end-end for flat schemas
 - Schema evolution & hardening remains
 - HoodieClientExample can now write mor tables as well
This commit is contained in:
Vinoth Chandar
2017-04-24 08:19:02 -07:00
committed by prazanna
parent 9f526396a0
commit 7b1446548f
12 changed files with 745 additions and 27 deletions

View File

@@ -0,0 +1,43 @@
/*
* Copyright (c) 2017 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*
*/
import org.apache.spark.sql.SparkSession;
/**
* Examples to do Spark SQL on Hoodie dataset.
*/
public class HoodieSparkSQLExample {
public static void main(String[] args) throws Exception {
SparkSession spark = SparkSession.builder()
.appName("Hoodie SparkSQL")
.config("hive.metastore.uris","thrift://localhost:10000")
.config("spark.sql.hive.convertMetastoreParquet", false)
.enableHiveSupport()
.master("local[2]")
.getOrCreate();
spark.sql("describe hoodie_rt").show();
spark.sql("select * from hoodie_rt").show();
spark.sql("select end_lon as e1, driver, rider as r1, datestr, driver, datestr, rider, _hoodie_record_key from hoodie_rt").show();
spark.sql("select fare, begin_lon, begin_lat, timestamp from hoodie_rt where fare > 2.0").show();
spark.sql("select count(*) as cnt, _hoodie_file_name as file from hoodie_rt group by _hoodie_file_name").show();
}
}