lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Timeline Service with Incremental View Syncing support

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-02-12 21:29:14 -08:00
committed by vinoth chandar
parent 446f99aa0f
commit 64fec64097
117 changed files with 8943 additions and 873 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
hoodie-timeline-service/src/test/java/com/uber/hoodie/timeline/table/view/RemoteHoodieTableFileSystemViewTest.java Normal file
View File

@@ -0,0 +1,57 @@
/*
* Copyright (c) 2019 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.uber.hoodie.timeline.table.view;
import com.uber.hoodie.common.SerializableConfiguration;
import com.uber.hoodie.common.table.HoodieTimeline;
import com.uber.hoodie.common.table.SyncableFileSystemView;
import com.uber.hoodie.common.table.view.FileSystemViewManager;
import com.uber.hoodie.common.table.view.FileSystemViewStorageConfig;
import com.uber.hoodie.common.table.view.FileSystemViewStorageType;
import com.uber.hoodie.common.table.view.HoodieTableFileSystemViewTest;
import com.uber.hoodie.common.table.view.RemoteHoodieTableFileSystemView;
import com.uber.hoodie.timeline.service.TimelineService;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
/**
* Bring up a remote Timeline Server and run all test-cases of HoodieTableFileSystemViewTest against it
*/
public class RemoteHoodieTableFileSystemViewTest extends HoodieTableFileSystemViewTest {
private static Logger log = LogManager.getLogger(RemoteHoodieTableFileSystemViewTest.class);
private TimelineService server;
private RemoteHoodieTableFileSystemView view;
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) {
FileSystemViewStorageConfig sConf = FileSystemViewStorageConfig.newBuilder()
.withStorageType(FileSystemViewStorageType.SPILLABLE_DISK).build();
try {
server =
new TimelineService(0,
FileSystemViewManager.createViewManager(new SerializableConfiguration(metaClient.getHadoopConf()),
sConf));
server.startService();
} catch (Exception ex) {
throw new RuntimeException(ex);
}
log.info("Connecting to Timeline Server :" + server.getServerPort());
view = new RemoteHoodieTableFileSystemView("localhost", server.getServerPort(), metaClient);
return view;
}
}

21
hoodie-timeline-service/src/test/resources/log4j-surefire-quiet.properties Normal file
View File

@@ -0,0 +1,21 @@
#
# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
log4j.rootLogger=WARN, A1
# A1 is set to be a ConsoleAppender.
log4j.appender.A1=org.apache.log4j.ConsoleAppender
# A1 uses PatternLayout.
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=[%-5p] %d %c %x - %m%n

25
hoodie-timeline-service/src/test/resources/log4j-surefire.properties Normal file
View File

@@ -0,0 +1,25 @@
#
# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
log4j.rootLogger=WARN, A1
log4j.category.com.uber=INFO
log4j.category.com.uber.hoodie.table.log=WARN
log4j.category.com.uber.hoodie.common.util=WARN
log4j.category.org.apache.parquet.hadoop=WARN
# A1 is set to be a ConsoleAppender.
log4j.appender.A1=org.apache.log4j.ConsoleAppender
# A1 uses PatternLayout.
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n