diff --git a/.travis.yml b/.travis.yml index f5c99a7f6..16a4f68db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1 +1,2 @@ -language: java \ No newline at end of file +language: java +sudo: required diff --git a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java b/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java index 301e10e99..00010f95a 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/common/HoodieTestDataGenerator.java @@ -45,10 +45,6 @@ import java.util.UUID; * Test data uses a toy Uber trips, data model. */ public class HoodieTestDataGenerator { - - - private static Logger logger = LogManager.getLogger(HoodieTestDataGenerator.class); - static class KeyPartition { HoodieKey key; String partitionPath; @@ -97,8 +93,6 @@ public class HoodieTestDataGenerator { kp.key = key; kp.partitionPath = partitionPath; existingKeysList.add(kp); - - logger.info(" GENERATING INSERT FOR :" + key + "," + record.getPartitionPath()); } return inserts; } @@ -109,7 +103,6 @@ public class HoodieTestDataGenerator { for (HoodieRecord baseRecord: baseRecords) { HoodieRecord record = new HoodieRecord(baseRecord.getKey(), generateRandomValue(baseRecord.getKey(), commitTime)); updates.add(record); - logger.info(" GENERATING UPDATE FOR :" + baseRecord.getKey()); } return updates; } @@ -123,7 +116,6 @@ public class HoodieTestDataGenerator { KeyPartition kp = existingKeysList.get(rand.nextInt(existingKeysList.size() - 1)); HoodieRecord record = new HoodieRecord(kp.key, generateRandomValue(kp.key, commitTime)); updates.add(record); - logger.info(" GENERATING UPDATE FOR :" + kp.key); } return updates; } diff --git a/hoodie-client/src/test/resources/log4j.properties b/hoodie-client/src/test/resources/log4j-surefire.properties similarity index 83% rename from hoodie-client/src/test/resources/log4j.properties rename to hoodie-client/src/test/resources/log4j-surefire.properties index 5a8b643fd..eab225528 100644 --- a/hoodie-client/src/test/resources/log4j.properties +++ b/hoodie-client/src/test/resources/log4j-surefire.properties @@ -5,7 +5,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -13,9 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN -# Set root logger level to DEBUG and its only appender to A1. -log4j.rootLogger=INFO, A1 # A1 is set to be a ConsoleAppender. log4j.appender.A1=org.apache.log4j.ConsoleAppender # A1 uses PatternLayout. diff --git a/hoodie-common/src/test/resources/log4j-surefire.properties b/hoodie-common/src/test/resources/log4j-surefire.properties new file mode 100644 index 000000000..1c03f27e6 --- /dev/null +++ b/hoodie-common/src/test/resources/log4j-surefire.properties @@ -0,0 +1,25 @@ +# +# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties b/hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties new file mode 100644 index 000000000..1c03f27e6 --- /dev/null +++ b/hoodie-hadoop-mr/src/test/resources/log4j-surefire.properties @@ -0,0 +1,25 @@ +# +# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java b/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java index 0436d5285..9fd95c004 100644 --- a/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java +++ b/hoodie-hive/src/main/java/com/uber/hoodie/hive/client/HoodieFSClient.java @@ -106,7 +106,9 @@ public class HoodieFSClient { Path path = files.next().getPath(); if (path.getName().endsWith(PARQUET_EXTENSION) || path.getName() .endsWith(PARQUET_EXTENSION_ZIPPED)) { - returnPath = path; + if(returnPath == null || path.toString().compareTo(returnPath.toString()) > 0) { + returnPath = path; + } } } if (returnPath != null) { diff --git a/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java b/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java index cdcb528d1..2a0c11c23 100644 --- a/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java +++ b/hoodie-hive/src/test/java/com/uber/hoodie/hive/HDroneDatasetTest.java @@ -92,7 +92,8 @@ public class HDroneDatasetTest { assertEquals("Table schema should be evolved schema", expectedSchema, newDatasetSchema); assertEquals("Table schema should have 6 fields", 6, hiveClient.getTableSchema(metadata).size()); - assertEquals("", "BIGINT", hiveClient.getTableSchema(metadata).get("region_key")); + assertEquals("Valid Evolution should be reflected", "BIGINT", + hiveClient.getTableSchema(metadata).get("region_key")); } } diff --git a/hoodie-hive/src/test/resources/log4j-surefire.properties b/hoodie-hive/src/test/resources/log4j-surefire.properties new file mode 100644 index 000000000..cc6a57052 --- /dev/null +++ b/hoodie-hive/src/test/resources/log4j-surefire.properties @@ -0,0 +1,26 @@ +# +# Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=WARN, A1 +log4j.category.com.uber=INFO +log4j.category.org.apache.parquet.hadoop=WARN +log4j.category.parquet.hadoop=WARN + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n diff --git a/pom.xml b/pom.xml index 8553fb378..2c2384785 100644 --- a/pom.xml +++ b/pom.xml @@ -130,6 +130,16 @@ maven-release-plugin 2.5.3 + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + + file:${project.build.testOutputDirectory}/log4j-surefire.properties + + + @@ -141,10 +151,6 @@ maven-jar-plugin ${maven-jar-plugin.version} - - maven-surefire-plugin - ${maven-surefire-plugin.version} - org.codehaus.mojo cobertura-maven-plugin