From 324de298bc51e3cc8a0edad891dfa76a89c2f76b Mon Sep 17 00:00:00 2001 From: Nishith Agarwal Date: Tue, 4 Sep 2018 02:05:15 -0700 Subject: [PATCH] Removing dependency on apache-commons lang 3, adding necessary classes as needed --- .../compact/HoodieRealtimeTableCompactor.java | 2 +- .../com/uber/hoodie/TestAsyncCompaction.java | 2 +- .../java/com/uber/hoodie/TestCleaner.java | 2 +- hoodie-common/pom.xml | 4 - .../hoodie/common/table/HoodieTimeline.java | 2 +- .../table/log/block/HoodieDeleteBlock.java | 4 +- .../table/timeline/HoodieActiveTimeline.java | 5 +- .../table/view/HoodieTableFileSystemView.java | 2 +- .../hoodie/common/util/CompactionUtils.java | 2 +- .../common/util/SerializationUtils.java | 171 +++++++++++++++++ .../uber/hoodie/common/util/StringUtils.java | 51 +++++ .../common/util/collection/ImmutablePair.java | 109 +++++++++++ .../util/collection/ImmutableTriple.java | 112 +++++++++++ .../hoodie/common/util/collection/Pair.java | 179 ++++++++++++++++++ .../hoodie/common/util/collection/Triple.java | 162 ++++++++++++++++ .../converter/HoodieRecordConverter.java | 12 +- .../util/queue/BoundedInMemoryExecutor.java | 4 +- ...java => HoodieSerializationException.java} | 10 +- .../hoodie/common/model/HoodieTestUtils.java | 2 +- .../view/HoodieTableFileSystemViewTest.java | 2 +- .../common/util/TestCompactionUtils.java | 2 +- hoodie-spark/pom.xml | 5 - pom.xml | 5 - 23 files changed, 812 insertions(+), 39 deletions(-) create mode 100644 hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java create mode 100644 hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java create mode 100644 hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java create mode 100644 hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java create mode 100644 hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java create mode 100644 hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java rename hoodie-common/src/main/java/com/uber/hoodie/exception/{HoodieNotSerializableException.java => HoodieSerializationException.java} (76%) diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java index 3659437c6..4d47e7605 100644 --- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java +++ b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java @@ -36,6 +36,7 @@ import com.uber.hoodie.common.table.log.HoodieMergedLogRecordScanner; import com.uber.hoodie.common.util.CompactionUtils; import com.uber.hoodie.common.util.FSUtils; import com.uber.hoodie.common.util.HoodieAvroUtils; +import com.uber.hoodie.common.util.collection.Pair; import com.uber.hoodie.config.HoodieWriteConfig; import com.uber.hoodie.io.compact.strategy.CompactionStrategy; import com.uber.hoodie.table.HoodieCopyOnWriteTable; @@ -49,7 +50,6 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.StreamSupport; import org.apache.avro.Schema; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileSystem; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java index f1a1b8ea8..26fddc17a 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java @@ -36,6 +36,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant; import com.uber.hoodie.common.table.view.HoodieTableFileSystemView; import com.uber.hoodie.common.util.AvroUtils; import com.uber.hoodie.common.util.CompactionUtils; +import com.uber.hoodie.common.util.collection.Pair; import com.uber.hoodie.config.HoodieCompactionConfig; import com.uber.hoodie.config.HoodieIndexConfig; import com.uber.hoodie.config.HoodieStorageConfig; @@ -49,7 +50,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileStatus; import org.apache.spark.api.java.JavaRDD; import org.junit.Test; diff --git a/hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java b/hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java index 2408333be..0a669683c 100644 --- a/hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java +++ b/hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java @@ -46,6 +46,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant.State; import com.uber.hoodie.common.util.AvroUtils; import com.uber.hoodie.common.util.CompactionUtils; import com.uber.hoodie.common.util.FSUtils; +import com.uber.hoodie.common.util.collection.Pair; import com.uber.hoodie.config.HoodieCompactionConfig; import com.uber.hoodie.config.HoodieWriteConfig; import com.uber.hoodie.index.HoodieIndex; @@ -64,7 +65,6 @@ import java.util.TreeSet; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; diff --git a/hoodie-common/pom.xml b/hoodie-common/pom.xml index 0276263e3..1e2f4fec4 100644 --- a/hoodie-common/pom.xml +++ b/hoodie-common/pom.xml @@ -110,10 +110,6 @@ hadoop-common tests - - org.apache.commons - commons-lang3 - com.esotericsoftware kryo diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java index 7adc9a164..4cd8ba709 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java @@ -19,11 +19,11 @@ package com.uber.hoodie.common.table; import com.uber.hoodie.common.table.timeline.HoodieDefaultTimeline; import com.uber.hoodie.common.table.timeline.HoodieInstant; import com.uber.hoodie.common.table.timeline.HoodieInstant.State; +import com.uber.hoodie.common.util.StringUtils; import java.io.Serializable; import java.util.Optional; import java.util.function.BiPredicate; import java.util.stream.Stream; -import org.apache.commons.lang3.StringUtils; /** * HoodieTimeline is a view of meta-data instants in the hoodie dataset. Instants are specific diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java index 2f4ed7421..eddae761d 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java @@ -18,6 +18,7 @@ package com.uber.hoodie.common.table.log.block; import com.uber.hoodie.common.model.HoodieLogFile; import com.uber.hoodie.common.storage.SizeAwareDataInputStream; +import com.uber.hoodie.common.util.StringUtils; import com.uber.hoodie.exception.HoodieIOException; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -28,7 +29,6 @@ import java.nio.charset.Charset; import java.util.HashMap; import java.util.Map; import java.util.Optional; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.fs.FSDataInputStream; /** @@ -64,7 +64,7 @@ public class HoodieDeleteBlock extends HoodieLogBlock { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(baos); - byte[] bytesToWrite = StringUtils.join(getKeysToDelete(), ',') + byte[] bytesToWrite = StringUtils.join(getKeysToDelete(), ",") .getBytes(Charset.forName("utf-8")); output.writeInt(HoodieLogBlock.version); output.writeInt(bytesToWrite.length); diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java index f2c1f3432..3640ad636 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java @@ -24,6 +24,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant.State; import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.Date; import java.util.HashSet; @@ -32,7 +33,6 @@ import java.util.Set; import java.util.function.Function; import java.util.stream.Stream; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.time.FastDateFormat; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; @@ -49,8 +49,7 @@ import org.apache.log4j.Logger; */ public class HoodieActiveTimeline extends HoodieDefaultTimeline { - public static final FastDateFormat COMMIT_FORMATTER = FastDateFormat - .getInstance("yyyyMMddHHmmss"); + public static final SimpleDateFormat COMMIT_FORMATTER = new SimpleDateFormat("yyyyMMddHHmmss"); public static final Set VALID_EXTENSIONS_IN_ACTIVE_TIMELINE = new HashSet<>(Arrays.asList( new String[]{COMMIT_EXTENSION, INFLIGHT_COMMIT_EXTENSION, DELTA_COMMIT_EXTENSION, diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java index edcaa8261..d33667e31 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java @@ -28,6 +28,7 @@ import com.uber.hoodie.common.table.TableFileSystemView; import com.uber.hoodie.common.table.timeline.HoodieInstant; import com.uber.hoodie.common.util.CompactionUtils; import com.uber.hoodie.common.util.FSUtils; +import com.uber.hoodie.common.util.collection.Pair; import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.io.Serializable; @@ -42,7 +43,6 @@ import java.util.Set; import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.log4j.LogManager; diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java index 2ee99eb8d..6ce2aa8d3 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java @@ -23,6 +23,7 @@ import com.uber.hoodie.common.model.FileSlice; import com.uber.hoodie.common.table.HoodieTableMetaClient; import com.uber.hoodie.common.table.HoodieTimeline; import com.uber.hoodie.common.table.timeline.HoodieInstant; +import com.uber.hoodie.common.util.collection.Pair; import com.uber.hoodie.exception.HoodieException; import java.io.IOException; import java.util.HashMap; @@ -32,7 +33,6 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; /** * Helper class to generate compaction plan from FileGroup/FileSlice abstraction diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java new file mode 100644 index 000000000..c52a66674 --- /dev/null +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.common.util; + +import com.uber.hoodie.exception.HoodieSerializationException; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.OutputStream; +import java.io.Serializable; + +/** + * (NOTE: Adapted from Apache commons-lang3) + * This class defines API's to serde an object. + */ +public class SerializationUtils { + // Serialize + //----------------------------------------------------------------------- + + /** + *

Serializes an {@code Object} to the specified stream.

+ * + *

The stream will be closed once the object is written. + * This avoids the need for a finally clause, and maybe also exception + * handling, in the application code.

+ * + *

The stream passed in is not buffered internally within this method. + * This is the responsibility of your application if desired.

+ * + * @param obj the object to serialize to bytes, may be null + * @param outputStream the stream to write to, must not be null + * @throws IllegalArgumentException if {@code outputStream} is {@code null} + * @throws HoodieSerializationException (runtime) if the serialization fails + */ + public static void serialize(final Serializable obj, final OutputStream outputStream) { + if (outputStream == null) { + throw new IllegalArgumentException("The OutputStream must not be null"); + } + ObjectOutputStream out = null; + try { + // stream closed in the finally + out = new ObjectOutputStream(outputStream); + out.writeObject(obj); + + } catch (final IOException ex) { + throw new HoodieSerializationException("unable to serialize object", ex); + } finally { + try { + if (out != null) { + out.close(); + } + } catch (final IOException ex) { // NOPMD + // ignore close exception + } + } + } + + /** + *

Serializes an {@code Object} to a byte array for + * storage/serialization.

+ * + * @param obj the object to serialize to bytes + * @return a byte[] with the converted Serializable + * @throws HoodieSerializationException (runtime) if the serialization fails + */ + public static byte[] serialize(final Serializable obj) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(512); + serialize(obj, baos); + return baos.toByteArray(); + } + + // Deserialize + //----------------------------------------------------------------------- + + /** + *

+ * Deserializes an {@code Object} from the specified stream. + *

+ * + *

+ * The stream will be closed once the object is written. This avoids the need for a finally clause, and maybe also + * exception handling, in the application code. + *

+ * + *

+ * The stream passed in is not buffered internally within this method. This is the responsibility of your + * application if desired. + *

+ * + *

+ * If the call site incorrectly types the return value, a {@link ClassCastException} is thrown from the call site. + * Without Generics in this declaration, the call site must type cast and can cause the same ClassCastException. + * Note that in both cases, the ClassCastException is in the call site, not in this method. + *

+ * + * @param the object type to be deserialized + * @param inputStream the serialized object input stream, must not be null + * @return the deserialized object + * @throws IllegalArgumentException if {@code inputStream} is {@code null} + * @throws HoodieSerializationException (runtime) if the serialization fails + */ + public static T deserialize(final InputStream inputStream) { + if (inputStream == null) { + throw new IllegalArgumentException("The InputStream must not be null"); + } + ObjectInputStream in = null; + try { + // stream closed in the finally + in = new ObjectInputStream(inputStream); + @SuppressWarnings("unchecked") // may fail with CCE if serialised form is incorrect + final T obj = (T) in.readObject(); + return obj; + + } catch (final ClassCastException ex) { + throw new HoodieSerializationException("cannot cast class", ex); + } catch (final ClassNotFoundException ex) { + throw new HoodieSerializationException("class not found", ex); + } catch (final IOException ex) { + throw new HoodieSerializationException("unable to deserialize to object", ex); + } finally { + try { + if (in != null) { + in.close(); + } + } catch (final IOException ex) { // NOPMD + // ignore close exception + } + } + } + + /** + *

+ * Deserializes a single {@code Object} from an array of bytes. + *

+ * + *

+ * If the call site incorrectly types the return value, a {@link ClassCastException} is thrown from the call site. + * Without Generics in this declaration, the call site must type cast and can cause the same ClassCastException. + * Note that in both cases, the ClassCastException is in the call site, not in this method. + *

+ * + * @param the object type to be deserialized + * @param objectData the serialized object, must not be null + * @return the deserialized object + * @throws IllegalArgumentException if {@code objectData} is {@code null} + * @throws HoodieSerializationException (runtime) if the serialization fails + */ + public static T deserialize(final byte[] objectData) { + if (objectData == null) { + throw new IllegalArgumentException("The byte[] must not be null"); + } + return deserialize(new ByteArrayInputStream(objectData)); + } +} diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java new file mode 100644 index 000000000..e9921a210 --- /dev/null +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.common.util; + +/** + * Simple utility for operations on strings + */ +public class StringUtils { + + /** + *

Joins the elements of the provided array into a single String + * containing the provided list of elements.

+ * + *

No separator is added to the joined String. + * Null objects or empty strings within the array are represented by + * empty strings.

+ * + * 
+   * StringUtils.join(null)            = null
+   * StringUtils.join([])              = ""
+   * StringUtils.join([null])          = ""
+   * StringUtils.join(["a", "b", "c"]) = "abc"
+   * StringUtils.join([null, "", "a"]) = "a"
+   *
+ */ + public static String join(final String... elements) { + return join(elements, ""); + } + + public static String join(final String[] array, final String separator) { + if (array == null) { + return null; + } + return org.apache.hadoop.util.StringUtils.join(separator, array); + } + +} diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java new file mode 100644 index 000000000..d2d4df07b --- /dev/null +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.common.util.collection; + +/** + * (NOTE: Adapted from Apache commons-lang3) + *

An immutable pair consisting of two {@code Object} elements.

+ * + *

Although the implementation is immutable, there is no restriction on the objects + * that may be stored. If mutable objects are stored in the pair, then the pair + * itself effectively becomes mutable. The class is also {@code final}, so a subclass + * can not add undesirable behaviour.

+ * + *

#ThreadSafe# if both paired objects are thread-safe

+ * + * @param the left element type + * @param the right element type + */ +public final class ImmutablePair extends Pair { + + /** + * Serialization version + */ + private static final long serialVersionUID = 4954918890077093841L; + + /** + * Left object + */ + public final L left; + /** + * Right object + */ + public final R right; + + /** + *

Obtains an immutable pair of from two objects inferring the generic types.

+ * + *

This factory allows the pair to be created using inference to + * obtain the generic types.

+ * + * @param the left element type + * @param the right element type + * @param left the left element, may be null + * @param right the right element, may be null + * @return a pair formed from the two parameters, not null + */ + public static ImmutablePair of(final L left, final R right) { + return new ImmutablePair(left, right); + } + + /** + * Create a new pair instance. + * + * @param left the left value, may be null + * @param right the right value, may be null + */ + public ImmutablePair(final L left, final R right) { + super(); + this.left = left; + this.right = right; + } + + //----------------------------------------------------------------------- + + /** + * {@inheritDoc} + */ + @Override + public L getLeft() { + return left; + } + + /** + * {@inheritDoc} + */ + @Override + public R getRight() { + return right; + } + + /** + *

Throws {@code UnsupportedOperationException}.

+ * + *

This pair is immutable, so this operation is not supported.

+ * + * @param value the value to set + * @return never + * @throws UnsupportedOperationException as this operation is not supported + */ + @Override + public R setValue(final R value) { + throw new UnsupportedOperationException(); + } + +} diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java new file mode 100644 index 000000000..f986cc9f6 --- /dev/null +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.common.util.collection; + +/** + * (NOTE: Adapted from Apache commons-lang3) + *

An immutable triple consisting of three {@code Object} elements.

+ * + *

Although the implementation is immutable, there is no restriction on the objects + * that may be stored. If mutable objects are stored in the triple, then the triple + * itself effectively becomes mutable. The class is also {@code final}, so a subclass + * can not add undesirable behaviour.

+ * + *

#ThreadSafe# if all three objects are thread-safe

+ * + * @param the left element type + * @param the middle element type + * @param the right element type + */ +public final class ImmutableTriple extends Triple { + + /** + * Serialization version + */ + private static final long serialVersionUID = 1L; + + /** + * Left object + */ + public final L left; + /** + * Middle object + */ + public final M middle; + /** + * Right object + */ + public final R right; + + /** + *

Obtains an immutable triple of from three objects inferring the generic types.

+ * + *

This factory allows the triple to be created using inference to + * obtain the generic types.

+ * + * @param the left element type + * @param the middle element type + * @param the right element type + * @param left the left element, may be null + * @param middle the middle element, may be null + * @param right the right element, may be null + * @return a triple formed from the three parameters, not null + */ + public static ImmutableTriple of(final L left, final M middle, final R right) { + return new ImmutableTriple(left, middle, right); + } + + /** + * Create a new triple instance. + * + * @param left the left value, may be null + * @param middle the middle value, may be null + * @param right the right value, may be null + */ + public ImmutableTriple(final L left, final M middle, final R right) { + super(); + this.left = left; + this.middle = middle; + this.right = right; + } + + //----------------------------------------------------------------------- + + /** + * {@inheritDoc} + */ + @Override + public L getLeft() { + return left; + } + + /** + * {@inheritDoc} + */ + @Override + public M getMiddle() { + return middle; + } + + /** + * {@inheritDoc} + */ + @Override + public R getRight() { + return right; + } +} + diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java new file mode 100644 index 000000000..60d26fe61 --- /dev/null +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.common.util.collection; + +import java.io.Serializable; +import java.util.Map; +import org.apache.commons.lang.builder.CompareToBuilder; + +/** + * (NOTE: Adapted from Apache commons-lang3) + *

A pair consisting of two elements.

+ * + *

This class is an abstract implementation defining the basic API. + * It refers to the elements as 'left' and 'right'. It also implements the + * {@code Map.Entry} interface where the key is 'left' and the value is 'right'.

+ * + *

Subclass implementations may be mutable or immutable. + * However, there is no restriction on the type of the stored objects that may be stored. + * If mutable objects are stored in the pair, then the pair itself effectively becomes mutable.

+ * + * @param the left element type + * @param the right element type + */ +public abstract class Pair implements Map.Entry, Comparable>, Serializable { + + /** + * Serialization version + */ + private static final long serialVersionUID = 4954918890077093841L; + + /** + *

Obtains an immutable pair of from two objects inferring the generic types.

+ * + *

This factory allows the pair to be created using inference to + * obtain the generic types.

+ * + * @param the left element type + * @param the right element type + * @param left the left element, may be null + * @param right the right element, may be null + * @return a pair formed from the two parameters, not null + */ + public static Pair of(final L left, final R right) { + return new ImmutablePair(left, right); + } + + //----------------------------------------------------------------------- + + /** + *

Gets the left element from this pair.

+ * + *

When treated as a key-value pair, this is the key.

+ * + * @return the left element, may be null + */ + public abstract L getLeft(); + + /** + *

Gets the right element from this pair.

+ * + *

When treated as a key-value pair, this is the value.

+ * + * @return the right element, may be null + */ + public abstract R getRight(); + + /** + *

Gets the key from this pair.

+ * + *

This method implements the {@code Map.Entry} interface returning the + * left element as the key.

+ * + * @return the left element as the key, may be null + */ + @Override + public final L getKey() { + return getLeft(); + } + + /** + *

Gets the value from this pair.

+ * + *

This method implements the {@code Map.Entry} interface returning the + * right element as the value.

+ * + * @return the right element as the value, may be null + */ + @Override + public R getValue() { + return getRight(); + } + + //----------------------------------------------------------------------- + + /** + *

Compares the pair based on the left element followed by the right element. + * The types must be {@code Comparable}.

+ * + * @param other the other pair, not null + * @return negative if this is less, zero if equal, positive if greater + */ + @Override + public int compareTo(final Pair other) { + return new CompareToBuilder().append(getLeft(), other.getLeft()) + .append(getRight(), other.getRight()).toComparison(); + } + + /** + *

Compares this pair to another based on the two elements.

+ * + * @param obj the object to compare to, null returns false + * @return true if the elements of the pair are equal + */ + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof Map.Entry) { + final Map.Entry other = (Map.Entry) obj; + return getKey().equals(other.getKey()) + && getValue().equals(other.getValue()); + } + return false; + } + + /** + *

Returns a suitable hash code. + * The hash code follows the definition in {@code Map.Entry}.

+ * + * @return the hash code + */ + @Override + public int hashCode() { + // see Map.Entry API specification + return (getKey() == null ? 0 : getKey().hashCode()) + ^ (getValue() == null ? 0 : getValue().hashCode()); + } + + /** + *

Returns a String representation of this pair using the format {@code ($left,$right)}.

+ * + * @return a string describing this object, not null + */ + @Override + public String toString() { + return new StringBuilder().append('(').append(getLeft()).append(',').append(getRight()).append(')').toString(); + } + + /** + *

Formats the receiver using the given format.

+ * + *

This uses {@link java.util.Formattable} to perform the formatting. Two variables may + * be used to embed the left and right elements. Use {@code %1$s} for the left + * element (key) and {@code %2$s} for the right element (value). + * The default format used by {@code toString()} is {@code (%1$s,%2$s)}.

+ * + * @param format the format string, optionally containing {@code %1$s} and {@code %2$s}, not null + * @return the formatted string, not null + */ + public String toString(final String format) { + return String.format(format, getLeft(), getRight()); + } + +} diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java new file mode 100644 index 000000000..eeec5fd49 --- /dev/null +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java @@ -0,0 +1,162 @@ +/* + * Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.uber.hoodie.common.util.collection; + +import java.io.Serializable; +import org.apache.commons.lang.builder.CompareToBuilder; + +/** + * (NOTE: Adapted from Apache commons-lang3) + *

A triple consisting of three elements.

+ * + *

This class is an abstract implementation defining the basic API. + * It refers to the elements as 'left', 'middle' and 'right'.

+ * + *

Subclass implementations may be mutable or immutable. + * However, there is no restriction on the type of the stored objects that may be stored. + * If mutable objects are stored in the triple, then the triple itself effectively becomes mutable.

+ * + * @param the left element type + * @param the middle element type + * @param the right element type + */ +public abstract class Triple implements Comparable>, Serializable { + + /** + * Serialization version + */ + private static final long serialVersionUID = 1L; + + /** + *

Obtains an immutable triple of from three objects inferring the generic types.

+ * + *

This factory allows the triple to be created using inference to + * obtain the generic types.

+ * + * @param the left element type + * @param the middle element type + * @param the right element type + * @param left the left element, may be null + * @param middle the middle element, may be null + * @param right the right element, may be null + * @return a triple formed from the three parameters, not null + */ + public static Triple of(final L left, final M middle, final R right) { + return new ImmutableTriple(left, middle, right); + } + + //----------------------------------------------------------------------- + + /** + *

Gets the left element from this triple.

+ * + * @return the left element, may be null + */ + public abstract L getLeft(); + + /** + *

Gets the middle element from this triple.

+ * + * @return the middle element, may be null + */ + public abstract M getMiddle(); + + /** + *

Gets the right element from this triple.

+ * + * @return the right element, may be null + */ + public abstract R getRight(); + + //----------------------------------------------------------------------- + + /** + *

Compares the triple based on the left element, followed by the middle element, + * finally the right element. + * The types must be {@code Comparable}.

+ * + * @param other the other triple, not null + * @return negative if this is less, zero if equal, positive if greater + */ + @Override + public int compareTo(final Triple other) { + return new CompareToBuilder().append(getLeft(), other.getLeft()) + .append(getMiddle(), other.getMiddle()) + .append(getRight(), other.getRight()).toComparison(); + } + + /** + *

Compares this triple to another based on the three elements.

+ * + * @param obj the object to compare to, null returns false + * @return true if the elements of the triple are equal + */ + @SuppressWarnings("deprecation") // ObjectUtils.equals(Object, Object) has been deprecated in 3.2 + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (obj instanceof Triple) { + final Triple other = (Triple) obj; + return getLeft().equals(other.getLeft()) + && getMiddle().equals(other.getMiddle()) + && getRight().equals(other.getRight()); + } + return false; + } + + /** + *

Returns a suitable hash code.

+ * + * @return the hash code + */ + @Override + public int hashCode() { + return (getLeft() == null ? 0 : getLeft().hashCode()) + ^ (getMiddle() == null ? 0 : getMiddle().hashCode()) + ^ (getRight() == null ? 0 : getRight().hashCode()); + } + + /** + *

Returns a String representation of this triple using the format {@code ($left,$middle,$right)}.

+ * + * @return a string describing this object, not null + */ + @Override + public String toString() { + return new StringBuilder().append('(').append(getLeft()).append(',').append(getMiddle()).append(',') + .append(getRight()).append(')').toString(); + } + + /** + *

Formats the receiver using the given format.

+ * + *

This uses {@link java.util.Formattable} to perform the formatting. Three variables may + * be used to embed the left and right elements. Use {@code %1$s} for the left + * element, {@code %2$s} for the middle and {@code %3$s} for the right element. + * The default format used by {@code toString()} is {@code (%1$s,%2$s,%3$s)}.

+ * + * @param format the format string, optionally containing {@code %1$s}, {@code %2$s} and {@code %3$s}, not null + * @return the formatted string, not null + */ + public String toString(final String format) { + return String.format(format, getLeft(), getMiddle(), getRight()); + } + +} + diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/converter/HoodieRecordConverter.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/converter/HoodieRecordConverter.java index 6aac7b365..1f5ad6b1f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/converter/HoodieRecordConverter.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/converter/HoodieRecordConverter.java @@ -22,14 +22,14 @@ import com.uber.hoodie.common.model.HoodieRecordLocation; import com.uber.hoodie.common.model.HoodieRecordPayload; import com.uber.hoodie.common.util.HoodieAvroUtils; import com.uber.hoodie.common.util.ReflectionUtils; -import com.uber.hoodie.exception.HoodieNotSerializableException; +import com.uber.hoodie.common.util.SerializationUtils; +import com.uber.hoodie.common.util.collection.Pair; +import com.uber.hoodie.common.util.collection.Triple; +import com.uber.hoodie.exception.HoodieSerializationException; import java.io.IOException; import java.util.Optional; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; -import org.apache.commons.lang3.SerializationUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.commons.lang3.tuple.Triple; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -71,7 +71,7 @@ public class HoodieRecordConverter implements hoodieRecord.getKey().getPartitionPath()), Pair.of(currentLocation, newLocation), val); return SerializationUtils.serialize(data); } catch (IOException io) { - throw new HoodieNotSerializableException("Cannot serialize value to bytes", io); + throw new HoodieSerializationException("Cannot serialize value to bytes", io); } } @@ -103,7 +103,7 @@ public class HoodieRecordConverter implements hoodieRecord.setNewLocation(newLocation); return hoodieRecord; } catch (IOException io) { - throw new HoodieNotSerializableException("Cannot de-serialize value from bytes", io); + throw new HoodieSerializationException("Cannot de-serialize value from bytes", io); } } } diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java b/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java index 1e7664f48..add92005f 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java @@ -23,6 +23,7 @@ import com.uber.hoodie.exception.HoodieException; import java.util.Arrays; import java.util.List; import java.util.Optional; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorCompletionService; import java.util.concurrent.ExecutorService; @@ -30,7 +31,6 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.commons.lang3.concurrent.ConcurrentUtils; import org.apache.log4j.LogManager; import org.apache.log4j.Logger; @@ -130,7 +130,7 @@ public class BoundedInMemoryExecutor { throw e; } }); - }).orElse(ConcurrentUtils.constantFuture(null)); + }).orElse(CompletableFuture.completedFuture(null)); } /** diff --git a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieNotSerializableException.java b/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieSerializationException.java similarity index 76% rename from hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieNotSerializableException.java rename to hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieSerializationException.java index 17650c79d..fc09a3cc7 100644 --- a/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieNotSerializableException.java +++ b/hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieSerializationException.java @@ -18,16 +18,20 @@ package com.uber.hoodie.exception; import java.io.IOException; -public class HoodieNotSerializableException extends HoodieException { +public class HoodieSerializationException extends HoodieException { private IOException ioException; - public HoodieNotSerializableException(String msg, IOException t) { + public HoodieSerializationException(String msg, IOException t) { super(msg, t); this.ioException = t; } - public HoodieNotSerializableException(String msg) { + public HoodieSerializationException(String msg, Exception t) { + super(msg, t); + } + + public HoodieSerializationException(String msg) { super(msg); } diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java b/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java index 475cc4f80..62877bd2c 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java +++ b/hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java @@ -43,6 +43,7 @@ import com.uber.hoodie.common.util.AvroUtils; import com.uber.hoodie.common.util.CompactionUtils; import com.uber.hoodie.common.util.FSUtils; import com.uber.hoodie.common.util.HoodieAvroUtils; +import com.uber.hoodie.common.util.collection.Pair; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; @@ -65,7 +66,6 @@ import java.util.stream.Stream; import org.apache.avro.Schema; import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.IndexedRecord; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java b/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java index 3035712ea..b650695f8 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java +++ b/hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java @@ -37,6 +37,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant.State; import com.uber.hoodie.common.util.AvroUtils; import com.uber.hoodie.common.util.CompactionUtils; import com.uber.hoodie.common.util.FSUtils; +import com.uber.hoodie.common.util.collection.Pair; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -48,7 +49,6 @@ import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.junit.Before; diff --git a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java b/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java index 65edb2de5..7557b5087 100644 --- a/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java +++ b/hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java @@ -30,6 +30,7 @@ import com.uber.hoodie.common.model.HoodieTestUtils; import com.uber.hoodie.common.table.HoodieTableMetaClient; import com.uber.hoodie.common.table.timeline.HoodieInstant; import com.uber.hoodie.common.table.timeline.HoodieInstant.State; +import com.uber.hoodie.common.util.collection.Pair; import com.uber.hoodie.exception.HoodieIOException; import java.io.IOException; import java.util.Arrays; @@ -42,7 +43,6 @@ import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.Path; import org.junit.Assert; import org.junit.Before; diff --git a/hoodie-spark/pom.xml b/hoodie-spark/pom.xml index 330f8547f..3942dac3e 100644 --- a/hoodie-spark/pom.xml +++ b/hoodie-spark/pom.xml @@ -177,11 +177,6 @@ avro - - org.apache.commons - commons-lang3 - - org.apache.commons commons-configuration2 diff --git a/pom.xml b/pom.xml index f0e960fa1..9f3394a72 100644 --- a/pom.xml +++ b/pom.xml @@ -585,11 +585,6 @@ hive-metastore ${hive.version}-cdh${cdh.version} - - org.apache.commons - commons-lang3 - 3.4 - junit