lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

Removing dependency on apache-commons lang 3, adding necessary classes as needed

Browse Source
This commit is contained in:
Nishith Agarwal
2018-09-04 02:05:15 -07:00
committed by vinoth chandar
parent 2eaa42abde
commit 324de298bc
23 changed files with 812 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
View File

@@ -36,6 +36,7 @@ import com.uber.hoodie.common.table.log.HoodieMergedLogRecordScanner;
import com.uber.hoodie.common.util.CompactionUtils;
import com.uber.hoodie.common.util.FSUtils;
import com.uber.hoodie.common.util.HoodieAvroUtils;
import com.uber.hoodie.common.util.collection.Pair;
import com.uber.hoodie.config.HoodieWriteConfig;
import com.uber.hoodie.io.compact.strategy.CompactionStrategy;
import com.uber.hoodie.table.HoodieCopyOnWriteTable;
@@ -49,7 +50,6 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import org.apache.avro.Schema;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

2
hoodie-client/src/test/java/com/uber/hoodie/TestAsyncCompaction.java
View File

@@ -36,6 +36,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant;
import com.uber.hoodie.common.table.view.HoodieTableFileSystemView;
import com.uber.hoodie.common.util.AvroUtils;
import com.uber.hoodie.common.util.CompactionUtils;
import com.uber.hoodie.common.util.collection.Pair;
import com.uber.hoodie.config.HoodieCompactionConfig;
import com.uber.hoodie.config.HoodieIndexConfig;
import com.uber.hoodie.config.HoodieStorageConfig;
@@ -49,7 +50,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.FileStatus;
import org.apache.spark.api.java.JavaRDD;
import org.junit.Test;

2
hoodie-client/src/test/java/com/uber/hoodie/TestCleaner.java
View File

@@ -46,6 +46,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant.State;
import com.uber.hoodie.common.util.AvroUtils;
import com.uber.hoodie.common.util.CompactionUtils;
import com.uber.hoodie.common.util.FSUtils;
import com.uber.hoodie.common.util.collection.Pair;
import com.uber.hoodie.config.HoodieCompactionConfig;
import com.uber.hoodie.config.HoodieWriteConfig;
import com.uber.hoodie.index.HoodieIndex;
@@ -64,7 +65,6 @@ import java.util.TreeSet;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;

4
hoodie-common/pom.xml
View File

@@ -110,10 +110,6 @@
<artifactId>hadoop-common</artifactId>
<classifier>tests</classifier>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo</artifactId>

2
hoodie-common/src/main/java/com/uber/hoodie/common/table/HoodieTimeline.java
View File

@@ -19,11 +19,11 @@ package com.uber.hoodie.common.table;
import com.uber.hoodie.common.table.timeline.HoodieDefaultTimeline;
import com.uber.hoodie.common.table.timeline.HoodieInstant;
import com.uber.hoodie.common.table.timeline.HoodieInstant.State;
import com.uber.hoodie.common.util.StringUtils;
import java.io.Serializable;
import java.util.Optional;
import java.util.function.BiPredicate;
import java.util.stream.Stream;
import org.apache.commons.lang3.StringUtils;
/**
* HoodieTimeline is a view of meta-data instants in the hoodie dataset. Instants are specific

4
hoodie-common/src/main/java/com/uber/hoodie/common/table/log/block/HoodieDeleteBlock.java
View File

@@ -18,6 +18,7 @@ package com.uber.hoodie.common.table.log.block;
import com.uber.hoodie.common.model.HoodieLogFile;
import com.uber.hoodie.common.storage.SizeAwareDataInputStream;
import com.uber.hoodie.common.util.StringUtils;
import com.uber.hoodie.exception.HoodieIOException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -28,7 +29,6 @@ import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
/**
@@ -64,7 +64,7 @@ public class HoodieDeleteBlock extends HoodieLogBlock {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream output = new DataOutputStream(baos);
byte[] bytesToWrite = StringUtils.join(getKeysToDelete(), ',')
byte[] bytesToWrite = StringUtils.join(getKeysToDelete(), ",")
.getBytes(Charset.forName("utf-8"));
output.writeInt(HoodieLogBlock.version);
output.writeInt(bytesToWrite.length);

5
hoodie-common/src/main/java/com/uber/hoodie/common/table/timeline/HoodieActiveTimeline.java
View File

@@ -24,6 +24,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant.State;
import com.uber.hoodie.exception.HoodieIOException;
import java.io.IOException;
import java.io.Serializable;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
@@ -32,7 +33,6 @@ import java.util.Set;
import java.util.function.Function;
import java.util.stream.Stream;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -49,8 +49,7 @@ import org.apache.log4j.Logger;
*/
public class HoodieActiveTimeline extends HoodieDefaultTimeline {
public static final FastDateFormat COMMIT_FORMATTER = FastDateFormat
.getInstance("yyyyMMddHHmmss");
public static final SimpleDateFormat COMMIT_FORMATTER = new SimpleDateFormat("yyyyMMddHHmmss");
public static final Set<String> VALID_EXTENSIONS_IN_ACTIVE_TIMELINE = new HashSet<>(Arrays.asList(
new String[]{COMMIT_EXTENSION, INFLIGHT_COMMIT_EXTENSION, DELTA_COMMIT_EXTENSION,

2
hoodie-common/src/main/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemView.java
View File

@@ -28,6 +28,7 @@ import com.uber.hoodie.common.table.TableFileSystemView;
import com.uber.hoodie.common.table.timeline.HoodieInstant;
import com.uber.hoodie.common.util.CompactionUtils;
import com.uber.hoodie.common.util.FSUtils;
import com.uber.hoodie.common.util.collection.Pair;
import com.uber.hoodie.exception.HoodieIOException;
import java.io.IOException;
import java.io.Serializable;
@@ -42,7 +43,6 @@ import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;

2
hoodie-common/src/main/java/com/uber/hoodie/common/util/CompactionUtils.java
View File

@@ -23,6 +23,7 @@ import com.uber.hoodie.common.model.FileSlice;
import com.uber.hoodie.common.table.HoodieTableMetaClient;
import com.uber.hoodie.common.table.HoodieTimeline;
import com.uber.hoodie.common.table.timeline.HoodieInstant;
import com.uber.hoodie.common.util.collection.Pair;
import com.uber.hoodie.exception.HoodieException;
import java.io.IOException;
import java.util.HashMap;
@@ -32,7 +33,6 @@ import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
/**
* Helper class to generate compaction plan from FileGroup/FileSlice abstraction

171
hoodie-common/src/main/java/com/uber/hoodie/common/util/SerializationUtils.java Normal file
View File

@@ -0,0 +1,171 @@
/*
* Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.uber.hoodie.common.util;
import com.uber.hoodie.exception.HoodieSerializationException;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
/**
* (NOTE: Adapted from Apache commons-lang3)
* This class defines API's to serde an object.
*/
public class SerializationUtils {
// Serialize
//-----------------------------------------------------------------------
/**
* <p>Serializes an {@code Object} to the specified stream.</p>
*
* <p>The stream will be closed once the object is written.
* This avoids the need for a finally clause, and maybe also exception
* handling, in the application code.</p>
*
* <p>The stream passed in is not buffered internally within this method.
* This is the responsibility of your application if desired.</p>
*
* @param obj the object to serialize to bytes, may be null
* @param outputStream the stream to write to, must not be null
* @throws IllegalArgumentException if {@code outputStream} is {@code null}
* @throws HoodieSerializationException (runtime) if the serialization fails
*/
public static void serialize(final Serializable obj, final OutputStream outputStream) {
if (outputStream == null) {
throw new IllegalArgumentException("The OutputStream must not be null");
}
ObjectOutputStream out = null;
try {
// stream closed in the finally
out = new ObjectOutputStream(outputStream);
out.writeObject(obj);
} catch (final IOException ex) {
throw new HoodieSerializationException("unable to serialize object", ex);
} finally {
try {
if (out != null) {
out.close();
}
} catch (final IOException ex) { // NOPMD
// ignore close exception
}
}
}
/**
* <p>Serializes an {@code Object} to a byte array for
* storage/serialization.</p>
*
* @param obj the object to serialize to bytes
* @return a byte[] with the converted Serializable
* @throws HoodieSerializationException (runtime) if the serialization fails
*/
public static byte[] serialize(final Serializable obj) {
final ByteArrayOutputStream baos = new ByteArrayOutputStream(512);
serialize(obj, baos);
return baos.toByteArray();
}
// Deserialize
//-----------------------------------------------------------------------
/**
* <p>
* Deserializes an {@code Object} from the specified stream.
* </p>
*
* <p>
* The stream will be closed once the object is written. This avoids the need for a finally clause, and maybe also
* exception handling, in the application code.
* </p>
*
* <p>
* The stream passed in is not buffered internally within this method. This is the responsibility of your
* application if desired.
* </p>
*
* <p>
* If the call site incorrectly types the return value, a {@link ClassCastException} is thrown from the call site.
* Without Generics in this declaration, the call site must type cast and can cause the same ClassCastException.
* Note that in both cases, the ClassCastException is in the call site, not in this method.
* </p>
*
* @param <T> the object type to be deserialized
* @param inputStream the serialized object input stream, must not be null
* @return the deserialized object
* @throws IllegalArgumentException if {@code inputStream} is {@code null}
* @throws HoodieSerializationException (runtime) if the serialization fails
*/
public static <T> T deserialize(final InputStream inputStream) {
if (inputStream == null) {
throw new IllegalArgumentException("The InputStream must not be null");
}
ObjectInputStream in = null;
try {
// stream closed in the finally
in = new ObjectInputStream(inputStream);
@SuppressWarnings("unchecked") // may fail with CCE if serialised form is incorrect
final T obj = (T) in.readObject();
return obj;
} catch (final ClassCastException ex) {
throw new HoodieSerializationException("cannot cast class", ex);
} catch (final ClassNotFoundException ex) {
throw new HoodieSerializationException("class not found", ex);
} catch (final IOException ex) {
throw new HoodieSerializationException("unable to deserialize to object", ex);
} finally {
try {
if (in != null) {
in.close();
}
} catch (final IOException ex) { // NOPMD
// ignore close exception
}
}
}
/**
* <p>
* Deserializes a single {@code Object} from an array of bytes.
* </p>
*
* <p>
* If the call site incorrectly types the return value, a {@link ClassCastException} is thrown from the call site.
* Without Generics in this declaration, the call site must type cast and can cause the same ClassCastException.
* Note that in both cases, the ClassCastException is in the call site, not in this method.
* </p>
*
* @param <T> the object type to be deserialized
* @param objectData the serialized object, must not be null
* @return the deserialized object
* @throws IllegalArgumentException if {@code objectData} is {@code null}
* @throws HoodieSerializationException (runtime) if the serialization fails
*/
public static <T> T deserialize(final byte[] objectData) {
if (objectData == null) {
throw new IllegalArgumentException("The byte[] must not be null");
}
return deserialize(new ByteArrayInputStream(objectData));
}
}

51
hoodie-common/src/main/java/com/uber/hoodie/common/util/StringUtils.java Normal file
View File

@@ -0,0 +1,51 @@
/*
* Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.uber.hoodie.common.util;
/**
* Simple utility for operations on strings
*/
public class StringUtils {
/**
* <p>Joins the elements of the provided array into a single String
* containing the provided list of elements.</p>
*
* <p>No separator is added to the joined String.
* Null objects or empty strings within the array are represented by
* empty strings.</p>
*
* <pre>
* StringUtils.join(null) = null
* StringUtils.join([]) = ""
* StringUtils.join([null]) = ""
* StringUtils.join(["a", "b", "c"]) = "abc"
* StringUtils.join([null, "", "a"]) = "a"
* </pre>
*/
public static <T> String join(final String... elements) {
return join(elements, "");
}
public static String join(final String[] array, final String separator) {
if (array == null) {
return null;
}
return org.apache.hadoop.util.StringUtils.join(separator, array);
}
}

109
hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutablePair.java Normal file
View File

@@ -0,0 +1,109 @@
/*
* Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.uber.hoodie.common.util.collection;
/**
* (NOTE: Adapted from Apache commons-lang3)
* <p>An immutable pair consisting of two {@code Object} elements.</p>
*
* <p>Although the implementation is immutable, there is no restriction on the objects
* that may be stored. If mutable objects are stored in the pair, then the pair
* itself effectively becomes mutable. The class is also {@code final}, so a subclass
* can not add undesirable behaviour.</p>
*
* <p>#ThreadSafe# if both paired objects are thread-safe</p>
*
* @param <L> the left element type
* @param <R> the right element type
*/
public final class ImmutablePair<L, R> extends Pair<L, R> {
/**
* Serialization version
*/
private static final long serialVersionUID = 4954918890077093841L;
/**
* Left object
*/
public final L left;
/**
* Right object
*/
public final R right;
/**
* <p>Obtains an immutable pair of from two objects inferring the generic types.</p>
*
* <p>This factory allows the pair to be created using inference to
* obtain the generic types.</p>
*
* @param <L> the left element type
* @param <R> the right element type
* @param left the left element, may be null
* @param right the right element, may be null
* @return a pair formed from the two parameters, not null
*/
public static <L, R> ImmutablePair<L, R> of(final L left, final R right) {
return new ImmutablePair<L, R>(left, right);
}
/**
* Create a new pair instance.
*
* @param left the left value, may be null
* @param right the right value, may be null
*/
public ImmutablePair(final L left, final R right) {
super();
this.left = left;
this.right = right;
}
//-----------------------------------------------------------------------
/**
* {@inheritDoc}
*/
@Override
public L getLeft() {
return left;
}
/**
* {@inheritDoc}
*/
@Override
public R getRight() {
return right;
}
/**
* <p>Throws {@code UnsupportedOperationException}.</p>
*
* <p>This pair is immutable, so this operation is not supported.</p>
*
* @param value the value to set
* @return never
* @throws UnsupportedOperationException as this operation is not supported
*/
@Override
public R setValue(final R value) {
throw new UnsupportedOperationException();
}
}

112
hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/ImmutableTriple.java Normal file
View File

@@ -0,0 +1,112 @@
/*
* Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.uber.hoodie.common.util.collection;
/**
* (NOTE: Adapted from Apache commons-lang3)
* <p>An immutable triple consisting of three {@code Object} elements.</p>
*
* <p>Although the implementation is immutable, there is no restriction on the objects
* that may be stored. If mutable objects are stored in the triple, then the triple
* itself effectively becomes mutable. The class is also {@code final}, so a subclass
* can not add undesirable behaviour.</p>
*
* <p>#ThreadSafe# if all three objects are thread-safe</p>
*
* @param <L> the left element type
* @param <M> the middle element type
* @param <R> the right element type
*/
public final class ImmutableTriple<L, M, R> extends Triple<L, M, R> {
/**
* Serialization version
*/
private static final long serialVersionUID = 1L;
/**
* Left object
*/
public final L left;
/**
* Middle object
*/
public final M middle;
/**
* Right object
*/
public final R right;
/**
* <p>Obtains an immutable triple of from three objects inferring the generic types.</p>
*
* <p>This factory allows the triple to be created using inference to
* obtain the generic types.</p>
*
* @param <L> the left element type
* @param <M> the middle element type
* @param <R> the right element type
* @param left the left element, may be null
* @param middle the middle element, may be null
* @param right the right element, may be null
* @return a triple formed from the three parameters, not null
*/
public static <L, M, R> ImmutableTriple<L, M, R> of(final L left, final M middle, final R right) {
return new ImmutableTriple<L, M, R>(left, middle, right);
}
/**
* Create a new triple instance.
*
* @param left the left value, may be null
* @param middle the middle value, may be null
* @param right the right value, may be null
*/
public ImmutableTriple(final L left, final M middle, final R right) {
super();
this.left = left;
this.middle = middle;
this.right = right;
}
//-----------------------------------------------------------------------
/**
* {@inheritDoc}
*/
@Override
public L getLeft() {
return left;
}
/**
* {@inheritDoc}
*/
@Override
public M getMiddle() {
return middle;
}
/**
* {@inheritDoc}
*/
@Override
public R getRight() {
return right;
}
}

179
hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Pair.java Normal file
View File

@@ -0,0 +1,179 @@
/*
* Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.uber.hoodie.common.util.collection;
import java.io.Serializable;
import java.util.Map;
import org.apache.commons.lang.builder.CompareToBuilder;
/**
* (NOTE: Adapted from Apache commons-lang3)
* <p>A pair consisting of two elements.</p>
*
* <p>This class is an abstract implementation defining the basic API.
* It refers to the elements as 'left' and 'right'. It also implements the
* {@code Map.Entry} interface where the key is 'left' and the value is 'right'.</p>
*
* <p>Subclass implementations may be mutable or immutable.
* However, there is no restriction on the type of the stored objects that may be stored.
* If mutable objects are stored in the pair, then the pair itself effectively becomes mutable.</p>
*
* @param <L> the left element type
* @param <R> the right element type
*/
public abstract class Pair<L, R> implements Map.Entry<L, R>, Comparable<Pair<L, R>>, Serializable {
/**
* Serialization version
*/
private static final long serialVersionUID = 4954918890077093841L;
/**
* <p>Obtains an immutable pair of from two objects inferring the generic types.</p>
*
* <p>This factory allows the pair to be created using inference to
* obtain the generic types.</p>
*
* @param <L> the left element type
* @param <R> the right element type
* @param left the left element, may be null
* @param right the right element, may be null
* @return a pair formed from the two parameters, not null
*/
public static <L, R> Pair<L, R> of(final L left, final R right) {
return new ImmutablePair<L, R>(left, right);
}
//-----------------------------------------------------------------------
/**
* <p>Gets the left element from this pair.</p>
*
* <p>When treated as a key-value pair, this is the key.</p>
*
* @return the left element, may be null
*/
public abstract L getLeft();
/**
* <p>Gets the right element from this pair.</p>
*
* <p>When treated as a key-value pair, this is the value.</p>
*
* @return the right element, may be null
*/
public abstract R getRight();
/**
* <p>Gets the key from this pair.</p>
*
* <p>This method implements the {@code Map.Entry} interface returning the
* left element as the key.</p>
*
* @return the left element as the key, may be null
*/
@Override
public final L getKey() {
return getLeft();
}
/**
* <p>Gets the value from this pair.</p>
*
* <p>This method implements the {@code Map.Entry} interface returning the
* right element as the value.</p>
*
* @return the right element as the value, may be null
*/
@Override
public R getValue() {
return getRight();
}
//-----------------------------------------------------------------------
/**
* <p>Compares the pair based on the left element followed by the right element.
* The types must be {@code Comparable}.</p>
*
* @param other the other pair, not null
* @return negative if this is less, zero if equal, positive if greater
*/
@Override
public int compareTo(final Pair<L, R> other) {
return new CompareToBuilder().append(getLeft(), other.getLeft())
.append(getRight(), other.getRight()).toComparison();
}
/**
* <p>Compares this pair to another based on the two elements.</p>
*
* @param obj the object to compare to, null returns false
* @return true if the elements of the pair are equal
*/
@Override
public boolean equals(final Object obj) {
if (obj == this) {
return true;
}
if (obj instanceof Map.Entry<?, ?>) {
final Map.Entry<?, ?> other = (Map.Entry<?, ?>) obj;
return getKey().equals(other.getKey())
&& getValue().equals(other.getValue());
}
return false;
}
/**
* <p>Returns a suitable hash code.
* The hash code follows the definition in {@code Map.Entry}.</p>
*
* @return the hash code
*/
@Override
public int hashCode() {
// see Map.Entry API specification
return (getKey() == null ? 0 : getKey().hashCode())
^ (getValue() == null ? 0 : getValue().hashCode());
}
/**
* <p>Returns a String representation of this pair using the format {@code ($left,$right)}.</p>
*
* @return a string describing this object, not null
*/
@Override
public String toString() {
return new StringBuilder().append('(').append(getLeft()).append(',').append(getRight()).append(')').toString();
}
/**
* <p>Formats the receiver using the given format.</p>
*
* <p>This uses {@link java.util.Formattable} to perform the formatting. Two variables may
* be used to embed the left and right elements. Use {@code %1$s} for the left
* element (key) and {@code %2$s} for the right element (value).
* The default format used by {@code toString()} is {@code (%1$s,%2$s)}.</p>
*
* @param format the format string, optionally containing {@code %1$s} and {@code %2$s}, not null
* @return the formatted string, not null
*/
public String toString(final String format) {
return String.format(format, getLeft(), getRight());
}
}

162
hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/Triple.java Normal file
View File

@@ -0,0 +1,162 @@
/*
* Copyright (c) 2018 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.uber.hoodie.common.util.collection;
import java.io.Serializable;
import org.apache.commons.lang.builder.CompareToBuilder;
/**
* (NOTE: Adapted from Apache commons-lang3)
* <p>A triple consisting of three elements.</p>
*
* <p>This class is an abstract implementation defining the basic API.
* It refers to the elements as 'left', 'middle' and 'right'.</p>
*
* <p>Subclass implementations may be mutable or immutable.
* However, there is no restriction on the type of the stored objects that may be stored.
* If mutable objects are stored in the triple, then the triple itself effectively becomes mutable.</p>
*
* @param <L> the left element type
* @param <M> the middle element type
* @param <R> the right element type
*/
public abstract class Triple<L, M, R> implements Comparable<Triple<L, M, R>>, Serializable {
/**
* Serialization version
*/
private static final long serialVersionUID = 1L;
/**
* <p>Obtains an immutable triple of from three objects inferring the generic types.</p>
*
* <p>This factory allows the triple to be created using inference to
* obtain the generic types.</p>
*
* @param <L> the left element type
* @param <M> the middle element type
* @param <R> the right element type
* @param left the left element, may be null
* @param middle the middle element, may be null
* @param right the right element, may be null
* @return a triple formed from the three parameters, not null
*/
public static <L, M, R> Triple<L, M, R> of(final L left, final M middle, final R right) {
return new ImmutableTriple<L, M, R>(left, middle, right);
}
//-----------------------------------------------------------------------
/**
* <p>Gets the left element from this triple.</p>
*
* @return the left element, may be null
*/
public abstract L getLeft();
/**
* <p>Gets the middle element from this triple.</p>
*
* @return the middle element, may be null
*/
public abstract M getMiddle();
/**
* <p>Gets the right element from this triple.</p>
*
* @return the right element, may be null
*/
public abstract R getRight();
//-----------------------------------------------------------------------
/**
* <p>Compares the triple based on the left element, followed by the middle element,
* finally the right element.
* The types must be {@code Comparable}.</p>
*
* @param other the other triple, not null
* @return negative if this is less, zero if equal, positive if greater
*/
@Override
public int compareTo(final Triple<L, M, R> other) {
return new CompareToBuilder().append(getLeft(), other.getLeft())
.append(getMiddle(), other.getMiddle())
.append(getRight(), other.getRight()).toComparison();
}
/**
* <p>Compares this triple to another based on the three elements.</p>
*
* @param obj the object to compare to, null returns false
* @return true if the elements of the triple are equal
*/
@SuppressWarnings("deprecation") // ObjectUtils.equals(Object, Object) has been deprecated in 3.2
@Override
public boolean equals(final Object obj) {
if (obj == this) {
return true;
}
if (obj instanceof Triple<?, ?, ?>) {
final Triple<?, ?, ?> other = (Triple<?, ?, ?>) obj;
return getLeft().equals(other.getLeft())
&& getMiddle().equals(other.getMiddle())
&& getRight().equals(other.getRight());
}
return false;
}
/**
* <p>Returns a suitable hash code.</p>
*
* @return the hash code
*/
@Override
public int hashCode() {
return (getLeft() == null ? 0 : getLeft().hashCode())
^ (getMiddle() == null ? 0 : getMiddle().hashCode())
^ (getRight() == null ? 0 : getRight().hashCode());
}
/**
* <p>Returns a String representation of this triple using the format {@code ($left,$middle,$right)}.</p>
*
* @return a string describing this object, not null
*/
@Override
public String toString() {
return new StringBuilder().append('(').append(getLeft()).append(',').append(getMiddle()).append(',')
.append(getRight()).append(')').toString();
}
/**
* <p>Formats the receiver using the given format.</p>
*
* <p>This uses {@link java.util.Formattable} to perform the formatting. Three variables may
* be used to embed the left and right elements. Use {@code %1$s} for the left
* element, {@code %2$s} for the middle and {@code %3$s} for the right element.
* The default format used by {@code toString()} is {@code (%1$s,%2$s,%3$s)}.</p>
*
* @param format the format string, optionally containing {@code %1$s}, {@code %2$s} and {@code %3$s}, not null
* @return the formatted string, not null
*/
public String toString(final String format) {
return String.format(format, getLeft(), getMiddle(), getRight());
}
}

12
hoodie-common/src/main/java/com/uber/hoodie/common/util/collection/converter/HoodieRecordConverter.java
View File

@@ -22,14 +22,14 @@ import com.uber.hoodie.common.model.HoodieRecordLocation;
import com.uber.hoodie.common.model.HoodieRecordPayload;
import com.uber.hoodie.common.util.HoodieAvroUtils;
import com.uber.hoodie.common.util.ReflectionUtils;
import com.uber.hoodie.exception.HoodieNotSerializableException;
import com.uber.hoodie.common.util.SerializationUtils;
import com.uber.hoodie.common.util.collection.Pair;
import com.uber.hoodie.common.util.collection.Triple;
import com.uber.hoodie.exception.HoodieSerializationException;
import java.io.IOException;
import java.util.Optional;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.commons.lang3.SerializationUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.tuple.Triple;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -71,7 +71,7 @@ public class HoodieRecordConverter<V> implements
hoodieRecord.getKey().getPartitionPath()), Pair.of(currentLocation, newLocation), val);
return SerializationUtils.serialize(data);
} catch (IOException io) {
throw new HoodieNotSerializableException("Cannot serialize value to bytes", io);
throw new HoodieSerializationException("Cannot serialize value to bytes", io);
}
}
@@ -103,7 +103,7 @@ public class HoodieRecordConverter<V> implements
hoodieRecord.setNewLocation(newLocation);
return hoodieRecord;
} catch (IOException io) {
throw new HoodieNotSerializableException("Cannot de-serialize value from bytes", io);
throw new HoodieSerializationException("Cannot de-serialize value from bytes", io);
}
}
}

4
hoodie-common/src/main/java/com/uber/hoodie/common/util/queue/BoundedInMemoryExecutor.java
View File

@@ -23,6 +23,7 @@ import com.uber.hoodie.exception.HoodieException;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
@@ -30,7 +31,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.concurrent.ConcurrentUtils;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -130,7 +130,7 @@ public class BoundedInMemoryExecutor<I, O, E> {
throw e;
}
});
}).orElse(ConcurrentUtils.constantFuture(null));
}).orElse(CompletableFuture.completedFuture(null));
}
/**

10
hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieNotSerializableException.java → hoodie-common/src/main/java/com/uber/hoodie/exception/HoodieSerializationException.java
View File

@@ -18,16 +18,20 @@ package com.uber.hoodie.exception;
import java.io.IOException;
public class HoodieNotSerializableException extends HoodieException {
public class HoodieSerializationException extends HoodieException {
private IOException ioException;
public HoodieNotSerializableException(String msg, IOException t) {
public HoodieSerializationException(String msg, IOException t) {
super(msg, t);
this.ioException = t;
}
public HoodieNotSerializableException(String msg) {
public HoodieSerializationException(String msg, Exception t) {
super(msg, t);
}
public HoodieSerializationException(String msg) {
super(msg);
}

2
hoodie-common/src/test/java/com/uber/hoodie/common/model/HoodieTestUtils.java
View File

@@ -43,6 +43,7 @@ import com.uber.hoodie.common.util.AvroUtils;
import com.uber.hoodie.common.util.CompactionUtils;
import com.uber.hoodie.common.util.FSUtils;
import com.uber.hoodie.common.util.HoodieAvroUtils;
import com.uber.hoodie.common.util.collection.Pair;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -65,7 +66,6 @@ import java.util.stream.Stream;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;

2
hoodie-common/src/test/java/com/uber/hoodie/common/table/view/HoodieTableFileSystemViewTest.java
View File

@@ -37,6 +37,7 @@ import com.uber.hoodie.common.table.timeline.HoodieInstant.State;
import com.uber.hoodie.common.util.AvroUtils;
import com.uber.hoodie.common.util.CompactionUtils;
import com.uber.hoodie.common.util.FSUtils;
import com.uber.hoodie.common.util.collection.Pair;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -48,7 +49,6 @@ import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.junit.Before;

2
hoodie-common/src/test/java/com/uber/hoodie/common/util/TestCompactionUtils.java
View File

@@ -30,6 +30,7 @@ import com.uber.hoodie.common.model.HoodieTestUtils;
import com.uber.hoodie.common.table.HoodieTableMetaClient;
import com.uber.hoodie.common.table.timeline.HoodieInstant;
import com.uber.hoodie.common.table.timeline.HoodieInstant.State;
import com.uber.hoodie.common.util.collection.Pair;
import com.uber.hoodie.exception.HoodieIOException;
import java.io.IOException;
import java.util.Arrays;
@@ -42,7 +43,6 @@ import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.fs.Path;
import org.junit.Assert;
import org.junit.Before;

5
hoodie-spark/pom.xml
View File

@@ -177,11 +177,6 @@
<artifactId>avro</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>

5
pom.xml
View File

@@ -585,11 +585,6 @@
<artifactId>hive-metastore</artifactId>
<version>${hive.version}-cdh${cdh.version}</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
<dependency>
<groupId>junit</groupId>