lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-625] Fixing performance issues around DiskBasedMap & kryo (#1352)

Browse Source
This commit is contained in:
lamber-ken
2020-02-25 14:40:37 +08:00
committed by GitHub
parent 4e7fcde4a6
commit 83c8ad5a38
1 changed files with 3 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
hudi-common/src/main/java/org/apache/hudi/common/util/SerializationUtils.java
View File

@@ -21,18 +21,13 @@ package org.apache.hudi.common.util;
import org.apache.hudi.exception.HoodieSerializationException; import org.apache.hudi.exception.HoodieSerializationException;
import com.esotericsoftware.kryo.Kryo; import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.Serializer;
import com.esotericsoftware.kryo.io.Input; import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output; import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.serializers.FieldSerializer; import org.objenesis.strategy.StdInstantiatorStrategy;
import com.esotericsoftware.reflectasm.ConstructorAccess;
import org.objenesis.instantiator.ObjectInstantiator;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.io.Serializable; import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
/** /**
* {@link SerializationUtils} class internally uses {@link Kryo} serializer for serializing / deserializing objects. * {@link SerializationUtils} class internally uses {@link Kryo} serializer for serializing / deserializing objects.
@@ -121,50 +116,16 @@ public class SerializationUtils {
public Kryo newKryo() { public Kryo newKryo() {
Kryo kryo = new KryoBase(); Kryo kryo = new Kryo();
// ensure that kryo doesn't fail if classes are not registered with kryo. // ensure that kryo doesn't fail if classes are not registered with kryo.
kryo.setRegistrationRequired(false); kryo.setRegistrationRequired(false);
// This would be used for object initialization if nothing else works out. // This would be used for object initialization if nothing else works out.
kryo.setInstantiatorStrategy(new org.objenesis.strategy.StdInstantiatorStrategy()); kryo.setInstantiatorStrategy(new Kryo.DefaultInstantiatorStrategy(new StdInstantiatorStrategy()));
// Handle cases where we may have an odd classloader setup like with libjars // Handle cases where we may have an odd classloader setup like with libjars
// for hadoop // for hadoop
kryo.setClassLoader(Thread.currentThread().getContextClassLoader()); kryo.setClassLoader(Thread.currentThread().getContextClassLoader());
return kryo; return kryo;
} }
private static class KryoBase extends Kryo {
@Override
protected Serializer newDefaultSerializer(Class type) {
final Serializer serializer = super.newDefaultSerializer(type);
if (serializer instanceof FieldSerializer) {
final FieldSerializer fieldSerializer = (FieldSerializer) serializer;
fieldSerializer.setIgnoreSyntheticFields(true);
}
return serializer;
}
@Override
protected ObjectInstantiator newInstantiator(Class type) {
return () -> {
// First try reflectasm - it is fastest way to instantiate an object.
try {
final ConstructorAccess access = ConstructorAccess.get(type);
return access.newInstance();
} catch (Throwable t) {
// ignore this exception. We may want to try other way.
}
// fall back to java based instantiation.
try {
final Constructor constructor = type.getConstructor();
constructor.setAccessible(true);
return constructor.newInstance();
} catch (NoSuchMethodException | IllegalAccessException | InstantiationException
| InvocationTargetException e) {
// ignore this exception. we will fall back to default instantiation strategy.
}
return super.getInstantiatorStrategy().newInstantiatorOf(type).newInstance();
};
}
}
} }
} }