1
0

Implement Compaction policy abstraction. Implement LogSizeBased Bounded IO Compaction as the default strategy

This commit is contained in:
Prasanna Rajaperumal
2017-04-04 12:37:28 -07:00
committed by prazanna
parent 82b211d2e6
commit 91b088f29f
18 changed files with 585 additions and 68 deletions

View File

@@ -75,6 +75,9 @@ public class HoodieLogFile {
return fileStatus;
}
public Optional<Long> getFileSize() {
return fileStatus.map(FileStatus::getLen);
}
public HoodieLogFile rollOver(FileSystem fs) throws IOException {
String fileId = getFileId();

View File

@@ -351,4 +351,8 @@ public class FSUtils {
fs.mkdirs(partitionPath);
}
}
public static Long getSizeInMB(long sizeInBytes) {
return sizeInBytes / (1024 * 1024);
}
}

View File

@@ -18,6 +18,7 @@ package com.uber.hoodie.common.util;
import com.uber.hoodie.common.model.HoodieRecordPayload;
import com.uber.hoodie.exception.HoodieException;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -40,4 +41,21 @@ public class ReflectionUtils {
throw new IOException("Could not load payload class " + recordPayloadClass, e);
}
}
public static <T> T loadClass(String fqcn) {
try {
if(clazzCache.get(fqcn) == null) {
Class<?> clazz = Class.<HoodieRecordPayload>forName(fqcn);
clazzCache.put(fqcn, clazz);
}
return (T) clazzCache.get(fqcn).newInstance();
} catch (ClassNotFoundException e) {
throw new HoodieException("Could not load class " + fqcn, e);
} catch (InstantiationException e) {
throw new HoodieException("Could not load class " + fqcn, e);
} catch (IllegalAccessException e) {
throw new HoodieException("Could not load class " + fqcn, e);
}
}
}