Implement reliable log file management for Merge on read, which is fault tolerant and allows random block level access on avro file

hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/HdfsTestService.java

@@ -0,0 +1,166 @@
+/*
+ *  Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.uber.hoodie.common.minicluster;
+
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.Files;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * An HDFS minicluster service implementation.
+ */
+public class HdfsTestService {
+
+    private static final Logger logger = LoggerFactory.getLogger(HdfsTestService.class);
+
+    /**
+     * Configuration settings
+     */
+    private Configuration hadoopConf;
+    private String workDir;
+    private String bindIP = "127.0.0.1";
+    private int namenodeRpcPort = 8020;
+    private int namenodeHttpPort = 50070;
+    private int datanodePort = 50010;
+    private int datanodeIpcPort = 50020;
+    private int datanodeHttpPort = 50075;
+
+    /**
+     * Embedded HDFS cluster
+     */
+    private MiniDFSCluster miniDfsCluster;
+
+    public HdfsTestService() {
+        hadoopConf = new Configuration();
+        workDir = Files.createTempDir().getAbsolutePath();
+    }
+
+    public Configuration getHadoopConf() {
+        return hadoopConf;
+    }
+
+    public MiniDFSCluster start(boolean format) throws IOException {
+        Preconditions
+            .checkState(workDir != null, "The work dir must be set before starting cluster.");
+
+        if (hadoopConf == null) {
+            hadoopConf = new Configuration();
+        }
+
+        // If clean, then remove the work dir so we can start fresh.
+        String localDFSLocation = getDFSLocation(workDir);
+        if (format) {
+            logger.info(
+                "Cleaning HDFS cluster data at: " + localDFSLocation + " and starting fresh.");
+            File file = new File(localDFSLocation);
+            FileUtils.deleteDirectory(file);
+        }
+
+        // Configure and start the HDFS cluster
+        // boolean format = shouldFormatDFSCluster(localDFSLocation, clean);
+        hadoopConf = configureDFSCluster(hadoopConf, localDFSLocation, bindIP, namenodeRpcPort,
+            namenodeHttpPort, datanodePort, datanodeIpcPort, datanodeHttpPort);
+        miniDfsCluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(1).format(format)
+            .checkDataNodeAddrConfig(true).checkDataNodeHostConfig(true).build();
+        logger.info("HDFS Minicluster service started.");
+        return miniDfsCluster;
+    }
+
+    public void stop() throws IOException {
+        miniDfsCluster.shutdown();
+        logger.info("HDFS Minicluster service shut down.");
+        miniDfsCluster = null;
+        hadoopConf = null;
+    }
+
+    /**
+     * Get the location on the local FS where we store the HDFS data.
+     *
+     * @param baseFsLocation The base location on the local filesystem we have write access to
+     *                       create dirs.
+     * @return The location for HDFS data.
+     */
+    private static String getDFSLocation(String baseFsLocation) {
+        return baseFsLocation + Path.SEPARATOR + "dfs";
+    }
+
+    /**
+     * Returns true if we should format the DFS Cluster. We'll format if clean is
+     * true, or if the dfsFsLocation does not exist.
+     *
+     * @param localDFSLocation The location on the local FS to hold the HDFS metadata and block
+     *                         data
+     * @param clean            Specifies if we want to start a clean cluster
+     * @return Returns true if we should format a DFSCluster, otherwise false
+     */
+    private static boolean shouldFormatDFSCluster(String localDFSLocation, boolean clean) {
+        boolean format = true;
+        File f = new File(localDFSLocation);
+        if (f.exists() && f.isDirectory() && !clean) {
+            format = false;
+        }
+        return format;
+    }
+
+    /**
+     * Configure the DFS Cluster before launching it.
+     *
+     * @param config           The already created Hadoop configuration we'll further configure
+     *                         for HDFS
+     * @param localDFSLocation The location on the local filesystem where cluster data is stored
+     * @param bindIP           An IP address we want to force the datanode and namenode to bind
+     *                         to.
+     * @param namenodeRpcPort
+     * @param namenodeHttpPort
+     * @param datanodePort
+     * @param datanodeIpcPort
+     * @param datanodeHttpPort
+     * @return The updated Configuration object.
+     */
+    private static Configuration configureDFSCluster(Configuration config, String localDFSLocation,
+        String bindIP, int namenodeRpcPort, int namenodeHttpPort, int datanodePort,
+        int datanodeIpcPort, int datanodeHttpPort) {
+
+        logger.info("HDFS force binding to ip: " + bindIP);
+        config.set(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + bindIP + ":" + namenodeRpcPort);
+        config.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, bindIP + ":" + datanodePort);
+        config.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, bindIP + ":" + datanodeIpcPort);
+        config.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, bindIP + ":" + datanodeHttpPort);
+        // When a datanode registers with the namenode, the Namenode do a hostname
+        // check of the datanode which will fail on OpenShift due to reverse DNS
+        // issues with the internal IP addresses. This config disables that check,
+        // and will allow a datanode to connect regardless.
+        config.setBoolean("dfs.namenode.datanode.registration.ip-hostname-check", false);
+        config.set("hdfs.minidfs.basedir", localDFSLocation);
+        // allow current user to impersonate others
+        String user = System.getProperty("user.name");
+        config.set("hadoop.proxyuser." + user + ".groups", "*");
+        config.set("hadoop.proxyuser." + user + ".hosts", "*");
+        return config;
+    }
+
+}

hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/MiniClusterUtil.java

@@ -0,0 +1,53 @@
+/*
+ *  Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.uber.hoodie.common.minicluster;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.zookeeper.server.ZooKeeperServer;
+
+import java.io.IOException;
+
+public class MiniClusterUtil {
+    private static MiniDFSCluster dfsCluster;
+    private static ZooKeeperServer zkServer;
+    public static Configuration configuration;
+    public static FileSystem fileSystem;
+
+    public static void setUp() throws IOException, InterruptedException {
+        if (dfsCluster == null) {
+            HdfsTestService service = new HdfsTestService();
+            dfsCluster = service.start(true);
+            configuration = service.getHadoopConf();
+        }
+        if (zkServer == null) {
+            ZookeeperTestService zkService = new ZookeeperTestService(configuration);
+            zkServer = zkService.start();
+        }
+        fileSystem = FileSystem.get(configuration);
+    }
+
+    public static void shutdown() {
+        if (dfsCluster != null) {
+            dfsCluster.shutdown();
+        }
+        if (zkServer != null) {
+            zkServer.shutdown();
+        }
+    }
+}

hoodie-common/src/test/java/com/uber/hoodie/common/minicluster/ZookeeperTestService.java

@@ -0,0 +1,241 @@
+/*
+ *  Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.uber.hoodie.common.minicluster;
+
+import com.google.common.base.Preconditions;
+import com.google.common.io.Files;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.zookeeper.server.NIOServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.persistence.FileTxnLog;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+
+/**
+ * A Zookeeper minicluster service implementation.
+ * <p/>
+ * This class was ripped from MiniZooKeeperCluster from the HBase tests. Changes
+ * made include:
+ * <p/>
+ * 1. It will now only launch 1 zookeeper server.
+ * <p/>
+ * 2. It will only attempt to bind to the port specified, and will fail if it
+ * can't.
+ * <p/>
+ * 3. The startup method now takes a bindAddress, which allows us to configure
+ * which IP the ZK server binds to. This was not configurable in the original
+ * class.
+ * <p/>
+ * 4. The ZK cluster will re-use a data dir on the local filesystem if it
+ * already exists instead of blowing it away.
+ */
+public class ZookeeperTestService {
+
+    private static final Logger logger = LoggerFactory.getLogger(ZookeeperTestService.class);
+
+    private static final int TICK_TIME = 2000;
+    private static final int CONNECTION_TIMEOUT = 30000;
+
+    /**
+     * Configuration settings
+     */
+    private Configuration hadoopConf;
+    private String workDir;
+    private Integer clientPort = 2828;
+    private String bindIP = "127.0.0.1";
+    private Boolean clean = false;
+    private int tickTime = 0;
+
+    /**
+     * Embedded ZooKeeper cluster
+     */
+    private NIOServerCnxnFactory standaloneServerFactory;
+    private ZooKeeperServer zooKeeperServer;
+    private boolean started = false;
+
+    public ZookeeperTestService(Configuration config) {
+        this.workDir = Files.createTempDir().getAbsolutePath();
+        this.hadoopConf = config;
+    }
+
+    public Configuration getHadoopConf() {
+        return hadoopConf;
+    }
+
+    public ZooKeeperServer start() throws IOException, InterruptedException {
+        Preconditions.checkState(workDir != null,
+            "The localBaseFsLocation must be set before starting cluster.");
+
+        setupTestEnv();
+        stop();
+
+        File dir = new File(workDir, "zookeeper").getAbsoluteFile();
+        recreateDir(dir, clean);
+        int tickTimeToUse;
+        if (this.tickTime > 0) {
+            tickTimeToUse = this.tickTime;
+        } else {
+            tickTimeToUse = TICK_TIME;
+        }
+        this.zooKeeperServer = new ZooKeeperServer(dir, dir, tickTimeToUse);
+        standaloneServerFactory = new NIOServerCnxnFactory();
+
+        // NOTE: Changed from the original, where InetSocketAddress was
+        // originally created to bind to the wildcard IP, we now configure it.
+        logger.info("Zookeeper force binding to: " + this.bindIP);
+        standaloneServerFactory.configure(new InetSocketAddress(bindIP, clientPort), 1000);
+
+        // Start up this ZK server
+        standaloneServerFactory.startup(zooKeeperServer);
+
+        String serverHostname;
+        if (bindIP.equals("0.0.0.0")) {
+            serverHostname = "localhost";
+        } else {
+            serverHostname = bindIP;
+        }
+        if (!waitForServerUp(serverHostname, clientPort, CONNECTION_TIMEOUT)) {
+            throw new IOException("Waiting for startup of standalone server");
+        }
+
+        started = true;
+        logger.info("Zookeeper Minicluster service started on client port: " + clientPort);
+        return zooKeeperServer;
+    }
+
+    public void stop() throws IOException {
+        if (!started) {
+            return;
+        }
+
+        standaloneServerFactory.shutdown();
+        if (!waitForServerDown(clientPort, CONNECTION_TIMEOUT)) {
+            throw new IOException("Waiting for shutdown of standalone server");
+        }
+
+        // clear everything
+        started = false;
+        standaloneServerFactory = null;
+        zooKeeperServer = null;
+
+        logger.info("Zookeeper Minicluster service shut down.");
+    }
+
+    private void recreateDir(File dir, boolean clean) throws IOException {
+        if (dir.exists() && clean) {
+            FileUtil.fullyDelete(dir);
+        } else if (dir.exists() && !clean) {
+            // the directory's exist, and we don't want to clean, so exit
+            return;
+        }
+        try {
+            dir.mkdirs();
+        } catch (SecurityException e) {
+            throw new IOException("creating dir: " + dir, e);
+        }
+    }
+
+    // / XXX: From o.a.zk.t.ClientBase
+    private static void setupTestEnv() {
+        // during the tests we run with 100K prealloc in the logs.
+        // on windows systems prealloc of 64M was seen to take ~15seconds
+        // resulting in test failure (client timeout on first session).
+        // set env and directly in order to handle static init/gc issues
+        System.setProperty("zookeeper.preAllocSize", "100");
+        FileTxnLog.setPreallocSize(100 * 1024);
+    }
+
+    // XXX: From o.a.zk.t.ClientBase
+    private static boolean waitForServerDown(int port, long timeout) {
+        long start = System.currentTimeMillis();
+        while (true) {
+            try {
+                Socket sock = new Socket("localhost", port);
+                try {
+                    OutputStream outstream = sock.getOutputStream();
+                    outstream.write("stat".getBytes());
+                    outstream.flush();
+                } finally {
+                    sock.close();
+                }
+            } catch (IOException e) {
+                return true;
+            }
+
+            if (System.currentTimeMillis() > start + timeout) {
+                break;
+            }
+            try {
+                Thread.sleep(250);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+        return false;
+    }
+
+    // XXX: From o.a.zk.t.ClientBase
+    private static boolean waitForServerUp(String hostname, int port, long timeout) {
+        long start = System.currentTimeMillis();
+        while (true) {
+            try {
+                Socket sock = new Socket(hostname, port);
+                BufferedReader reader = null;
+                try {
+                    OutputStream outstream = sock.getOutputStream();
+                    outstream.write("stat".getBytes());
+                    outstream.flush();
+
+                    Reader isr = new InputStreamReader(sock.getInputStream());
+                    reader = new BufferedReader(isr);
+                    String line = reader.readLine();
+                    if (line != null && line.startsWith("Zookeeper version:")) {
+                        return true;
+                    }
+                } finally {
+                    sock.close();
+                    if (reader != null) {
+                        reader.close();
+                    }
+                }
+            } catch (IOException e) {
+                // ignore as this is expected
+                logger.info("server " + hostname + ":" + port + " not up " + e);
+            }
+
+            if (System.currentTimeMillis() > start + timeout) {
+                break;
+            }
+            try {
+                Thread.sleep(250);
+            } catch (InterruptedException e) {
+                // ignore
+            }
+        }
+        return false;
+    }
+}

hoodie-common/src/test/java/com/uber/hoodie/common/table/log/avro/AvroLogAppenderTest.java

@@ -0,0 +1,315 @@
+/*
+ *  Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.uber.hoodie.common.table.log.avro;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.uber.hoodie.common.minicluster.MiniClusterUtil;
+import com.uber.hoodie.common.table.log.HoodieLogAppendConfig;
+import com.uber.hoodie.common.table.log.HoodieLogFile;
+import com.uber.hoodie.common.util.FSUtils;
+import com.uber.hoodie.common.util.SchemaTestUtil;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.commons.collections.IteratorUtils;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class AvroLogAppenderTest {
+    private FileSystem fs;
+    private Path partitionPath;
+
+    @BeforeClass
+    public static void setUpClass() throws IOException, InterruptedException {
+        // Append is not supported in LocalFileSystem. HDFS needs to be setup.
+        MiniClusterUtil.setUp();
+    }
+
+    @AfterClass
+    public static void tearDownClass() {
+        MiniClusterUtil.shutdown();
+    }
+
+    @Before
+    public void setUp() throws IOException, InterruptedException {
+        this.fs = MiniClusterUtil.fileSystem;
+        TemporaryFolder folder = new TemporaryFolder();
+        folder.create();
+        assertTrue(fs.mkdirs(new Path(folder.getRoot().getPath())));
+        this.partitionPath = new Path(folder.getRoot().getPath());
+    }
+
+    @After
+    public void tearDown() throws IOException {
+        fs.delete(partitionPath, true);
+    }
+
+    @Test
+    public void testBasicAppend() throws IOException, URISyntaxException, InterruptedException {
+        HoodieLogAppendConfig logConfig =
+            HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+                .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+                .withSchema(SchemaTestUtil.getSimpleSchema()).withFs(fs).build();
+        RollingAvroLogAppender logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(0, 100));
+        long size1 = logAppender.getCurrentSize();
+        assertTrue("", size1 > 0);
+        assertEquals("", size1, fs.getFileStatus(logConfig.getLogFile().getPath()).getLen());
+        logAppender.close();
+
+        // Close and Open again and append 100 more records
+        logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(100, 100));
+        long size2 = logAppender.getCurrentSize();
+        assertTrue("", size2 > size1);
+        assertEquals("", size2, fs.getFileStatus(logConfig.getLogFile().getPath()).getLen());
+        logAppender.close();
+
+        // Close and Open again and append 100 more records
+        logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(200, 100));
+        long size3 = logAppender.getCurrentSize();
+        assertTrue("", size3 > size2);
+        assertEquals("", size3, fs.getFileStatus(logConfig.getLogFile().getPath()).getLen());
+        logAppender.close();
+        // Cannot get the current size after closing the log
+        try {
+            logAppender.getCurrentSize();
+            fail("getCurrentSize should fail after the logAppender is closed");
+        } catch (IllegalStateException e) {
+            // pass
+        }
+    }
+
+    @Test
+    public void testLeaseRecovery() throws IOException, URISyntaxException, InterruptedException {
+        HoodieLogAppendConfig logConfig =
+            HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+                .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+                .withSchema(SchemaTestUtil.getSimpleSchema()).withFs(fs).build();
+        RollingAvroLogAppender logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(0, 100));
+        // do not close this log appender
+        // logAppender.close();
+
+        // Try opening again and append 100 more records
+        logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(100, 100));
+        assertEquals("", logAppender.getCurrentSize(),
+            fs.getFileStatus(logConfig.getLogFile().getPath()).getLen());
+        logAppender.close();
+    }
+
+    @Test
+    public void testAppendOnCorruptedBlock()
+        throws IOException, URISyntaxException, InterruptedException {
+        HoodieLogAppendConfig logConfig =
+            HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+                .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+                .withSchema(SchemaTestUtil.getSimpleSchema()).withFs(fs).build();
+        RollingAvroLogAppender logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(0, 100));
+        logAppender.close();
+
+        // Append some arbit byte[] to thee end of the log (mimics a partially written commit)
+        assertTrue(fs.exists(logConfig.getLogFile().getPath()));
+        fs = FileSystem.get(fs.getConf());
+        FSDataOutputStream outputStream =
+            fs.append(logConfig.getLogFile().getPath(), logConfig.getBufferSize());
+        outputStream.write("something-random".getBytes());
+        outputStream.flush();
+        outputStream.close();
+
+        logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(100, 100));
+        logAppender.close();
+    }
+
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testBasicWriteAndRead()
+        throws IOException, URISyntaxException, InterruptedException {
+        HoodieLogAppendConfig logConfig =
+            HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+                .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+                .withSchema(SchemaTestUtil.getSimpleSchema()).withFs(fs).build();
+        RollingAvroLogAppender logAppender = new RollingAvroLogAppender(logConfig);
+        long size1 = logAppender.getCurrentSize();
+
+        List<IndexedRecord> inputRecords = SchemaTestUtil.generateTestRecords(0, 100);
+        logAppender.append(inputRecords);
+        logAppender.close();
+
+        AvroLogReader logReader =
+            new AvroLogReader(logConfig.getLogFile(), fs, logConfig.getSchema());
+        List<GenericRecord> result = IteratorUtils.toList(logReader.readBlock(size1));
+        assertEquals("Random access should return 100 records", 100, result.size());
+        assertEquals("both lists should be the same. (ordering guaranteed)", inputRecords, result);
+    }
+
+    @SuppressWarnings("unchecked")
+    @Test
+    public void testBasicAppendAndRead()
+        throws IOException, URISyntaxException, InterruptedException {
+        HoodieLogAppendConfig logConfig =
+            HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+                .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+                .withSchema(SchemaTestUtil.getSimpleSchema()).withFs(fs).build();
+        RollingAvroLogAppender logAppender = new RollingAvroLogAppender(logConfig);
+        logAppender.append(SchemaTestUtil.generateTestRecords(0, 100));
+        long size1 = logAppender.getCurrentSize();
+        logAppender.close();
+
+        // Close and Open again and append 100 more records
+        logAppender = new RollingAvroLogAppender(logConfig);
+        List<IndexedRecord> secondBatchInput = SchemaTestUtil.generateTestRecords(100, 100);
+        logAppender.append(secondBatchInput);
+        long size2 = logAppender.getCurrentSize();
+        logAppender.close();
+
+        // Close and Open again and append 100 more records
+        logAppender = new RollingAvroLogAppender(logConfig);
+        List<IndexedRecord> lastBatchInput = SchemaTestUtil.generateTestRecords(200, 100);
+        logAppender.append(lastBatchInput);
+        long size3 = logAppender.getCurrentSize();
+        logAppender.close();
+
+        AvroLogReader logReader =
+            new AvroLogReader(logConfig.getLogFile(), fs, logConfig.getSchema());
+
+        // Try to grab the middle block here
+        List<GenericRecord> secondBatch = IteratorUtils.toList(logReader.readBlock(size1));
+        assertEquals("Stream collect should return 100 records", 100, secondBatch.size());
+        assertEquals("Collected list should match the input list (ordering guaranteed)",
+            secondBatchInput, secondBatch);
+
+        // Try to grab the middle block here
+        List<GenericRecord> lastBatch = IteratorUtils.toList(logReader.readBlock(size2));
+        assertEquals("Stream collect should return 100 records", 100, secondBatch.size());
+        assertEquals("Collected list should match the input list (ordering guaranteed)",
+            lastBatchInput, lastBatch);
+
+        List<GenericRecord> imaginaryBatch = IteratorUtils.toList(logReader.readBlock(size3));
+        assertEquals("Stream collect should return 0 records", 0, imaginaryBatch.size());
+    }
+
+    @Test
+    public void testAppendAndReadOnCorruptedLog()
+        throws IOException, URISyntaxException, InterruptedException {
+        HoodieLogAppendConfig logConfig =
+            HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+                .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+                .withSchema(SchemaTestUtil.getSimpleSchema()).withFs(fs).build();
+        RollingAvroLogAppender logAppender = new RollingAvroLogAppender(logConfig);
+        long size1 = logAppender.getCurrentSize();
+        logAppender.append(SchemaTestUtil.generateTestRecords(0, 100));
+        logAppender.close();
+
+        // Append some arbit byte[] to thee end of the log (mimics a partially written commit)
+        assertTrue(fs.exists(logConfig.getLogFile().getPath()));
+        fs = FileSystem.get(fs.getConf());
+        FSDataOutputStream outputStream =
+            fs.append(logConfig.getLogFile().getPath(), logConfig.getBufferSize());
+        outputStream.write("something-random".getBytes());
+        outputStream.flush();
+        outputStream.close();
+
+        logAppender = new RollingAvroLogAppender(logConfig);
+        long size2 = logAppender.getCurrentSize();
+        logAppender.append(SchemaTestUtil.generateTestRecords(100, 100));
+        logAppender.close();
+
+        AvroLogReader logReader =
+            new AvroLogReader(logConfig.getLogFile(), fs, logConfig.getSchema());
+
+        // Try to grab the middle block here
+        List<GenericRecord> secondBatch = IteratorUtils.toList(logReader.readBlock(size1));
+        assertEquals("Stream collect should return 100 records", 100, secondBatch.size());
+
+        // Try to grab the last block here
+        List<GenericRecord> lastBatch = IteratorUtils.toList(logReader.readBlock(size2));
+        assertEquals("Stream collect should return 100 records", 100, lastBatch.size());
+    }
+
+    @Test
+    public void testCompositeAvroLogReader()
+        throws IOException, URISyntaxException, InterruptedException {
+        // Set a small threshold so that every block is a new version
+        HoodieLogAppendConfig logConfig =
+            HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+                .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+                .withSchema(SchemaTestUtil.getSimpleSchema()).withSizeThreshold(500).withFs(fs)
+                .build();
+
+        RollingAvroLogAppender logAppender = new RollingAvroLogAppender(logConfig);
+        long size1 = logAppender.getCurrentSize();
+        List<IndexedRecord> input1 = SchemaTestUtil.generateTestRecords(0, 100);
+        logAppender.append(input1);
+        logAppender.close();
+
+        // Need to rebuild config to set the latest version as path
+        logConfig = HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+            .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+            .withSchema(SchemaTestUtil.getSimpleSchema()).withSizeThreshold(500).withFs(fs).build();
+        logAppender = new RollingAvroLogAppender(logConfig);
+        long size2 = logAppender.getCurrentSize();
+        List<IndexedRecord> input2 = SchemaTestUtil.generateTestRecords(100, 100);
+        logAppender.append(input2);
+        logAppender.close();
+
+        logConfig = HoodieLogAppendConfig.newBuilder().onPartitionPath(partitionPath)
+            .withLogFileExtension(HoodieLogFile.DELTA_EXTENSION).withFileId("test-fileid1")
+            .withSchema(SchemaTestUtil.getSimpleSchema()).withSizeThreshold(500).withFs(fs).build();
+        List<HoodieLogFile> allLogFiles = FSUtils
+            .getAllLogFiles(fs, partitionPath, logConfig.getLogFile().getFileId(),
+                HoodieLogFile.DELTA_EXTENSION).collect(Collectors.toList());
+        assertEquals("", 2, allLogFiles.size());
+
+        SortedMap<Integer, List<Long>> offsets = Maps.newTreeMap();
+        offsets.put(1, Lists.newArrayList(size1));
+        offsets.put(2, Lists.newArrayList(size2));
+        CompositeAvroLogReader reader =
+            new CompositeAvroLogReader(partitionPath, logConfig.getLogFile().getFileId(), fs,
+                logConfig.getSchema(), HoodieLogFile.DELTA_EXTENSION);
+        Iterator<GenericRecord> results = reader.readBlocks(offsets);
+        List<GenericRecord> totalBatch = IteratorUtils.toList(results);
+        assertEquals("Stream collect should return all 200 records", 200, totalBatch.size());
+        input1.addAll(input2);
+        assertEquals("CompositeAvroLogReader should return 200 records from 2 versions", input1,
+            totalBatch);
+    }
+}

hoodie-common/src/test/java/com/uber/hoodie/common/util/SchemaTestUtil.java

@@ -0,0 +1,62 @@
+/*
+ *  Copyright (c) 2016 Uber Technologies, Inc. (hoodie-dev-group@uber.com)
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *           http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package com.uber.hoodie.common.util;
+
+import com.uber.hoodie.exception.HoodieIOException;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.IndexedRecord;
+import org.apache.avro.io.DecoderFactory;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class SchemaTestUtil {
+    public static Schema getSimpleSchema() throws IOException {
+        return new Schema.Parser()
+            .parse(SchemaTestUtil.class.getResourceAsStream("/simple-test.avro"));
+    }
+
+    public static List<IndexedRecord> generateTestRecords(int from, int limit)
+        throws IOException, URISyntaxException {
+        return toRecords(getSimpleSchema(), getSimpleSchema(), from, limit);
+    }
+
+    private static List<IndexedRecord> toRecords(Schema writerSchema, Schema readerSchema, int from,
+        int limit) throws IOException, URISyntaxException {
+        GenericDatumReader<IndexedRecord> reader =
+            new GenericDatumReader<>(writerSchema, readerSchema);
+        try (Stream<String> stream = Files
+            .lines(Paths.get(SchemaTestUtil.class.getResource("/sample.data").toURI()))) {
+            return stream.skip(from).limit(limit).map(s -> {
+                try {
+                    return reader.read(null, DecoderFactory.get().jsonDecoder(readerSchema, s));
+                } catch (IOException e) {
+                    throw new HoodieIOException("Could not read data from simple_data.json", e);
+                }
+            }).collect(Collectors.toList());
+        } catch (IOException e) {
+            throw new HoodieIOException("Could not read data from simple_data.json", e);
+        }
+    }
+
+}

hoodie-common/src/test/resources/simple-test.avro

@@ -0,0 +1,10 @@
+{
+"namespace": "example.avro",
+ "type": "record",
+ "name": "User",
+ "fields": [
+     {"name": "name", "type": "string"},
+     {"name": "favorite_number",  "type": "long"},
+     {"name": "favorite_color", "type": "string"}
+ ]
+}