[HUDI-1373] Add Support for OpenJ9 JVM (#2231)

* add supoort for OpenJ9 VM
* add 32bit openJ9
* Pulled the memory layout specs into their own classes.

hudi-common/src/main/java/org/apache/hudi/common/util/ObjectSizeCalculator.java

@@ -18,6 +18,14 @@
 
 package org.apache.hudi.common.util;
 
+import org.apache.hudi.common.util.jvm.MemoryLayoutSpecification;
+import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification32bit;
+import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification64bit;
+import org.apache.hudi.common.util.jvm.HotSpotMemoryLayoutSpecification64bitCompressed;
+import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification32bit;
+import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification64bit;
+import org.apache.hudi.common.util.jvm.OpenJ9MemoryLayoutSpecification64bitCompressed;
+
 import java.lang.management.ManagementFactory;
 import java.lang.management.MemoryPoolMXBean;
 import java.lang.reflect.Array;
@@ -35,7 +43,7 @@ import java.util.Objects;
 import java.util.Set;
 
 /**
- * Contains utility methods for calculating the memory usage of objects. It only works on the HotSpot JVM, and infers
+ * Contains utility methods for calculating the memory usage of objects. It only works on the HotSpot and OpenJ9 JVMs, and infers
  * the actual memory layout (32 bit vs. 64 bit word size, compressed object pointers vs. uncompressed) from best
  * available indicators. It can reliably detect a 32 bit vs. 64 bit JVM. It can only make an educated guess at whether
  * compressed OOPs are used, though; specifically, it knows what the JVM's default choice of OOP compression would be
@@ -46,48 +54,6 @@ import java.util.Set;
  * @author Attila Szegedi
  */
 public class ObjectSizeCalculator {
-
-  /**
-   * Describes constant memory overheads for various constructs in a JVM implementation.
-   */
-  public interface MemoryLayoutSpecification {
-
-    /**
-     * Returns the fixed overhead of an array of any type or length in this JVM.
-     *
-     * @return the fixed overhead of an array.
-     */
-    int getArrayHeaderSize();
-
-    /**
-     * Returns the fixed overhead of for any {@link Object} subclass in this JVM.
-     *
-     * @return the fixed overhead of any object.
-     */
-    int getObjectHeaderSize();
-
-    /**
-     * Returns the quantum field size for a field owned by an object in this JVM.
-     *
-     * @return the quantum field size for an object.
-     */
-    int getObjectPadding();
-
-    /**
-     * Returns the fixed size of an object reference in this JVM.
-     *
-     * @return the size of all object references.
-     */
-    int getReferenceSize();
-
-    /**
-     * Returns the quantum field size for a field owned by one of an object's ancestor superclasses in this JVM.
-     *
-     * @return the quantum field size for a superclass field.
-     */
-    int getSuperclassFieldPadding();
-  }
-
   private static class CurrentLayout {
 
     private static final MemoryLayoutSpecification SPEC = getEffectiveMemoryLayoutSpecification();
@@ -328,109 +294,59 @@ public class ObjectSizeCalculator {
   static MemoryLayoutSpecification getEffectiveMemoryLayoutSpecification() {
     final String vmName = System.getProperty("java.vm.name");
     if (vmName == null || !(vmName.startsWith("Java HotSpot(TM) ") || vmName.startsWith("OpenJDK")
-        || vmName.startsWith("TwitterJDK"))) {
-      throw new UnsupportedOperationException("ObjectSizeCalculator only supported on HotSpot VM");
-    }
-
-    final String dataModel = System.getProperty("sun.arch.data.model");
-    if ("32".equals(dataModel)) {
-      // Running with 32-bit data model
-      return new MemoryLayoutSpecification() {
-        @Override
-        public int getArrayHeaderSize() {
-          return 12;
-        }
-
-        @Override
-        public int getObjectHeaderSize() {
-          return 8;
-        }
-
-        @Override
-        public int getObjectPadding() {
-          return 8;
-        }
-
-        @Override
-        public int getReferenceSize() {
-          return 4;
-        }
-
-        @Override
-        public int getSuperclassFieldPadding() {
-          return 4;
-        }
-      };
-    } else if (!"64".equals(dataModel)) {
-      throw new UnsupportedOperationException(
-          "Unrecognized value '" + dataModel + "' of sun.arch.data.model system property");
+        || vmName.startsWith("TwitterJDK") || vmName.startsWith("Eclipse OpenJ9"))) {
+      throw new UnsupportedOperationException("ObjectSizeCalculator only supported on HotSpot or Eclipse OpenJ9 VMs");
     }
 
     final String strVmVersion = System.getProperty("java.vm.version");
-    final int vmVersion = Integer.parseInt(strVmVersion.substring(0, strVmVersion.indexOf('.')));
-    if (vmVersion >= 17) {
+    // Support for OpenJ9 JVM
+    if (strVmVersion.startsWith("openj9")) {
+      final String dataModel = System.getProperty("sun.arch.data.model");
+      if ("32".equals(dataModel)) {
+        // Running with 32-bit data model
+        return new OpenJ9MemoryLayoutSpecification32bit();
+      } else if (!"64".equals(dataModel)) {
+        throw new UnsupportedOperationException(
+                "Unrecognized value '" + dataModel + "' of sun.arch.data.model system property");
+      }
+
       long maxMemory = 0;
       for (MemoryPoolMXBean mp : ManagementFactory.getMemoryPoolMXBeans()) {
         maxMemory += mp.getUsage().getMax();
       }
-      if (maxMemory < 30L * 1024 * 1024 * 1024) {
-        // HotSpot 17.0 and above use compressed OOPs below 30GB of RAM total
-        // for all memory pools (yes, including code cache).
-        return new MemoryLayoutSpecification() {
-          @Override
-          public int getArrayHeaderSize() {
-            return 16;
-          }
-
-          @Override
-          public int getObjectHeaderSize() {
-            return 12;
-          }
-
-          @Override
-          public int getObjectPadding() {
-            return 8;
-          }
-
-          @Override
-          public int getReferenceSize() {
-            return 4;
-          }
-
-          @Override
-          public int getSuperclassFieldPadding() {
-            return 4;
-          }
-        };
+      if (maxMemory < 57L * 1024 * 1024 * 1024) {
+        // OpenJ9 use compressed references below 57GB of RAM total
+        return new OpenJ9MemoryLayoutSpecification64bitCompressed();
+      } else {
+        // it's a 64-bit uncompressed references object model
+        return new OpenJ9MemoryLayoutSpecification64bit();
       }
+    } else {
+      // Support for HotSpot JVM
+      final String dataModel = System.getProperty("sun.arch.data.model");
+      if ("32".equals(dataModel)) {
+        // Running with 32-bit data model
+        return new HotSpotMemoryLayoutSpecification32bit();
+      } else if (!"64".equals(dataModel)) {
+        throw new UnsupportedOperationException(
+                "Unrecognized value '" + dataModel + "' of sun.arch.data.model system property");
+      }
+
+      final int vmVersion = Integer.parseInt(strVmVersion.substring(0, strVmVersion.indexOf('.')));
+      if (vmVersion >= 17) {
+        long maxMemory = 0;
+        for (MemoryPoolMXBean mp : ManagementFactory.getMemoryPoolMXBeans()) {
+          maxMemory += mp.getUsage().getMax();
+        }
+        if (maxMemory < 30L * 1024 * 1024 * 1024) {
+          // HotSpot 17.0 and above use compressed OOPs below 30GB of RAM total
+          // for all memory pools (yes, including code cache).
+          return new HotSpotMemoryLayoutSpecification64bitCompressed();
+        }
+      }
+
+      // In other cases, it's a 64-bit uncompressed OOPs object model
+      return new HotSpotMemoryLayoutSpecification64bit();
     }
-
-    // In other cases, it's a 64-bit uncompressed OOPs object model
-    return new MemoryLayoutSpecification() {
-      @Override
-      public int getArrayHeaderSize() {
-        return 24;
-      }
-
-      @Override
-      public int getObjectHeaderSize() {
-        return 16;
-      }
-
-      @Override
-      public int getObjectPadding() {
-        return 8;
-      }
-
-      @Override
-      public int getReferenceSize() {
-        return 8;
-      }
-
-      @Override
-      public int getSuperclassFieldPadding() {
-        return 8;
-      }
-    };
   }
 }

hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification32bit.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class HotSpotMemoryLayoutSpecification32bit implements MemoryLayoutSpecification {
+  @Override
+  public int getArrayHeaderSize() {
+    return 12;
+  }
+
+  @Override
+  public int getObjectHeaderSize() {
+    return 8;
+  }
+
+  @Override
+  public int getObjectPadding() {
+    return 8;
+  }
+
+  @Override
+  public int getReferenceSize() {
+    return 4;
+  }
+
+  @Override
+  public int getSuperclassFieldPadding() {
+    return 4;
+  }
+}

hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bit.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class HotSpotMemoryLayoutSpecification64bit implements MemoryLayoutSpecification {
+  @Override
+  public int getArrayHeaderSize() {
+    return 24;
+  }
+
+  @Override
+  public int getObjectHeaderSize() {
+    return 16;
+  }
+
+  @Override
+  public int getObjectPadding() {
+    return 8;
+  }
+
+  @Override
+  public int getReferenceSize() {
+    return 8;
+  }
+
+  @Override
+  public int getSuperclassFieldPadding() {
+    return 8;
+  }
+}

hudi-common/src/main/java/org/apache/hudi/common/util/jvm/HotSpotMemoryLayoutSpecification64bitCompressed.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class HotSpotMemoryLayoutSpecification64bitCompressed implements MemoryLayoutSpecification {
+  @Override
+  public int getArrayHeaderSize() {
+    return 16;
+  }
+
+  @Override
+  public int getObjectHeaderSize() {
+    return 12;
+  }
+
+  @Override
+  public int getObjectPadding() {
+    return 8;
+  }
+
+  @Override
+  public int getReferenceSize() {
+    return 4;
+  }
+
+  @Override
+  public int getSuperclassFieldPadding() {
+    return 4;
+  }
+}

hudi-common/src/main/java/org/apache/hudi/common/util/jvm/MemoryLayoutSpecification.java

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+/**
+ * Describes constant memory overheads for various constructs in a JVM implementation.
+ */
+public interface MemoryLayoutSpecification {
+
+  /**
+   * Returns the fixed overhead of an array of any type or length in this JVM.
+   *
+   * @return the fixed overhead of an array.
+   */
+  int getArrayHeaderSize();
+
+  /**
+   * Returns the fixed overhead of for any {@link Object} subclass in this JVM.
+   *
+   * @return the fixed overhead of any object.
+   */
+  int getObjectHeaderSize();
+
+  /**
+   * Returns the quantum field size for a field owned by an object in this JVM.
+   *
+   * @return the quantum field size for an object.
+   */
+  int getObjectPadding();
+
+  /**
+   * Returns the fixed size of an object reference in this JVM.
+   *
+   * @return the size of all object references.
+   */
+  int getReferenceSize();
+
+  /**
+   * Returns the quantum field size for a field owned by one of an object's ancestor superclasses in this JVM.
+   *
+   * @return the quantum field size for a superclass field.
+   */
+  int getSuperclassFieldPadding();
+}

hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification32bit.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class OpenJ9MemoryLayoutSpecification32bit implements MemoryLayoutSpecification {
+  @Override
+  public int getArrayHeaderSize() {
+    return 16;
+  }
+
+  @Override
+  public int getObjectHeaderSize() {
+    return 4;
+  }
+
+  @Override
+  public int getObjectPadding() {
+    return 4;
+  }
+
+  @Override
+  public int getReferenceSize() {
+    return 4;
+  }
+
+  @Override
+  public int getSuperclassFieldPadding() {
+    return 4;
+  }
+}

hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bit.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class OpenJ9MemoryLayoutSpecification64bit implements MemoryLayoutSpecification {
+  @Override
+  public int getArrayHeaderSize() {
+    return 16;
+  }
+
+  @Override
+  public int getObjectHeaderSize() {
+    return 16;
+  }
+
+  @Override
+  public int getObjectPadding() {
+    return 8;
+  }
+
+  @Override
+  public int getReferenceSize() {
+    return 8;
+  }
+
+  @Override
+  public int getSuperclassFieldPadding() {
+    return 8;
+  }
+}

hudi-common/src/main/java/org/apache/hudi/common/util/jvm/OpenJ9MemoryLayoutSpecification64bitCompressed.java

@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.common.util.jvm;
+
+public class OpenJ9MemoryLayoutSpecification64bitCompressed implements MemoryLayoutSpecification {
+  @Override
+  public int getArrayHeaderSize() {
+    return 16;
+  }
+
+  @Override
+  public int getObjectHeaderSize() {
+    return 4;
+  }
+
+  @Override
+  public int getObjectPadding() {
+    return 4;
+  }
+
+  @Override
+  public int getReferenceSize() {
+    return 4;
+  }
+
+  @Override
+  public int getSuperclassFieldPadding() {
+    return 4;
+  }
+}