[HUDI-2090] Ensure Disk Maps create a subfolder with appropriate prefixes and cleans them up on close (#3329)

* Add UUID to the folder name for External Spillable File System

* Fix to ensure that Disk maps folders do not interefere across users

* Fix test

* Fix test

* Rebase with latest mater and address comments

* Add Shutdown Hooks for the Disk Map

Co-authored-by: Rajesh Mahindra <rmahindra@Rajeshs-MacBook-Pro.local>

hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestBitCaskDiskMap.java

@@ -41,6 +41,7 @@ import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.URISyntaxException;
@@ -50,6 +51,7 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
@@ -89,6 +91,8 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
       Option<IndexedRecord> value = payload.getInsertValue(HoodieAvroUtils.addMetadataFields(getSimpleSchema()));
       assertEquals(originalRecord, value.get());
     }
+
+    verifyCleanup(records);
   }
 
   @ParameterizedTest
@@ -111,6 +115,8 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
       oRecords.add(rec);
       assert recordKeys.contains(rec.getRecordKey());
     }
+
+    verifyCleanup(records);
   }
 
   @ParameterizedTest
@@ -154,6 +160,7 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
         throw new UncheckedIOException(io);
       }
     }
+    verifyCleanup(records);
   }
 
   @Test
@@ -236,4 +243,11 @@ public class TestBitCaskDiskMap extends HoodieCommonTestHarness {
     System.out.println("Time taken :" + timeTaken);
     assertTrue(timeTaken < 100);
   }
+
+  private void verifyCleanup(BitCaskDiskMap<String, HoodieRecord> records) {
+    File basePathDir = new File(basePath);
+    assert Objects.requireNonNull(basePathDir.list()).length > 0;
+    records.close();
+    assertEquals(Objects.requireNonNull(basePathDir.list()).length, 0);
+  }
 }

hudi-common/src/test/java/org/apache/hudi/common/util/collection/TestExternalSpillableMap.java

@@ -342,6 +342,7 @@ public class TestExternalSpillableMap extends HoodieCommonTestHarness {
   public void testLargeInsertUpsert() {}
 
   private static Stream<Arguments> testArguments() {
+    // Arguments : 1. Disk Map Type 2. isCompressionEnabled for BitCaskMap
     return Stream.of(
         arguments(ExternalSpillableMap.DiskMapType.BITCASK, false),
         arguments(ExternalSpillableMap.DiskMapType.ROCKS_DB, false),