From acae6586f3031a5969cdc51fcfa8fd0764f41f8e Mon Sep 17 00:00:00 2001
From: Jian Xu <jian@uber.com>
Date: Thu, 21 Dec 2017 09:58:51 -0800
Subject: [PATCH] Incorporating code review feedback for finalizeWrite for COW
 #3

---
 .../src/main/java/com/uber/hoodie/HoodieWriteClient.java | 6 ------
 .../java/com/uber/hoodie/config/HoodieWriteConfig.java   | 2 +-
 .../com/uber/hoodie/table/HoodieCopyOnWriteTable.java    | 9 +++++++--
 .../src/main/java/com/uber/hoodie/table/HoodieTable.java | 4 ----
 4 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java b/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
index 230a60324..df80ba587 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
@@ -440,9 +440,6 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> implements Seriali
             );
         }
 
-        // Clean temp files
-        table.cleanTemporaryDataFiles(jsc);
-
         // add in extra metadata
         if (extraMetadata.isPresent()) {
             extraMetadata.get().forEach((k, v) -> metadata.addMetadata(k, v));
@@ -698,9 +695,6 @@ public class HoodieWriteClient<T extends HoodieRecordPayload> implements Seriali
             }
         });
 
-        // clean data files in temporary folder
-        table.cleanTemporaryDataFiles(jsc);
-
         try {
             if (commitTimeline.empty() && inflightTimeline.empty()) {
                 // nothing to rollback
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
index 0677f4500..082ececdb 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/config/HoodieWriteConfig.java
@@ -61,7 +61,7 @@ public class HoodieWriteConfig extends DefaultHoodieConfig {
   private static final String HOODIE_COPYONWRITE_USE_TEMP_FOLDER = "hoodie.copyonwrite.use.temp.folder";
   private static final String DEFAULT_HOODIE_COPYONWRITE_USE_TEMP_FOLDER = "false";
   private static final String FINALIZE_WRITE_PARALLELISM = "hoodie.finalize.write.parallelism";
-  private static final String DEFAULT_FINALIZE_WRITE_PARALLELISM = "5";
+  private static final String DEFAULT_FINALIZE_WRITE_PARALLELISM = DEFAULT_PARALLELISM;
 
   private HoodieWriteConfig(Properties props) {
     super(props);
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
index 511c867fa..4bcd1d292 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieCopyOnWriteTable.java
@@ -569,6 +569,9 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
               .withDeletedFileResults(results).build();
         }).collect();
 
+    // clean temporary data files
+    cleanTemporaryDataFiles(jsc);
+
     // Remove the rolled back inflight commits
     commits.stream().map(s -> new HoodieInstant(true, actionType, s))
         .forEach(activeTimeline::deleteInflight);
@@ -609,11 +612,13 @@ public class HoodieCopyOnWriteTable<T extends HoodieRecordPayload> extends Hoodi
           return new Tuple2<>(writeStat.getPath(), true);
         }).collect();
 
+    // clean temporary data files
+    cleanTemporaryDataFiles(jsc);
+
     return Optional.of(results.size());
   }
 
-  @Override
-  public void cleanTemporaryDataFiles(JavaSparkContext jsc) {
+  private void cleanTemporaryDataFiles(JavaSparkContext jsc) {
     if (!config.shouldUseTempFolderForCopyOnWrite()) {
       return;
     }
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java
index c395450bb..06eec8e45 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/table/HoodieTable.java
@@ -281,8 +281,4 @@ public abstract class HoodieTable<T extends HoodieRecordPayload> implements Seri
    */
   public abstract Optional<Integer> finalizeWrite(JavaSparkContext jsc, List<Tuple2<String, HoodieWriteStat>> writeStatuses);
 
-  /**
-   * Clean temporary data files after data files are finalized or commit is rolled back.
-   */
-  public abstract void cleanTemporaryDataFiles(JavaSparkContext jsc);
 }