[HUDI-4145] Archives the metadata file in HoodieInstant.State sequence (#5669)

hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/HoodieTimelineArchiver.java

@@ -72,7 +72,6 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
@@ -507,10 +506,16 @@ public class HoodieTimelineArchiver<T extends HoodieAvroPayload, I, K, O> {
       List<HoodieInstant> instantsToStream = groupByTsAction.get(Pair.of(hoodieInstant.getTimestamp(),
                 HoodieInstant.getComparableAction(hoodieInstant.getAction())));
       if (instantsToStream != null) {
-        return instantsToStream.stream();
+        // sorts the instants in natural order to make sure the metadata files be removed
+        // in HoodieInstant.State sequence: requested -> inflight -> completed,
+        // this is important because when a COMPLETED metadata file is removed first,
+        // other monitors on the timeline(such as the compaction or clustering services) would
+        // mistakenly recognize the pending file as a pending operation,
+        // then all kinds of weird bugs occur.
+        return instantsToStream.stream().sorted();
       } else {
         // if a concurrent writer archived the instant
-        return Collections.EMPTY_LIST.stream();
+        return Stream.empty();
       }
     });
   }