lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[MINOR] Use HoodieActiveTimeline.COMMIT_FORMATTER (#1874)

Browse Source
This commit is contained in:
Raymond Xu
2020-07-24 18:48:56 -07:00
committed by GitHub
parent 467d097dae
commit 0cb24e4a2d
3 changed files with 16 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
hudi-common/src/test/java/org/apache/hudi/common/fs/TestFSUtils.java
View File

@@ -37,7 +37,6 @@ import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Date; import java.util.Date;
@@ -46,6 +45,7 @@ import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.apache.hudi.common.table.timeline.HoodieActiveTimeline.COMMIT_FORMATTER;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -72,14 +72,14 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test @Test
public void testMakeDataFileName() { public void testMakeDataFileName() {
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String instantTime = COMMIT_FORMATTER.format(new Date());
String fileName = UUID.randomUUID().toString(); String fileName = UUID.randomUUID().toString();
assertEquals(FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName), fileName + "_" + TEST_WRITE_TOKEN + "_" + instantTime + ".parquet"); assertEquals(FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName), fileName + "_" + TEST_WRITE_TOKEN + "_" + instantTime + ".parquet");
} }
@Test @Test
public void testMaskFileName() { public void testMaskFileName() {
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String instantTime = COMMIT_FORMATTER.format(new Date());
int taskPartitionId = 2; int taskPartitionId = 2;
assertEquals(FSUtils.maskWithoutFileId(instantTime, taskPartitionId), "*_" + taskPartitionId + "_" + instantTime + ".parquet"); assertEquals(FSUtils.maskWithoutFileId(instantTime, taskPartitionId), "*_" + taskPartitionId + "_" + instantTime + ".parquet");
} }
@@ -144,7 +144,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test @Test
public void testGetCommitTime() { public void testGetCommitTime() {
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String instantTime = COMMIT_FORMATTER.format(new Date());
String fileName = UUID.randomUUID().toString(); String fileName = UUID.randomUUID().toString();
String fullFileName = FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName); String fullFileName = FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName);
assertEquals(instantTime, FSUtils.getCommitTime(fullFileName)); assertEquals(instantTime, FSUtils.getCommitTime(fullFileName));
@@ -152,7 +152,7 @@ public class TestFSUtils extends HoodieCommonTestHarness {
@Test @Test
public void testGetFileNameWithoutMeta() { public void testGetFileNameWithoutMeta() {
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String instantTime = COMMIT_FORMATTER.format(new Date());
String fileName = UUID.randomUUID().toString(); String fileName = UUID.randomUUID().toString();
String fullFileName = FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName); String fullFileName = FSUtils.makeDataFileName(instantTime, TEST_WRITE_TOKEN, fileName);
assertEquals(fileName, FSUtils.getFileId(fullFileName)); assertEquals(fileName, FSUtils.getFileId(fullFileName));

4
hudi-common/src/test/java/org/apache/hudi/common/model/TestHoodieWriteStat.java
View File

@@ -23,10 +23,10 @@ import org.apache.hudi.common.fs.FSUtils;
import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.UUID; import java.util.UUID;
import static org.apache.hudi.common.table.timeline.HoodieActiveTimeline.COMMIT_FORMATTER;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertNull;
@@ -37,7 +37,7 @@ public class TestHoodieWriteStat {
@Test @Test
public void testSetPaths() { public void testSetPaths() {
String instantTime = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); String instantTime = COMMIT_FORMATTER.format(new Date());
String basePathString = "/data/tables/some-hoodie-table"; String basePathString = "/data/tables/some-hoodie-table";
String partitionPathString = "2017/12/31"; String partitionPathString = "2017/12/31";
String fileName = UUID.randomUUID().toString(); String fileName = UUID.randomUUID().toString();

19
hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
View File

@@ -18,12 +18,6 @@
package org.apache.hudi.common.testutils; package org.apache.hudi.common.testutils;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hudi.avro.HoodieAvroUtils; import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.model.HoodieActionInstant; import org.apache.hudi.avro.model.HoodieActionInstant;
import org.apache.hudi.avro.model.HoodieCleanMetadata; import org.apache.hudi.avro.model.HoodieCleanMetadata;
@@ -47,7 +41,6 @@ import org.apache.hudi.common.table.log.HoodieLogFormat;
import org.apache.hudi.common.table.log.HoodieLogFormat.Writer; import org.apache.hudi.common.table.log.HoodieLogFormat.Writer;
import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock; import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock;
import org.apache.hudi.common.table.log.block.HoodieLogBlock; import org.apache.hudi.common.table.log.block.HoodieLogBlock;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant; import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstant.State; import org.apache.hudi.common.table.timeline.HoodieInstant.State;
import org.apache.hudi.common.table.timeline.HoodieTimeline; import org.apache.hudi.common.table.timeline.HoodieTimeline;
@@ -66,6 +59,12 @@ import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord; import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.IndexedRecord; import org.apache.avro.generic.IndexedRecord;
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.StringUtils;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
@@ -78,7 +77,6 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel; import java.nio.channels.FileChannel;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Calendar; import java.util.Calendar;
@@ -94,6 +92,7 @@ import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static org.apache.hudi.common.table.timeline.HoodieActiveTimeline.COMMIT_FORMATTER;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail; import static org.junit.jupiter.api.Assertions.fail;
@@ -160,7 +159,7 @@ public class HoodieTestUtils {
} }
public static String makeNewCommitTime() { public static String makeNewCommitTime() {
return new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()); return COMMIT_FORMATTER.format(new Date());
} }
public static void createCommitFiles(String basePath, String... instantTimes) throws IOException { public static void createCommitFiles(String basePath, String... instantTimes) throws IOException {
@@ -488,7 +487,7 @@ public class HoodieTestUtils {
cal.add(Calendar.SECOND, startSecsDelta); cal.add(Calendar.SECOND, startSecsDelta);
List<String> commits = new ArrayList<>(); List<String> commits = new ArrayList<>();
for (int i = 0; i < numTimestamps; i++) { for (int i = 0; i < numTimestamps; i++) {
commits.add(HoodieActiveTimeline.COMMIT_FORMATTER.format(cal.getTime())); commits.add(COMMIT_FORMATTER.format(cal.getTime()));
cal.add(Calendar.SECOND, 1); cal.add(Calendar.SECOND, 1);
} }
return commits; return commits;