[HUDI-2077] Fix TestHoodieDeltaStreamerWithMultiWriter (#3849)
Remove the logic of using deltastreamer to prep test table. Use fixture (compressed test table) instead.
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hudi.common.testutils;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.net.URL;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Objects;
|
||||
import java.util.zip.ZipEntry;
|
||||
import java.util.zip.ZipInputStream;
|
||||
|
||||
public final class FixtureUtils {
|
||||
|
||||
public static Path prepareFixtureTable(URL fixtureResource, Path basePath) throws IOException {
|
||||
File zippedFixtureTable = new File(fixtureResource.getFile());
|
||||
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zippedFixtureTable))) {
|
||||
byte[] buffer = new byte[1024];
|
||||
ZipEntry zipEntry = zis.getNextEntry();
|
||||
Path tableBasePath = basePath.resolve(Objects.requireNonNull(zipEntry).getName()
|
||||
.replaceAll(File.separator + "$", ""));
|
||||
while (zipEntry != null) {
|
||||
File newFile = newFile(basePath.toFile(), zipEntry);
|
||||
if (zipEntry.isDirectory()) {
|
||||
if (!newFile.isDirectory() && !newFile.mkdirs()) {
|
||||
throw new IOException("Failed to create directory " + newFile);
|
||||
}
|
||||
} else {
|
||||
// fix for Windows-created archives
|
||||
File parent = newFile.getParentFile();
|
||||
if (!parent.isDirectory() && !parent.mkdirs()) {
|
||||
throw new IOException("Failed to create directory " + parent);
|
||||
}
|
||||
|
||||
// write file content
|
||||
try (FileOutputStream fos = new FileOutputStream(newFile)) {
|
||||
int len;
|
||||
while ((len = zis.read(buffer)) > 0) {
|
||||
fos.write(buffer, 0, len);
|
||||
}
|
||||
}
|
||||
}
|
||||
zipEntry = zis.getNextEntry();
|
||||
}
|
||||
zis.closeEntry();
|
||||
return tableBasePath;
|
||||
}
|
||||
}
|
||||
|
||||
public static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException {
|
||||
File destFile = new File(destinationDir, zipEntry.getName());
|
||||
|
||||
String destDirPath = destinationDir.getCanonicalPath();
|
||||
String destFilePath = destFile.getCanonicalPath();
|
||||
|
||||
if (!destFilePath.startsWith(destDirPath + File.separator)) {
|
||||
throw new IOException("Entry is outside of the target dir: " + zipEntry.getName());
|
||||
}
|
||||
|
||||
return destFile;
|
||||
}
|
||||
}
|
||||
@@ -160,6 +160,7 @@ public class HoodieTestDataGenerator {
|
||||
this.existingKeysBySchema = new HashMap<>();
|
||||
existingKeysBySchema.put(TRIP_EXAMPLE_SCHEMA, keyPartitionMap);
|
||||
numKeysBySchema = new HashMap<>();
|
||||
numKeysBySchema.put(TRIP_EXAMPLE_SCHEMA, keyPartitionMap.size());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -844,8 +845,8 @@ public class HoodieTestDataGenerator {
|
||||
|
||||
public static class KeyPartition implements Serializable {
|
||||
|
||||
HoodieKey key;
|
||||
String partitionPath;
|
||||
public HoodieKey key;
|
||||
public String partitionPath;
|
||||
}
|
||||
|
||||
public void close() {
|
||||
|
||||
Reference in New Issue
Block a user