1
0

[HUDI-2077] Fix TestHoodieDeltaStreamerWithMultiWriter (#3849)

Remove the logic of using deltastreamer to prep test table. Use fixture (compressed test table) instead.
This commit is contained in:
Raymond Xu
2021-10-24 21:14:39 -07:00
committed by GitHub
parent 91845e241d
commit d8560377c3
10 changed files with 178 additions and 65 deletions

View File

@@ -0,0 +1,81 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hudi.common.testutils;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
import java.nio.file.Path;
import java.util.Objects;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
public final class FixtureUtils {
public static Path prepareFixtureTable(URL fixtureResource, Path basePath) throws IOException {
File zippedFixtureTable = new File(fixtureResource.getFile());
try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zippedFixtureTable))) {
byte[] buffer = new byte[1024];
ZipEntry zipEntry = zis.getNextEntry();
Path tableBasePath = basePath.resolve(Objects.requireNonNull(zipEntry).getName()
.replaceAll(File.separator + "$", ""));
while (zipEntry != null) {
File newFile = newFile(basePath.toFile(), zipEntry);
if (zipEntry.isDirectory()) {
if (!newFile.isDirectory() && !newFile.mkdirs()) {
throw new IOException("Failed to create directory " + newFile);
}
} else {
// fix for Windows-created archives
File parent = newFile.getParentFile();
if (!parent.isDirectory() && !parent.mkdirs()) {
throw new IOException("Failed to create directory " + parent);
}
// write file content
try (FileOutputStream fos = new FileOutputStream(newFile)) {
int len;
while ((len = zis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
}
}
zipEntry = zis.getNextEntry();
}
zis.closeEntry();
return tableBasePath;
}
}
public static File newFile(File destinationDir, ZipEntry zipEntry) throws IOException {
File destFile = new File(destinationDir, zipEntry.getName());
String destDirPath = destinationDir.getCanonicalPath();
String destFilePath = destFile.getCanonicalPath();
if (!destFilePath.startsWith(destDirPath + File.separator)) {
throw new IOException("Entry is outside of the target dir: " + zipEntry.getName());
}
return destFile;
}
}

View File

@@ -160,6 +160,7 @@ public class HoodieTestDataGenerator {
this.existingKeysBySchema = new HashMap<>();
existingKeysBySchema.put(TRIP_EXAMPLE_SCHEMA, keyPartitionMap);
numKeysBySchema = new HashMap<>();
numKeysBySchema.put(TRIP_EXAMPLE_SCHEMA, keyPartitionMap.size());
}
/**
@@ -844,8 +845,8 @@ public class HoodieTestDataGenerator {
public static class KeyPartition implements Serializable {
HoodieKey key;
String partitionPath;
public HoodieKey key;
public String partitionPath;
}
public void close() {