[HUDI-2790] Fix the changelog mode of HoodieTableSource (#4029)
This commit is contained in:
@@ -100,4 +100,14 @@ public class OptionsResolver {
|
|||||||
public static boolean isPartitionedTable(Configuration conf) {
|
public static boolean isPartitionedTable(Configuration conf) {
|
||||||
return FilePathUtils.extractPartitionKeys(conf).length > 0;
|
return FilePathUtils.extractPartitionKeys(conf).length > 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns whether the source should emit changelog.
|
||||||
|
*
|
||||||
|
* @return true if the source is read as streaming with changelog mode enabled
|
||||||
|
*/
|
||||||
|
public static boolean emitChangelog(Configuration conf) {
|
||||||
|
return conf.getBoolean(FlinkOptions.READ_AS_STREAMING)
|
||||||
|
&& conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import org.apache.hudi.common.table.TableSchemaResolver;
|
|||||||
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
|
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
|
||||||
import org.apache.hudi.common.util.Option;
|
import org.apache.hudi.common.util.Option;
|
||||||
import org.apache.hudi.configuration.FlinkOptions;
|
import org.apache.hudi.configuration.FlinkOptions;
|
||||||
|
import org.apache.hudi.configuration.OptionsResolver;
|
||||||
import org.apache.hudi.exception.HoodieException;
|
import org.apache.hudi.exception.HoodieException;
|
||||||
import org.apache.hudi.hadoop.HoodieROTablePathFilter;
|
import org.apache.hudi.hadoop.HoodieROTablePathFilter;
|
||||||
import org.apache.hudi.source.FileIndex;
|
import org.apache.hudi.source.FileIndex;
|
||||||
@@ -196,11 +197,9 @@ public class HoodieTableSource implements
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ChangelogMode getChangelogMode() {
|
public ChangelogMode getChangelogMode() {
|
||||||
return conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED)
|
// when read as streaming and changelog mode is enabled, emit as FULL mode;
|
||||||
? ChangelogModes.FULL
|
// when all the changes are compacted or read as batch, emit as INSERT mode.
|
||||||
// when all the changes are persisted or read as batch,
|
return OptionsResolver.emitChangelog(conf) ? ChangelogModes.FULL : ChangelogMode.insertOnly();
|
||||||
// use INSERT mode.
|
|
||||||
: ChangelogMode.insertOnly();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import org.apache.hudi.common.table.log.InstantRange;
|
|||||||
import org.apache.hudi.common.util.ClosableIterator;
|
import org.apache.hudi.common.util.ClosableIterator;
|
||||||
import org.apache.hudi.common.util.Option;
|
import org.apache.hudi.common.util.Option;
|
||||||
import org.apache.hudi.configuration.FlinkOptions;
|
import org.apache.hudi.configuration.FlinkOptions;
|
||||||
|
import org.apache.hudi.configuration.OptionsResolver;
|
||||||
import org.apache.hudi.exception.HoodieException;
|
import org.apache.hudi.exception.HoodieException;
|
||||||
import org.apache.hudi.keygen.KeyGenUtils;
|
import org.apache.hudi.keygen.KeyGenUtils;
|
||||||
import org.apache.hudi.table.format.FilePathUtils;
|
import org.apache.hudi.table.format.FilePathUtils;
|
||||||
@@ -179,8 +180,7 @@ public class MergeOnReadInputFormat
|
|||||||
}
|
}
|
||||||
} else if (!split.getBasePath().isPresent()) {
|
} else if (!split.getBasePath().isPresent()) {
|
||||||
// log files only
|
// log files only
|
||||||
if (conf.getBoolean(FlinkOptions.READ_AS_STREAMING)
|
if (OptionsResolver.emitChangelog(conf)) {
|
||||||
&& conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED)) {
|
|
||||||
this.iterator = new LogFileOnlyIterator(getUnMergedLogFileIterator(split));
|
this.iterator = new LogFileOnlyIterator(getUnMergedLogFileIterator(split));
|
||||||
} else {
|
} else {
|
||||||
this.iterator = new LogFileOnlyIterator(getLogFileIterator(split));
|
this.iterator = new LogFileOnlyIterator(getLogFileIterator(split));
|
||||||
|
|||||||
Reference in New Issue
Block a user