28dafa774e
- Add a new action called INDEX, whose state transition is described in the RFC. - Changes in timeline to support the new action. - Add an index planner in ScheduleIndexActionExecutor. - Add index plan executor in RunIndexActionExecutor. - Add 3 APIs in HoodieTableMetadataWriter; a) scheduleIndex: will generate an index plan based on latest completed instant, initialize file groups and add a requested INDEX instant, b) index: executes the index plan and also takes care of writes that happened after indexing was requested, c) dropIndex: will drop index by removing the given metadata partition. - Add 2 new table configs to serve as the source of truth for inflight and completed indexes. - Support upgrade/downgrade taking care of the newly added configs. - Add tool to trigger indexing in HoodieIndexer. - Handle corner cases related to partial failures. - Abort gracefully after deleting partition and instant. - Handle other actions in timeline to consider before catching up
264 lines
8.8 KiB
XML
264 lines
8.8 KiB
XML
<?xml version="1.0" encoding="UTF-8"?>
|
|
<!--
|
|
Licensed to the Apache Software Foundation (ASF) under one or more
|
|
contributor license agreements. See the NOTICE file distributed with
|
|
this work for additional information regarding copyright ownership.
|
|
The ASF licenses this file to You under the Apache License, Version 2.0
|
|
(the "License"); you may not use this file except in compliance with
|
|
the License. You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
-->
|
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
<parent>
|
|
<artifactId>hudi</artifactId>
|
|
<groupId>org.apache.hudi</groupId>
|
|
<version>0.11.0-SNAPSHOT</version>
|
|
</parent>
|
|
<modelVersion>4.0.0</modelVersion>
|
|
|
|
<artifactId>hudi-common</artifactId>
|
|
|
|
<properties>
|
|
<main.basedir>${project.parent.basedir}</main.basedir>
|
|
</properties>
|
|
|
|
<build>
|
|
<resources>
|
|
<resource>
|
|
<directory>src/main/resources</directory>
|
|
</resource>
|
|
</resources>
|
|
|
|
<plugins>
|
|
<plugin>
|
|
<groupId>org.apache.maven.plugins</groupId>
|
|
<artifactId>maven-jar-plugin</artifactId>
|
|
<version>${maven-jar-plugin.version}</version>
|
|
<executions>
|
|
<execution>
|
|
<goals>
|
|
<goal>test-jar</goal>
|
|
</goals>
|
|
<phase>test-compile</phase>
|
|
</execution>
|
|
</executions>
|
|
<configuration>
|
|
<skip>false</skip>
|
|
</configuration>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.rat</groupId>
|
|
<artifactId>apache-rat-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.jacoco</groupId>
|
|
<artifactId>jacoco-maven-plugin</artifactId>
|
|
</plugin>
|
|
<plugin>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro-maven-plugin</artifactId>
|
|
<configuration>
|
|
<imports>
|
|
<!-- import avro files -->
|
|
<import>${basedir}/src/main/avro/HoodieCommitMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieCompactionOperation.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieSavePointMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieCompactionMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieCleanPartitionMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieCleanMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieCleanerPlan.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieInstantInfo.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieRollbackMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieRestoreMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieReplaceCommitMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodiePath.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieFSPermission.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieFileStatus.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieBootstrapSourceFilePartitionInfo.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieBootstrapIndexInfo.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieBootstrapMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieSliceInfo.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieClusteringGroup.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieClusteringStrategy.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieClusteringPlan.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieRequestedReplaceMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieIndexPartitionInfo.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieIndexPlan.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieIndexCommitMetadata.avsc</import>
|
|
<import>${basedir}/src/main/avro/HoodieArchivedMetaEntry.avsc</import>
|
|
</imports>
|
|
</configuration>
|
|
</plugin>
|
|
</plugins>
|
|
</build>
|
|
|
|
<dependencies>
|
|
<!-- Fasterxml -->
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-annotations</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>com.fasterxml.jackson.core</groupId>
|
|
<artifactId>jackson-databind</artifactId>
|
|
</dependency>
|
|
|
|
<!-- Avro -->
|
|
<dependency>
|
|
<groupId>org.apache.avro</groupId>
|
|
<artifactId>avro</artifactId>
|
|
</dependency>
|
|
|
|
<!-- Parquet -->
|
|
<dependency>
|
|
<groupId>org.apache.parquet</groupId>
|
|
<artifactId>parquet-avro</artifactId>
|
|
</dependency>
|
|
|
|
<!-- ORC -->
|
|
<dependency>
|
|
<groupId>org.apache.orc</groupId>
|
|
<artifactId>orc-core</artifactId>
|
|
<version>${orc.version}</version>
|
|
<classifier>nohive</classifier>
|
|
</dependency>
|
|
|
|
<!-- Httpcomponents -->
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>fluent-hc</artifactId>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.httpcomponents</groupId>
|
|
<artifactId>httpclient</artifactId>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.rocksdb</groupId>
|
|
<artifactId>rocksdbjni</artifactId>
|
|
</dependency>
|
|
|
|
<!-- Hadoop -->
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-client</artifactId>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-common</artifactId>
|
|
<classifier>tests</classifier>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-hdfs</artifactId>
|
|
<scope>provided</scope>
|
|
</dependency>
|
|
<dependency>
|
|
<groupId>org.apache.hadoop</groupId>
|
|
<artifactId>hadoop-hdfs</artifactId>
|
|
<classifier>tests</classifier>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.jupiter</groupId>
|
|
<artifactId>junit-jupiter-api</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.jupiter</groupId>
|
|
<artifactId>junit-jupiter-engine</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.vintage</groupId>
|
|
<artifactId>junit-vintage-engine</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.junit.jupiter</groupId>
|
|
<artifactId>junit-jupiter-params</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.mockito</groupId>
|
|
<artifactId>mockito-junit-jupiter</artifactId>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.esotericsoftware</groupId>
|
|
<artifactId>kryo-shaded</artifactId>
|
|
<version>4.0.2</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>com.github.stefanbirkner</groupId>
|
|
<artifactId>system-rules</artifactId>
|
|
<version>1.17.2</version>
|
|
<scope>test</scope>
|
|
</dependency>
|
|
|
|
<!-- HBase -->
|
|
<dependency>
|
|
<groupId>org.apache.hbase</groupId>
|
|
<artifactId>hbase-client</artifactId>
|
|
<version>${hbase.version}</version>
|
|
</dependency>
|
|
|
|
<dependency>
|
|
<groupId>org.apache.hbase</groupId>
|
|
<artifactId>hbase-server</artifactId>
|
|
<version>${hbase.version}</version>
|
|
<!-- Unfortunately, HFile is packaged ONLY under hbase-server -->
|
|
<scope>compile</scope>
|
|
<exclusions>
|
|
<exclusion>
|
|
<groupId>javax.servlet</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.codehaus.jackson</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>org.mortbay.jetty</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
<exclusion>
|
|
<groupId>tomcat</groupId>
|
|
<artifactId>*</artifactId>
|
|
</exclusion>
|
|
</exclusions>
|
|
</dependency>
|
|
|
|
<!-- LZ4 Hash Utils -->
|
|
<dependency>
|
|
<groupId>org.lz4</groupId>
|
|
<artifactId>lz4-java</artifactId>
|
|
<version>1.8.0</version>
|
|
</dependency>
|
|
|
|
</dependencies>
|
|
</project>
|