[HUDI-3837] Fix license and rat check settings (#5273)

- add missing licenses
- fix CI setting to run rat plugin
- fix deploy script to include integ test modules

.github/workflows/bot.yml

@@ -76,7 +76,7 @@ jobs:
           SPARK_VERSION: ${{ matrix.sparkVersion }}
           FLINK_PROFILE: ${{ matrix.flinkProfile }}
         run:
-          mvn -T 2.5C clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+          mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -Pintegration-tests -DskipTests=true -B -V
       - name: Quickstart Test
         env:
           SCALA_PROFILE: ${{ matrix.scalaProfile }}

docker/README.md

@@ -92,7 +92,7 @@ docker push apachehudi/hudi-hadoop_2.8.4-trinobase_368
 You can also easily push the image to the Docker Hub using Docker Desktop app: go to `Images`, search for the image by
 the name, and then click on the three dots and `Push to Hub`.
 
-![Push to Docker Hub](push_to_docker_hub.png)
+![Push to Docker Hub](images/push_to_docker_hub.png)
 
 Note that you need to ask for permission to upload the Hudi Docker Demo images to the repositories.
 

hudi-examples/hudi-examples-dbt/README.md

@@ -1,3 +1,19 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
 ## Testing dbt project: `hudi_examples_dbt`
 
 This dbt project transforms demonstrates hudi integration with dbt, it has a few models to demonstrate the different ways in which you can create hudi datasets using dbt.

hudi-spark-datasource/hudi-spark2-common/pom.xml

@@ -1,4 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/util/SyncUtilHelpers.java

@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 package org.apache.hudi.sync.common.util;
 
 import org.apache.hudi.common.config.TypedProperties;

pom.xml

@@ -140,7 +140,7 @@
     <scala12.version>2.12.10</scala12.version>
     <scala.version>${scala11.version}</scala.version>
     <scala.binary.version>2.11</scala.binary.version>
-    <apache-rat-plugin.version>0.12</apache-rat-plugin.version>
+    <apache-rat-plugin.version>0.13</apache-rat-plugin.version>
     <scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
     <scalatest.version>3.0.1</scalatest.version>
     <scalatest.spark3.version>3.1.0</scalatest.spark3.version>
@@ -312,7 +312,7 @@
         <configuration>
           <autoVersionSubmodules>true</autoVersionSubmodules>
           <useReleaseProfile>false</useReleaseProfile>
-          <releaseProfiles>release</releaseProfiles>
+          <releaseProfiles>release,integration-tests</releaseProfiles>
           <goals>deploy</goals>
         </configuration>
       </plugin>
@@ -432,6 +432,7 @@
               <exclude>DISCLAIMER</exclude>
               <exclude>**/.*</exclude>
               <exclude>**/*.json</exclude>
+              <exclude>**/*.hfile</exclude>
               <exclude>**/*.log</exclude>
               <exclude>**/*.sqltemplate</exclude>
               <exclude>**/compose_env</exclude>
@@ -444,6 +445,9 @@
               <exclude>**/generated-sources/**</exclude>
               <exclude>.github/**</exclude>
               <exclude>**/*.sql</exclude>
+              <!-- local files not in version control -->
+              <exclude>**/*.iml</exclude>
+              <exclude>.mvn/**</exclude>
             </excludes>
           </configuration>
           <executions>
@@ -1200,9 +1204,6 @@
           <value>true</value>
         </property>
       </activation>
-      <properties>
-        <integration-tests>true</integration-tests>
-      </properties>
       <build>
         <plugins>
           <plugin>
@@ -1375,7 +1376,8 @@
       <id>integration-tests</id>
       <activation>
         <property>
-          <name>integration-tests</name>
+          <name>deployArtifacts</name>
+          <value>true</value>
         </property>
       </activation>
       <modules>
@@ -1386,7 +1388,7 @@
       <properties>
         <skipUTs>true</skipUTs>
         <skipFTs>true</skipFTs>
-        <skipITs>false</skipITs>
+        <skipITs>${skipTests}</skipITs>
       </properties>
       <build>
         <plugins>

rfc/rfc-34/rfc-34.md

@@ -1,3 +1,19 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
 # Hudi BigQuery Integration
 
 ## Abstract

scripts/release/create_source_release.sh

@@ -72,6 +72,7 @@ rsync -a \
   --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \
   --exclude "docs/content" --exclude ".rubydeps" \
   --exclude "rfc" \
+  --exclude "docker/images" \
   . hudi-$RELEASE_VERSION
 
 tar czf ${RELEASE_DIR}/hudi-${RELEASE_VERSION}.src.tgz hudi-$RELEASE_VERSION

scripts/release/deploy_staging_jars.sh

@@ -73,6 +73,6 @@ fi
 for v in "${ALL_VERSION_OPTS[@]}"
 do
   echo "Deploying to repository.apache.org with version option ${v}"
-  COMMON_OPTIONS="${v} -Prelease -DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
+  COMMON_OPTIONS="${v} -DdeployArtifacts=true -DskipTests -DretryFailedDeploymentCount=10"
   $MVN clean deploy $COMMON_OPTIONS
 done

scripts/release/validate_staged_release.sh

@@ -124,7 +124,7 @@ numBinaryFiles=`find . -iname '*' | xargs -I {} file -I {} | grep -va directory
 if [ "$numBinaryFiles" -gt "0" ]; then
   echo -e "There were non-text files in source release. Please check below\n"
   find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -v "/src/test/" | grep -va 'application/json' | grep -va 'text/' |  grep -va 'application/xml'
-  exit -1
+  exit 1
 fi
 echo -e "\t\tNo Binary Files in Source Release? - [OK]\n"
 ### END: Binary Files Check
@@ -134,7 +134,7 @@ echo "Checking for DISCLAIMER"
 disclaimerFile="./DISCLAIMER"
 if [ -f "$disclaimerFile" ]; then
   echo "DISCLAIMER file should not be present "
-  exit -1
+  exit 1
 fi
 echo -e "\t\tDISCLAIMER file exists ? [OK]\n"
 
@@ -144,23 +144,23 @@ licenseFile="./LICENSE"
 noticeFile="./NOTICE"
 if [ ! -f "$licenseFile" ]; then
   echo "License file missing"
-  exit -1
+  exit 1
 fi
 echo -e "\t\tLicense file exists ? [OK]"
 
 if [ ! -f "$noticeFile" ]; then
   echo "Notice file missing"
-  exit -1
+  exit 1
 fi
 echo -e "\t\tNotice file exists ? [OK]\n"
 
 ### Licensing Check
 echo "Performing custom Licensing Check "
-numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data'| grep -v '.commit' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
+numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
 if [ "$numfilesWithNoLicense" -gt  "0" ]; then
   echo "There were some source files that did not have Apache License"
-  find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
+  find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.hfile' | grep -v '.data' | grep -v '.commit' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | grep -v 'ObjectSizeCalculator.java' | grep -v 'AvroConversionHelper.scala' | grep -v "fixtures" | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
-  exit -1
+  exit 1
 fi
 echo -e "\t\tLicensing Check Passed [OK]\n"