HUDI-121 : Address comments during RC2 voting

1. Remove dnl utils jar from git 2. Add LICENSE Headers in missing files 3. Fix NOTICE and LICENSE in all HUDI packages and in top-level 4. Fix License wording in certain HUDI source files 5. Include non java/scala code in RAT licensing check 6. Use whitelist to include dependencies as part of timeline-server bundling
2019-09-20 23:01:32 -07:00
parent 50a073ff57
commit 6e8a28bcae
113 changed files with 3215 additions and 15249 deletions
--- a/scripts/release/create_source_release.sh
+++ b/scripts/release/create_source_release.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Variables with defaults (if not overwritten by environment)
+##
+MVN=${MVN:-mvn}
+
+# fail immediately
+set -o errexit
+set -o nounset
+# print command before executing
+set -o xtrace
+
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "release" ]] ; then
+  echo "You have to call the script from the release/ dir"
+  exit 1
+fi
+
+RELEASE_VERSION=`grep -A 5 "<artifactId>hudi</artifactId>" ../pom.xml  | grep '<version>' | sed -e 's/<version>//' -e 's/<\/version>//' -e 's/ //g'`
+
+if [ -z "${RELEASE_VERSION}" ]; then
+    echo "RELEASE_VERSION was not set."
+    exit 1
+fi
+
+echo "RELEASE_VERSION=${RELEASE_VERSION}"
+
+if [ "$(uname)" == "Darwin" ]; then
+    SHASUM="shasum -a 512"
+else
+    SHASUM="sha512sum"
+fi
+
+###########################
+
+cd ..
+
+HUDI_DIR=`pwd`
+RELEASE_DIR=${HUDI_DIR}/src_release
+CLONE_DIR=${RELEASE_DIR}/hudi-tmp-clone
+
+echo "Creating source package"
+
+rm -rf ${RELEASE_DIR}
+mkdir -p ${RELEASE_DIR}
+
+# create a temporary git clone to ensure that we have a pristine source release
+git clone ${HUDI_DIR} ${CLONE_DIR}
+cd ${CLONE_DIR}
+
+rsync -a \
+  --exclude ".git" --exclude ".gitignore" --exclude ".gitattributes" --exclude ".travis.yml" \
+  --exclude ".github" --exclude "target" \
+  --exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \
+  --exclude "docs/content" --exclude ".rubydeps" \
+  . hudi-$RELEASE_VERSION
+
+tar czf ${RELEASE_DIR}/hudi-${RELEASE_VERSION}.src.tgz hudi-$RELEASE_VERSION
+gpg --armor --detach-sig ${RELEASE_DIR}/hudi-${RELEASE_VERSION}.src.tgz
+cd ${RELEASE_DIR}
+$SHASUM hudi-${RELEASE_VERSION}.src.tgz > hudi-${RELEASE_VERSION}.src.tgz.sha512
+
+cd ${CURR_DIR}
+rm -rf ${CLONE_DIR}
--- a/scripts/release/cut_release_branch.sh
+++ b/scripts/release/cut_release_branch.sh
@@ -0,0 +1,163 @@
+#!/bin/bash
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# This script will update apache hudi (incubating) master branch with next release version
+# and cut release branch for current development version.
+
+# Parse parameters passing into the script
+
+set -e
+
+function clean_up(){
+  echo "Do you want to clean local clone repo? [y|N]"
+  read confirmation
+  if [[ $confirmation = "y" ]]; then
+    cd ~
+    rm -rf ${LOCAL_CLONE_DIR}
+    echo "Clean up local repo."
+  fi
+}
+
+if [[ $# -eq 1 && $1 = "-h" ]]; then
+	echo "This script will update apache hudi(incubating) master branch with next release version and cut release branch for current development version."
+	echo "There are two params required:"
+	echo "--release=\${CURRENT_RELEASE_VERSION}"
+	echo "--next_release=\${NEXT_RELEASE_VERSION}"
+	exit
+else
+	for param in "$@"
+	do
+		if [[ $param =~ --release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
+			RELEASE=${BASH_REMATCH[1]}
+		fi
+		if [[ $param =~ --next_release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
+			NEXT_VERSION_IN_BASE_BRANCH=${BASH_REMATCH[1]}
+		fi
+	done
+fi
+if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" ]]; then
+	echo "This sricpt needs to be ran with params, please run with -h to get more instructions."
+	exit
+fi
+
+
+MASTER_BRANCH=master
+RELEASE_BRANCH=release-${RELEASE}
+GITHUB_REPO_URL=git@github.com:apache/incubator-hudi.git
+HUDI_ROOT_DIR=incubator-hudi
+LOCAL_CLONE_DIR=hudi_release_${RELEASE}
+
+echo "=====================Environment Variables====================="
+echo "version: ${RELEASE}"
+echo "next_release: ${NEXT_VERSION_IN_BASE_BRANCH}"
+echo "working master branch: ${MASTER_BRANCH}"
+echo "working release branch: ${RELEASE_BRANCH}"
+echo "local repo dir: ~/${LOCAL_CLONE_DIR}/${HUDI_ROOT_DIR}"
+echo "==============================================================="
+
+cd ~
+if [[ -d ${LOCAL_CLONE_DIR} ]]; then
+  rm -rf ${LOCAL_CLONE_DIR}
+fi
+mkdir ${LOCAL_CLONE_DIR}
+cd ${LOCAL_CLONE_DIR}
+git clone ${GITHUB_REPO_URL}
+cd ${HUDI_ROOT_DIR}
+
+# Update Notice.txt
+mvn notice:generate
+
+echo "==============Update NOTICE.txt in master branch as following================"
+git diff
+echo "==============================================================="
+
+echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
+read confirmation
+if [[ $confirmation != "y" ]]; then
+  echo "Exit without committing any changes on master branch."
+  clean_up
+  exit
+fi
+
+git commit -am "Updating NOTICE.txt in master" --allow-empty
+
+# Pushing NOTICE.txt changes to master
+if git push origin ${MASTER_BRANCH}; then
+  break
+else
+  clean_up
+  exit
+fi
+
+# Now, create local release branch
+git branch ${RELEASE_BRANCH}
+
+git checkout ${MASTER_BRANCH}
+
+echo "====================Current working branch====================="
+echo ${MASTER_BRANCH}
+echo "==============================================================="
+
+# Update master branch
+mvn versions:set -DnewVersion=${NEXT_VERSION_IN_BASE_BRANCH}-SNAPSHOT
+
+echo "==============Update master branch as following================"
+git diff
+echo "==============================================================="
+
+echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
+read confirmation
+if [[ $confirmation != "y" ]]; then
+  echo "Exit without committing any changes on master branch."
+  clean_up
+  exit
+fi
+
+git commit -am "Moving to ${NEXT_VERSION_IN_BASE_BRANCH}-SNAPSHOT on master branch."
+
+if git push origin ${MASTER_BRANCH}; then
+  break
+else
+  clean_up
+  exit
+fi
+
+# Checkout and update release branch - Add incubating and remove snapshot
+git checkout ${RELEASE_BRANCH}
+mvn versions:set -DnewVersion=${RELEASE}-incubating
+
+echo "==================Current working branch======================="
+echo ${RELEASE_BRANCH}
+echo "==============================================================="
+
+echo "===============Update release branch as following=============="
+git diff
+echo "==============================================================="
+
+echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
+read confirmation
+if [[ $confirmation != "y" ]]; then
+  echo "Exit without committing any changes on release branch."
+  clean_up
+  exit
+fi
+
+git commit -am "Create release branch for version ${RELEASE}."
+git push --set-upstream origin ${RELEASE_BRANCH}
+
+clean_up
--- a/scripts/release/deploy_staging_jars.sh
+++ b/scripts/release/deploy_staging_jars.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+##
+## Variables with defaults (if not overwritten by environment)
+##
+MVN=${MVN:-mvn}
+
+# fail immediately
+set -o errexit
+set -o nounset
+# print command before executing
+set -o xtrace
+
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "release" ]] ; then
+  echo "You have to call the script from the release/ dir"
+  exit 1
+fi
+
+###########################
+
+cd ..
+
+echo "Deploying to repository.apache.org"
+
+COMMON_OPTIONS="-Prelease -DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
+$MVN clean deploy $COMMON_OPTIONS
--- a/scripts/release/generate_jar_path.py
+++ b/scripts/release/generate_jar_path.py
@@ -0,0 +1,39 @@
+#!/usr/bin/python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import fileinput
+import sys
+import os
+
+for line in fileinput.input():
+  line = line.rstrip()
+  cols = line.split(":")
+  classifier = ""
+  scope = ""
+  pkg = ""
+  type = ""
+  version = ""
+  if len(cols) == 4:
+   (scope,pkg,type,version) = cols
+  else:
+   (scope,pkg,type,classifier,version) = cols
+   classifier = "-{}".format(classifier)
+  path = "{5},{6}/.m2/repository/{0}/{1}/{2}/{1}-{2}{3}.{4}".format(scope.replace(".","/"), pkg, version,classifier,type, line, os.environ['HOME'])
+  print path
+   
--- a/scripts/release/generate_notice.sh
+++ b/scripts/release/generate_notice.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+path=`echo $1 | cut -f 2 -d ","`;
+mvncoord=`echo $1 | cut -f 1 -d ","`;
+scope=`echo ${mvncoord} | cut -f 1 -d ":"`;
+mkdir tmp > /dev/null
+pushd tmp > /dev/null
+jar -xf $path
+
+pkg=`echo $path | rev | cut -d "/" -f 1 | rev | sed -e 's/\.jar//'`
+echo "The binary distribution of this org.apache.hudi package bundles binary of ${scope}:$pkg with the following NOTICE"
+echo "============================================================================================================"
+find . -iname '*NOTICE*' | xargs cat 
+echo "============================================================================================================"
+echo ""
+echo ""
+popd > /dev/null
+rm -rf tmp
--- a/scripts/release/preparation_before_release.sh
+++ b/scripts/release/preparation_before_release.sh
@@ -0,0 +1,104 @@
+#!/bin/bash
+#
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# This script will install and configure GPG key.
+
+set -e
+
+LOCAL_SVN_DIR=local_svn_dir
+ROOT_SVN_URL=https://dist.apache.org/repos/dist/
+DEV_REPO=dev/incubator
+RELEASE_REPO=release/incubator
+HUDI_REPO=hudi
+
+cd ~
+
+echo "=================Checking GPG Key===================="
+echo "You need a GPG key which reflects your Apache account."
+echo "Do you want to generate a new GPG key associated with your Apache account? [y|N]"
+read confirmation
+if [[ $confirmation = "y" ]]; then
+  echo "===============Generating new GPG key================"
+  sudo apt-get install rng-tools
+  sudo rngd -r /dev/urandom
+  gpg --full-generate-key
+fi
+
+echo "================Listing all GPG keys================="
+gpg --list-keys
+echo "Please copy the public key which is associated with your Apache account:"
+read pub_key
+
+echo "===========Configuring git signing key==============="
+git config --global user.signingkey $pub_key
+git config --list
+
+echo "===========Adding your key into KEYS file============"
+echo "It's required to append your key into KEYS file in dist.apache.org"
+echo "Have you put your key in KEYS? [y|N]"
+read confirmation
+if [[ $confirmation != "y" ]]; then
+  echo "Only PMC member can write into dist.apache.org. Are you a PMC member? [y|N]"
+  read pmc_permission
+  if [[ $pmc_permission != "y" ]]; then
+    echo "Please ask a PMC member to help you add your key in dev@ list."
+    echo "Skip adding key into dist.apache.org/KEYS file."
+  else
+    echo "Please input your name: "
+    read name
+    echo "======Starting updating KEYS file in dev repo===="
+    if [[ -d ${LOCAL_SVN_DIR} ]]; then
+      rm -rf ${LOCAL_SVN_DIR}
+    fi
+    mkdir ${LOCAL_SVN_DIR}
+    cd ${LOCAL_SVN_DIR}
+    svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}
+    cd ${HUDI_REPO}
+    (gpg --list-sigs ${name} && gpg --armor --export ${name}) >> KEYS
+    svn status
+    echo "Please review all changes. Do you confirm to commit? [y|N]"
+    read commit_confirmation
+    if [[ $commit_confirmation = "y" ]]; then
+      svn commit --no-auth-cache KEYS
+    else
+      echo "Not commit new changes into ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}${DEV_REPO}/KEYS"
+    fi
+
+    cd ~/${LOCAL_SVN_DIR}
+    echo "===Starting updating KEYS file in release repo==="
+    svn co ${ROOT_SVN_URL}/${RELEASE_REPO}/${HUDI_REPO}
+    cd ${HUDI_REPO}
+    (gpg --list-sigs ${name} && gpg --armor --export ${name}) >> KEYS
+    svn status
+    echo "Please review all changes. Do you confirm to commit? [y|N]"
+    read commit_confirmation
+    if [[ $commit_confirmation = "y" ]]; then
+      svn commit --no-auth-cache KEYS
+    else
+      echo "Not commit new changes into ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}${RELEASE_REPO}/KEYS"
+    fi
+
+    cd ~
+    rm -rf ${LOCAL_SVN_DIR}
+  fi
+fi
+
+echo "================Setting up gpg agent================="
+eval $(gpg-agent --daemon --no-grab --write-env-file $HOME/.gpg-agent-info)
+export GPG_TTY=$(tty)
+export GPG_AGENT_INFO
--- a/scripts/release/validate_staged_release.sh
+++ b/scripts/release/validate_staged_release.sh
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# fail immediately
+set -o errexit
+set -o nounset
+# print command before executing
+set -o xtrace
+
+CURR_DIR=`pwd`
+if [[ `basename $CURR_DIR` != "release" ]] ; then
+  echo "You have to call the script from the release/ dir"
+  exit 1
+fi
+
+if [ "$(uname)" == "Darwin" ]; then
+    SHASUM="shasum -a 512"
+else
+    SHASUM="sha512sum"
+fi
+
+
+# Get to a scratch dir
+RELEASE_TOOL_DIR=`pwd`
+WORK_DIR=${RELEASE_TOOL_DIR}/validation_scratch_dir
+rm -rf $WORK_DIR
+mkdir $WORK_DIR
+pushd $WORK_DIR
+
+# Checkout dist incubator repo
+LOCAL_SVN_DIR=local_svn_dir
+ROOT_SVN_URL=https://dist.apache.org/repos/dist/
+DEV_REPO=dev/incubator
+#RELEASE_REPO=release/incubator
+HUDI_REPO=hudi
+
+rm -rf $LOCAL_SVN_DIR
+mkdir $LOCAL_SVN_DIR
+cd $LOCAL_SVN_DIR
+svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}
+
+cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
+$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512
+
+echo "Checking Checksum of Source Release"
+diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512
+echo "Checksum Check of Source Release - [OK]"
+
+# GPG Check
+echo "Checking Signature"
+gpg --import ../KEYS
+gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz 
+echo "Signature Check - [OK]"
+
+# Untar 
+tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz 
+pushd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
+
+### BEGIN: Binary Files Check
+echo "Checking for binary files in source release"
+numBinaryFiles=`find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/' | wc -l`
+
+if [ "$numBinaryFiles" > "0" ]; then
+  echo "There were non-text files in source release. Please check below\n"
+  find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/'
+  exit -1
+fi
+
+echo "Binary Files in Source Release Check - [OK]"
+### END: Binary Files Check
+
+### Checking for DISCLAIMER
+disclaimerFile="./DISCLAIMER"
+if [ ! -f "$disclaimerFile" ]; then
+  echo "DISCLAIMER file missing"
+  exit -1
+fi
+echo "DISCLAIMER file exists ? [OK]"
+
+### Checking for LICENSE and NOTICE
+licenseFile="./LICENSE"
+noticeFile="./NOTICE"
+if [ ! -f "$licenseFile" ]; then
+  echo "License file missing"
+  exit -1
+fi
+echo "License file exists ? [OK]"
+
+if [ ! -f "$noticeFile" ]; then
+  echo "Notice file missing"
+  exit -1
+fi
+echo "Notice file exists ? [OK]"
+
+### Checking for RAT
+echo "Running RAT Check Check"
+mvn rat:check
+echo "RAT Check Passed [OK]"
+
+### Licensing Check
+echo "Performing custom Licensing Check "
+numfilesWithNoLicense=`find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
+if [ "$numfilesWithNoLicense" > "0" ]; then
+  echo "There were some source files that did not have Apache License"
+  find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
+  exit -1
+fi
+echo "Licensing Check Passed [OK]"
+popd