1
0

HUDI-121 : Address comments during RC2 voting

1. Remove dnl utils jar from git
2. Add LICENSE Headers in missing files
3. Fix NOTICE and LICENSE in all HUDI packages and in top-level
4. Fix License wording in certain HUDI source files
5. Include non java/scala code in RAT licensing check
6. Use whitelist to include dependencies as part of timeline-server bundling
This commit is contained in:
Balaji Varadarajan
2019-09-20 23:01:32 -07:00
committed by Balaji Varadarajan
parent 50a073ff57
commit 6e8a28bcae
113 changed files with 3215 additions and 15249 deletions

View File

@@ -0,0 +1,82 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##
## Variables with defaults (if not overwritten by environment)
##
MVN=${MVN:-mvn}
# fail immediately
set -o errexit
set -o nounset
# print command before executing
set -o xtrace
CURR_DIR=`pwd`
if [[ `basename $CURR_DIR` != "release" ]] ; then
echo "You have to call the script from the release/ dir"
exit 1
fi
RELEASE_VERSION=`grep -A 5 "<artifactId>hudi</artifactId>" ../pom.xml | grep '<version>' | sed -e 's/<version>//' -e 's/<\/version>//' -e 's/ //g'`
if [ -z "${RELEASE_VERSION}" ]; then
echo "RELEASE_VERSION was not set."
exit 1
fi
echo "RELEASE_VERSION=${RELEASE_VERSION}"
if [ "$(uname)" == "Darwin" ]; then
SHASUM="shasum -a 512"
else
SHASUM="sha512sum"
fi
###########################
cd ..
HUDI_DIR=`pwd`
RELEASE_DIR=${HUDI_DIR}/src_release
CLONE_DIR=${RELEASE_DIR}/hudi-tmp-clone
echo "Creating source package"
rm -rf ${RELEASE_DIR}
mkdir -p ${RELEASE_DIR}
# create a temporary git clone to ensure that we have a pristine source release
git clone ${HUDI_DIR} ${CLONE_DIR}
cd ${CLONE_DIR}
rsync -a \
--exclude ".git" --exclude ".gitignore" --exclude ".gitattributes" --exclude ".travis.yml" \
--exclude ".github" --exclude "target" \
--exclude ".idea" --exclude "*.iml" --exclude ".DS_Store" --exclude "build-target" \
--exclude "docs/content" --exclude ".rubydeps" \
. hudi-$RELEASE_VERSION
tar czf ${RELEASE_DIR}/hudi-${RELEASE_VERSION}.src.tgz hudi-$RELEASE_VERSION
gpg --armor --detach-sig ${RELEASE_DIR}/hudi-${RELEASE_VERSION}.src.tgz
cd ${RELEASE_DIR}
$SHASUM hudi-${RELEASE_VERSION}.src.tgz > hudi-${RELEASE_VERSION}.src.tgz.sha512
cd ${CURR_DIR}
rm -rf ${CLONE_DIR}

View File

@@ -0,0 +1,163 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script will update apache hudi (incubating) master branch with next release version
# and cut release branch for current development version.
# Parse parameters passing into the script
set -e
function clean_up(){
echo "Do you want to clean local clone repo? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
cd ~
rm -rf ${LOCAL_CLONE_DIR}
echo "Clean up local repo."
fi
}
if [[ $# -eq 1 && $1 = "-h" ]]; then
echo "This script will update apache hudi(incubating) master branch with next release version and cut release branch for current development version."
echo "There are two params required:"
echo "--release=\${CURRENT_RELEASE_VERSION}"
echo "--next_release=\${NEXT_RELEASE_VERSION}"
exit
else
for param in "$@"
do
if [[ $param =~ --release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
RELEASE=${BASH_REMATCH[1]}
fi
if [[ $param =~ --next_release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
NEXT_VERSION_IN_BASE_BRANCH=${BASH_REMATCH[1]}
fi
done
fi
if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" ]]; then
echo "This sricpt needs to be ran with params, please run with -h to get more instructions."
exit
fi
MASTER_BRANCH=master
RELEASE_BRANCH=release-${RELEASE}
GITHUB_REPO_URL=git@github.com:apache/incubator-hudi.git
HUDI_ROOT_DIR=incubator-hudi
LOCAL_CLONE_DIR=hudi_release_${RELEASE}
echo "=====================Environment Variables====================="
echo "version: ${RELEASE}"
echo "next_release: ${NEXT_VERSION_IN_BASE_BRANCH}"
echo "working master branch: ${MASTER_BRANCH}"
echo "working release branch: ${RELEASE_BRANCH}"
echo "local repo dir: ~/${LOCAL_CLONE_DIR}/${HUDI_ROOT_DIR}"
echo "==============================================================="
cd ~
if [[ -d ${LOCAL_CLONE_DIR} ]]; then
rm -rf ${LOCAL_CLONE_DIR}
fi
mkdir ${LOCAL_CLONE_DIR}
cd ${LOCAL_CLONE_DIR}
git clone ${GITHUB_REPO_URL}
cd ${HUDI_ROOT_DIR}
# Update Notice.txt
mvn notice:generate
echo "==============Update NOTICE.txt in master branch as following================"
git diff
echo "==============================================================="
echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
echo "Exit without committing any changes on master branch."
clean_up
exit
fi
git commit -am "Updating NOTICE.txt in master" --allow-empty
# Pushing NOTICE.txt changes to master
if git push origin ${MASTER_BRANCH}; then
break
else
clean_up
exit
fi
# Now, create local release branch
git branch ${RELEASE_BRANCH}
git checkout ${MASTER_BRANCH}
echo "====================Current working branch====================="
echo ${MASTER_BRANCH}
echo "==============================================================="
# Update master branch
mvn versions:set -DnewVersion=${NEXT_VERSION_IN_BASE_BRANCH}-SNAPSHOT
echo "==============Update master branch as following================"
git diff
echo "==============================================================="
echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
echo "Exit without committing any changes on master branch."
clean_up
exit
fi
git commit -am "Moving to ${NEXT_VERSION_IN_BASE_BRANCH}-SNAPSHOT on master branch."
if git push origin ${MASTER_BRANCH}; then
break
else
clean_up
exit
fi
# Checkout and update release branch - Add incubating and remove snapshot
git checkout ${RELEASE_BRANCH}
mvn versions:set -DnewVersion=${RELEASE}-incubating
echo "==================Current working branch======================="
echo ${RELEASE_BRANCH}
echo "==============================================================="
echo "===============Update release branch as following=============="
git diff
echo "==============================================================="
echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
echo "Exit without committing any changes on release branch."
clean_up
exit
fi
git commit -am "Create release branch for version ${RELEASE}."
git push --set-upstream origin ${RELEASE_BRANCH}
clean_up

View File

@@ -0,0 +1,44 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
##
## Variables with defaults (if not overwritten by environment)
##
MVN=${MVN:-mvn}
# fail immediately
set -o errexit
set -o nounset
# print command before executing
set -o xtrace
CURR_DIR=`pwd`
if [[ `basename $CURR_DIR` != "release" ]] ; then
echo "You have to call the script from the release/ dir"
exit 1
fi
###########################
cd ..
echo "Deploying to repository.apache.org"
COMMON_OPTIONS="-Prelease -DskipTests -DretryFailedDeploymentCount=10 -DdeployArtifacts=true"
$MVN clean deploy $COMMON_OPTIONS

View File

@@ -0,0 +1,39 @@
#!/usr/bin/python
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import fileinput
import sys
import os
for line in fileinput.input():
line = line.rstrip()
cols = line.split(":")
classifier = ""
scope = ""
pkg = ""
type = ""
version = ""
if len(cols) == 4:
(scope,pkg,type,version) = cols
else:
(scope,pkg,type,classifier,version) = cols
classifier = "-{}".format(classifier)
path = "{5},{6}/.m2/repository/{0}/{1}/{2}/{1}-{2}{3}.{4}".format(scope.replace(".","/"), pkg, version,classifier,type, line, os.environ['HOME'])
print path

View File

@@ -0,0 +1,36 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
path=`echo $1 | cut -f 2 -d ","`;
mvncoord=`echo $1 | cut -f 1 -d ","`;
scope=`echo ${mvncoord} | cut -f 1 -d ":"`;
mkdir tmp > /dev/null
pushd tmp > /dev/null
jar -xf $path
pkg=`echo $path | rev | cut -d "/" -f 1 | rev | sed -e 's/\.jar//'`
echo "The binary distribution of this org.apache.hudi package bundles binary of ${scope}:$pkg with the following NOTICE"
echo "============================================================================================================"
find . -iname '*NOTICE*' | xargs cat
echo "============================================================================================================"
echo ""
echo ""
popd > /dev/null
rm -rf tmp

View File

@@ -0,0 +1,104 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script will install and configure GPG key.
set -e
LOCAL_SVN_DIR=local_svn_dir
ROOT_SVN_URL=https://dist.apache.org/repos/dist/
DEV_REPO=dev/incubator
RELEASE_REPO=release/incubator
HUDI_REPO=hudi
cd ~
echo "=================Checking GPG Key===================="
echo "You need a GPG key which reflects your Apache account."
echo "Do you want to generate a new GPG key associated with your Apache account? [y|N]"
read confirmation
if [[ $confirmation = "y" ]]; then
echo "===============Generating new GPG key================"
sudo apt-get install rng-tools
sudo rngd -r /dev/urandom
gpg --full-generate-key
fi
echo "================Listing all GPG keys================="
gpg --list-keys
echo "Please copy the public key which is associated with your Apache account:"
read pub_key
echo "===========Configuring git signing key==============="
git config --global user.signingkey $pub_key
git config --list
echo "===========Adding your key into KEYS file============"
echo "It's required to append your key into KEYS file in dist.apache.org"
echo "Have you put your key in KEYS? [y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
echo "Only PMC member can write into dist.apache.org. Are you a PMC member? [y|N]"
read pmc_permission
if [[ $pmc_permission != "y" ]]; then
echo "Please ask a PMC member to help you add your key in dev@ list."
echo "Skip adding key into dist.apache.org/KEYS file."
else
echo "Please input your name: "
read name
echo "======Starting updating KEYS file in dev repo===="
if [[ -d ${LOCAL_SVN_DIR} ]]; then
rm -rf ${LOCAL_SVN_DIR}
fi
mkdir ${LOCAL_SVN_DIR}
cd ${LOCAL_SVN_DIR}
svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}
cd ${HUDI_REPO}
(gpg --list-sigs ${name} && gpg --armor --export ${name}) >> KEYS
svn status
echo "Please review all changes. Do you confirm to commit? [y|N]"
read commit_confirmation
if [[ $commit_confirmation = "y" ]]; then
svn commit --no-auth-cache KEYS
else
echo "Not commit new changes into ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}${DEV_REPO}/KEYS"
fi
cd ~/${LOCAL_SVN_DIR}
echo "===Starting updating KEYS file in release repo==="
svn co ${ROOT_SVN_URL}/${RELEASE_REPO}/${HUDI_REPO}
cd ${HUDI_REPO}
(gpg --list-sigs ${name} && gpg --armor --export ${name}) >> KEYS
svn status
echo "Please review all changes. Do you confirm to commit? [y|N]"
read commit_confirmation
if [[ $commit_confirmation = "y" ]]; then
svn commit --no-auth-cache KEYS
else
echo "Not commit new changes into ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}${RELEASE_REPO}/KEYS"
fi
cd ~
rm -rf ${LOCAL_SVN_DIR}
fi
fi
echo "================Setting up gpg agent================="
eval $(gpg-agent --daemon --no-grab --write-env-file $HOME/.gpg-agent-info)
export GPG_TTY=$(tty)
export GPG_AGENT_INFO

View File

@@ -0,0 +1,125 @@
#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# fail immediately
set -o errexit
set -o nounset
# print command before executing
set -o xtrace
CURR_DIR=`pwd`
if [[ `basename $CURR_DIR` != "release" ]] ; then
echo "You have to call the script from the release/ dir"
exit 1
fi
if [ "$(uname)" == "Darwin" ]; then
SHASUM="shasum -a 512"
else
SHASUM="sha512sum"
fi
# Get to a scratch dir
RELEASE_TOOL_DIR=`pwd`
WORK_DIR=${RELEASE_TOOL_DIR}/validation_scratch_dir
rm -rf $WORK_DIR
mkdir $WORK_DIR
pushd $WORK_DIR
# Checkout dist incubator repo
LOCAL_SVN_DIR=local_svn_dir
ROOT_SVN_URL=https://dist.apache.org/repos/dist/
DEV_REPO=dev/incubator
#RELEASE_REPO=release/incubator
HUDI_REPO=hudi
rm -rf $LOCAL_SVN_DIR
mkdir $LOCAL_SVN_DIR
cd $LOCAL_SVN_DIR
svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}
cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512
echo "Checking Checksum of Source Release"
diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512
echo "Checksum Check of Source Release - [OK]"
# GPG Check
echo "Checking Signature"
gpg --import ../KEYS
gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz
echo "Signature Check - [OK]"
# Untar
tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz
pushd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
### BEGIN: Binary Files Check
echo "Checking for binary files in source release"
numBinaryFiles=`find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/' | wc -l`
if [ "$numBinaryFiles" > "0" ]; then
echo "There were non-text files in source release. Please check below\n"
find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/'
exit -1
fi
echo "Binary Files in Source Release Check - [OK]"
### END: Binary Files Check
### Checking for DISCLAIMER
disclaimerFile="./DISCLAIMER"
if [ ! -f "$disclaimerFile" ]; then
echo "DISCLAIMER file missing"
exit -1
fi
echo "DISCLAIMER file exists ? [OK]"
### Checking for LICENSE and NOTICE
licenseFile="./LICENSE"
noticeFile="./NOTICE"
if [ ! -f "$licenseFile" ]; then
echo "License file missing"
exit -1
fi
echo "License file exists ? [OK]"
if [ ! -f "$noticeFile" ]; then
echo "Notice file missing"
exit -1
fi
echo "Notice file exists ? [OK]"
### Checking for RAT
echo "Running RAT Check Check"
mvn rat:check
echo "RAT Check Passed [OK]"
### Licensing Check
echo "Performing custom Licensing Check "
numfilesWithNoLicense=`find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
if [ "$numfilesWithNoLicense" > "0" ]; then
echo "There were some source files that did not have Apache License"
find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
exit -1
fi
echo "Licensing Check Passed [OK]"
popd

35
scripts/run_travis_tests.sh Executable file
View File

@@ -0,0 +1,35 @@
#!/bin/bash
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
mode=$1
if [ "$mode" = "unit" ];
then
echo "Running Unit Tests"
mvn test -DskipITs=true -B
elif [ "$mode" = "integration" ];
then
echo "Running Integration Tests"
mvn verify -DskipUTs=true -B
else
echo "Unknown mode $mode"
exit 1;
fi