[HUDI-121] Fix bugs in Release Scripts found during RC creation
This commit is contained in:
Balaji Varadarajan
committed by
Balaji Varadarajan
Balaji Varadarajan
parent
1d2e3dcc65
commit
e41835fd91
@@ -35,9 +35,10 @@ function clean_up(){
|
||||
|
||||
if [[ $# -eq 1 && $1 = "-h" ]]; then
|
||||
echo "This script will update apache hudi(incubating) master branch with next release version and cut release branch for current development version."
|
||||
echo "There are two params required:"
|
||||
echo "There are 3 params required:"
|
||||
echo "--release=\${CURRENT_RELEASE_VERSION}"
|
||||
echo "--next_release=\${NEXT_RELEASE_VERSION}"
|
||||
echo "--rc_num=\${RC_NUM}"
|
||||
exit
|
||||
else
|
||||
for param in "$@"
|
||||
@@ -48,9 +49,13 @@ else
|
||||
if [[ $param =~ --next_release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
|
||||
NEXT_VERSION_IN_BASE_BRANCH=${BASH_REMATCH[1]}
|
||||
fi
|
||||
if [[ $param =~ --rc_num\=([0-9]*) ]]; then
|
||||
RC_NUM=${BASH_REMATCH[1]}
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" ]]; then
|
||||
|
||||
if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" || -z "$RC_NUM" ]]; then
|
||||
echo "This sricpt needs to be ran with params, please run with -h to get more instructions."
|
||||
exit
|
||||
fi
|
||||
@@ -68,42 +73,19 @@ echo "next_release: ${NEXT_VERSION_IN_BASE_BRANCH}"
|
||||
echo "working master branch: ${MASTER_BRANCH}"
|
||||
echo "working release branch: ${RELEASE_BRANCH}"
|
||||
echo "local repo dir: ~/${LOCAL_CLONE_DIR}/${HUDI_ROOT_DIR}"
|
||||
echo "RC_NUM: $RC_NUM
|
||||
echo "==============================================================="
|
||||
|
||||
cd ~
|
||||
if [[ -d ${LOCAL_CLONE_DIR} ]]; then
|
||||
rm -rf ${LOCAL_CLONE_DIR}
|
||||
fi
|
||||
|
||||
mkdir ${LOCAL_CLONE_DIR}
|
||||
cd ${LOCAL_CLONE_DIR}
|
||||
git clone ${GITHUB_REPO_URL}
|
||||
cd ${HUDI_ROOT_DIR}
|
||||
|
||||
# Update Notice.txt
|
||||
mvn notice:generate
|
||||
|
||||
echo "==============Update NOTICE.txt in master branch as following================"
|
||||
git diff
|
||||
echo "==============================================================="
|
||||
|
||||
echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
|
||||
read confirmation
|
||||
if [[ $confirmation != "y" ]]; then
|
||||
echo "Exit without committing any changes on master branch."
|
||||
clean_up
|
||||
exit
|
||||
fi
|
||||
|
||||
git commit -am "Updating NOTICE.txt in master" --allow-empty
|
||||
|
||||
# Pushing NOTICE.txt changes to master
|
||||
if git push origin ${MASTER_BRANCH}; then
|
||||
break
|
||||
else
|
||||
clean_up
|
||||
exit
|
||||
fi
|
||||
|
||||
# Now, create local release branch
|
||||
git branch ${RELEASE_BRANCH}
|
||||
|
||||
@@ -139,7 +121,7 @@ fi
|
||||
|
||||
# Checkout and update release branch - Add incubating and remove snapshot
|
||||
git checkout ${RELEASE_BRANCH}
|
||||
mvn versions:set -DnewVersion=${RELEASE}-incubating
|
||||
mvn versions:set -DnewVersion=${RELEASE}-incubating-rc${RC_NUM}
|
||||
|
||||
echo "==================Current working branch======================="
|
||||
echo ${RELEASE_BRANCH}
|
||||
|
||||
@@ -21,14 +21,36 @@
|
||||
set -o errexit
|
||||
set -o nounset
|
||||
# print command before executing
|
||||
set -o xtrace
|
||||
#set -o xtrace
|
||||
|
||||
CURR_DIR=`pwd`
|
||||
if [[ `basename $CURR_DIR` != "release" ]] ; then
|
||||
echo "You have to call the script from the release/ dir"
|
||||
if [[ `basename $CURR_DIR` != "scripts" ]] ; then
|
||||
echo "You have to call the script from the scripts/ dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
REDIRECT=' > /dev/null 2>&1'
|
||||
if [[ $# -lt 2 ]]; then
|
||||
echo "This script will validate source release candidate published in dist for apache hudi(incubating)"
|
||||
echo "There are two params required:"
|
||||
echo "--release=\${CURRENT_RELEASE_VERSION}"
|
||||
echo "--rc_num=\${RC_NUM}"
|
||||
exit
|
||||
else
|
||||
for param in "$@"
|
||||
do
|
||||
if [[ $param =~ --release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
|
||||
RELEASE_VERSION=${BASH_REMATCH[1]}
|
||||
fi
|
||||
if [[ $param =~ --rc_num\=([0-9]*) ]]; then
|
||||
RC_NUM=${BASH_REMATCH[1]}
|
||||
fi
|
||||
if [[ $param =~ --verbose ]]; then
|
||||
REDIRECT=""
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$(uname)" == "Darwin" ]; then
|
||||
SHASUM="shasum -a 512"
|
||||
else
|
||||
@@ -38,7 +60,7 @@ fi
|
||||
|
||||
# Get to a scratch dir
|
||||
RELEASE_TOOL_DIR=`pwd`
|
||||
WORK_DIR=${RELEASE_TOOL_DIR}/validation_scratch_dir
|
||||
WORK_DIR=/tmp/validation_scratch_dir_001
|
||||
rm -rf $WORK_DIR
|
||||
mkdir $WORK_DIR
|
||||
pushd $WORK_DIR
|
||||
@@ -53,73 +75,75 @@ HUDI_REPO=hudi
|
||||
rm -rf $LOCAL_SVN_DIR
|
||||
mkdir $LOCAL_SVN_DIR
|
||||
cd $LOCAL_SVN_DIR
|
||||
svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO}
|
||||
bash -c "svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO} $REDIRECT"
|
||||
|
||||
cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
|
||||
$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512
|
||||
$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512
|
||||
|
||||
echo "Checking Checksum of Source Release"
|
||||
diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512
|
||||
echo "Checksum Check of Source Release - [OK]"
|
||||
diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512
|
||||
echo -e "\t\tChecksum Check of Source Release - [OK]\n"
|
||||
|
||||
# GPG Check
|
||||
echo "Checking Signature"
|
||||
gpg --import ../KEYS
|
||||
gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz
|
||||
echo "Signature Check - [OK]"
|
||||
bash -c "gpg --import ../KEYS $REDIRECT"
|
||||
bash -c "gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT"
|
||||
echo -e "\t\tSignature Check - [OK]\n"
|
||||
|
||||
# Untar
|
||||
tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz
|
||||
pushd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
|
||||
bash -c "tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT"
|
||||
cd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
|
||||
|
||||
### BEGIN: Binary Files Check
|
||||
echo "Checking for binary files in source release"
|
||||
numBinaryFiles=`find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/' | wc -l`
|
||||
numBinaryFiles=`find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'text/' | wc -l | sed -e s'/ //g'`
|
||||
|
||||
if [ "$numBinaryFiles" > "0" ]; then
|
||||
echo "There were non-text files in source release. Please check below\n"
|
||||
find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/'
|
||||
if [ "$numBinaryFiles" -gt "0" ]; then
|
||||
echo -e "There were non-text files in source release. Please check below\n"
|
||||
find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'text/'
|
||||
exit -1
|
||||
fi
|
||||
|
||||
echo "Binary Files in Source Release Check - [OK]"
|
||||
echo -e "\t\tBinary Files in Source Release Check - [OK]\n"
|
||||
### END: Binary Files Check
|
||||
|
||||
### Checking for DISCLAIMER
|
||||
echo "Checking for DISCLAIMER"
|
||||
disclaimerFile="./DISCLAIMER"
|
||||
if [ ! -f "$disclaimerFile" ]; then
|
||||
echo "DISCLAIMER file missing"
|
||||
exit -1
|
||||
fi
|
||||
echo "DISCLAIMER file exists ? [OK]"
|
||||
echo -e "\t\tDISCLAIMER file exists ? [OK]\n"
|
||||
|
||||
### Checking for LICENSE and NOTICE
|
||||
echo "Checking for LICENSE and NOTICE"
|
||||
licenseFile="./LICENSE"
|
||||
noticeFile="./NOTICE"
|
||||
if [ ! -f "$licenseFile" ]; then
|
||||
echo "License file missing"
|
||||
exit -1
|
||||
fi
|
||||
echo "License file exists ? [OK]"
|
||||
echo -e "\t\tLicense file exists ? [OK]"
|
||||
|
||||
if [ ! -f "$noticeFile" ]; then
|
||||
echo "Notice file missing"
|
||||
exit -1
|
||||
fi
|
||||
echo "Notice file exists ? [OK]"
|
||||
|
||||
### Checking for RAT
|
||||
echo "Running RAT Check Check"
|
||||
mvn rat:check
|
||||
echo "RAT Check Passed [OK]"
|
||||
echo -e "\t\tNotice file exists ? [OK]\n"
|
||||
|
||||
### Licensing Check
|
||||
echo "Performing custom Licensing Check "
|
||||
numfilesWithNoLicense=`find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
|
||||
if [ "$numfilesWithNoLicense" > "0" ]; then
|
||||
numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
|
||||
if [ "$numfilesWithNoLicense" -gt "0" ]; then
|
||||
echo "There were some source files that did not have Apache License"
|
||||
find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
|
||||
find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
|
||||
exit -1
|
||||
fi
|
||||
echo "Licensing Check Passed [OK]"
|
||||
echo -e "\t\tLicensing Check Passed [OK]\n"
|
||||
|
||||
### Checking for RAT
|
||||
echo "Running RAT Check"
|
||||
bash -c "mvn apache-rat:check $REDIRECT"
|
||||
echo -e "\t\tRAT Check Passed [OK]\n"
|
||||
|
||||
popd
|
||||
|
||||
Reference in New Issue
Block a user