lanyuanxiaoyao/hudi
1
0
Fork 0
Code Activity

[HUDI-121] Fix bugs in Release Scripts found during RC creation

Browse Source
This commit is contained in:
Balaji Varadarajan
2019-10-01 13:48:05 -07:00
committed by Balaji Varadarajan
parent 1d2e3dcc65
commit e41835fd91
2 changed files with 65 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
scripts/release/cut_release_branch.sh
View File

@@ -35,9 +35,10 @@ function clean_up(){
if [[ $# -eq 1 && $1 = "-h" ]]; then if [[ $# -eq 1 && $1 = "-h" ]]; then
echo "This script will update apache hudi(incubating) master branch with next release version and cut release branch for current development version." echo "This script will update apache hudi(incubating) master branch with next release version and cut release branch for current development version."
echo "There are two params required:" echo "There are 3 params required:"
echo "--release=\${CURRENT_RELEASE_VERSION}" echo "--release=\${CURRENT_RELEASE_VERSION}"
echo "--next_release=\${NEXT_RELEASE_VERSION}" echo "--next_release=\${NEXT_RELEASE_VERSION}"
echo "--rc_num=\${RC_NUM}"
exit exit
else else
for param in "$@" for param in "$@"
@@ -48,9 +49,13 @@ else
if [[ $param =~ --next_release\=([0-9]\.[0-9]*\.[0-9]) ]]; then if [[ $param =~ --next_release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
NEXT_VERSION_IN_BASE_BRANCH=${BASH_REMATCH[1]} NEXT_VERSION_IN_BASE_BRANCH=${BASH_REMATCH[1]}
fi fi
if [[ $param =~ --rc_num\=([0-9]*) ]]; then
RC_NUM=${BASH_REMATCH[1]}
fi
done done
fi fi
if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" ]]; then
if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" || -z "$RC_NUM" ]]; then
echo "This sricpt needs to be ran with params, please run with -h to get more instructions." echo "This sricpt needs to be ran with params, please run with -h to get more instructions."
exit exit
fi fi
@@ -68,42 +73,19 @@ echo "next_release: ${NEXT_VERSION_IN_BASE_BRANCH}"
echo "working master branch: ${MASTER_BRANCH}" echo "working master branch: ${MASTER_BRANCH}"
echo "working release branch: ${RELEASE_BRANCH}" echo "working release branch: ${RELEASE_BRANCH}"
echo "local repo dir: ~/${LOCAL_CLONE_DIR}/${HUDI_ROOT_DIR}" echo "local repo dir: ~/${LOCAL_CLONE_DIR}/${HUDI_ROOT_DIR}"
echo "RC_NUM: $RC_NUM
echo "===============================================================" echo "==============================================================="
cd ~ cd ~
if [[ -d ${LOCAL_CLONE_DIR} ]]; then if [[ -d ${LOCAL_CLONE_DIR} ]]; then
rm -rf ${LOCAL_CLONE_DIR} rm -rf ${LOCAL_CLONE_DIR}
fi fi
mkdir ${LOCAL_CLONE_DIR} mkdir ${LOCAL_CLONE_DIR}
cd ${LOCAL_CLONE_DIR} cd ${LOCAL_CLONE_DIR}
git clone ${GITHUB_REPO_URL} git clone ${GITHUB_REPO_URL}
cd ${HUDI_ROOT_DIR} cd ${HUDI_ROOT_DIR}
# Update Notice.txt
mvn notice:generate
echo "==============Update NOTICE.txt in master branch as following================"
git diff
echo "==============================================================="
echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]"
read confirmation
if [[ $confirmation != "y" ]]; then
echo "Exit without committing any changes on master branch."
clean_up
exit
fi
git commit -am "Updating NOTICE.txt in master" --allow-empty
# Pushing NOTICE.txt changes to master
if git push origin ${MASTER_BRANCH}; then
break
else
clean_up
exit
fi
# Now, create local release branch # Now, create local release branch
git branch ${RELEASE_BRANCH} git branch ${RELEASE_BRANCH}
@@ -139,7 +121,7 @@ fi
# Checkout and update release branch - Add incubating and remove snapshot # Checkout and update release branch - Add incubating and remove snapshot
git checkout ${RELEASE_BRANCH} git checkout ${RELEASE_BRANCH}
mvn versions:set -DnewVersion=${RELEASE}-incubating mvn versions:set -DnewVersion=${RELEASE}-incubating-rc${RC_NUM}
echo "==================Current working branch=======================" echo "==================Current working branch======================="
echo ${RELEASE_BRANCH} echo ${RELEASE_BRANCH}

82
scripts/release/validate_staged_release.sh
View File

@@ -21,14 +21,36 @@
set -o errexit set -o errexit
set -o nounset set -o nounset
# print command before executing # print command before executing
set -o xtrace #set -o xtrace
CURR_DIR=`pwd` CURR_DIR=`pwd`
if [[ `basename $CURR_DIR` != "release" ]] ; then if [[ `basename $CURR_DIR` != "scripts" ]] ; then
echo "You have to call the script from the release/ dir" echo "You have to call the script from the scripts/ dir"
exit 1 exit 1
fi fi
REDIRECT=' > /dev/null 2>&1'
if [[ $# -lt 2 ]]; then
echo "This script will validate source release candidate published in dist for apache hudi(incubating)"
echo "There are two params required:"
echo "--release=\${CURRENT_RELEASE_VERSION}"
echo "--rc_num=\${RC_NUM}"
exit
else
for param in "$@"
do
if [[ $param =~ --release\=([0-9]\.[0-9]*\.[0-9]) ]]; then
RELEASE_VERSION=${BASH_REMATCH[1]}
fi
if [[ $param =~ --rc_num\=([0-9]*) ]]; then
RC_NUM=${BASH_REMATCH[1]}
fi
if [[ $param =~ --verbose ]]; then
REDIRECT=""
fi
done
fi
if [ "$(uname)" == "Darwin" ]; then if [ "$(uname)" == "Darwin" ]; then
SHASUM="shasum -a 512" SHASUM="shasum -a 512"
else else
@@ -38,7 +60,7 @@ fi
# Get to a scratch dir # Get to a scratch dir
RELEASE_TOOL_DIR=`pwd` RELEASE_TOOL_DIR=`pwd`
WORK_DIR=${RELEASE_TOOL_DIR}/validation_scratch_dir WORK_DIR=/tmp/validation_scratch_dir_001
rm -rf $WORK_DIR rm -rf $WORK_DIR
mkdir $WORK_DIR mkdir $WORK_DIR
pushd $WORK_DIR pushd $WORK_DIR
@@ -53,73 +75,75 @@ HUDI_REPO=hudi
rm -rf $LOCAL_SVN_DIR rm -rf $LOCAL_SVN_DIR
mkdir $LOCAL_SVN_DIR mkdir $LOCAL_SVN_DIR
cd $LOCAL_SVN_DIR cd $LOCAL_SVN_DIR
svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO} bash -c "svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO} $REDIRECT"
cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM} cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512 $SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512
echo "Checking Checksum of Source Release" echo "Checking Checksum of Source Release"
diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512 diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512
echo "Checksum Check of Source Release - [OK]" echo -e "\t\tChecksum Check of Source Release - [OK]\n"
# GPG Check # GPG Check
echo "Checking Signature" echo "Checking Signature"
gpg --import ../KEYS bash -c "gpg --import ../KEYS $REDIRECT"
gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz bash -c "gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT"
echo "Signature Check - [OK]" echo -e "\t\tSignature Check - [OK]\n"
# Untar # Untar
tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz bash -c "tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT"
pushd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM} cd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}
### BEGIN: Binary Files Check ### BEGIN: Binary Files Check
echo "Checking for binary files in source release" echo "Checking for binary files in source release"
numBinaryFiles=`find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/' | wc -l` numBinaryFiles=`find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'text/' | wc -l | sed -e s'/ //g'`
if [ "$numBinaryFiles" > "0" ]; then if [ "$numBinaryFiles" -gt "0" ]; then
echo "There were non-text files in source release. Please check below\n" echo -e "There were non-text files in source release. Please check below\n"
find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/' find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'text/'
exit -1 exit -1
fi fi
echo -e "\t\tBinary Files in Source Release Check - [OK]\n"
echo "Binary Files in Source Release Check - [OK]"
### END: Binary Files Check ### END: Binary Files Check
### Checking for DISCLAIMER ### Checking for DISCLAIMER
echo "Checking for DISCLAIMER"
disclaimerFile="./DISCLAIMER" disclaimerFile="./DISCLAIMER"
if [ ! -f "$disclaimerFile" ]; then if [ ! -f "$disclaimerFile" ]; then
echo "DISCLAIMER file missing" echo "DISCLAIMER file missing"
exit -1 exit -1
fi fi
echo "DISCLAIMER file exists ? [OK]" echo -e "\t\tDISCLAIMER file exists ? [OK]\n"
### Checking for LICENSE and NOTICE ### Checking for LICENSE and NOTICE
echo "Checking for LICENSE and NOTICE"
licenseFile="./LICENSE" licenseFile="./LICENSE"
noticeFile="./NOTICE" noticeFile="./NOTICE"
if [ ! -f "$licenseFile" ]; then if [ ! -f "$licenseFile" ]; then
echo "License file missing" echo "License file missing"
exit -1 exit -1
fi fi
echo "License file exists ? [OK]" echo -e "\t\tLicense file exists ? [OK]"
if [ ! -f "$noticeFile" ]; then if [ ! -f "$noticeFile" ]; then
echo "Notice file missing" echo "Notice file missing"
exit -1 exit -1
fi fi
echo "Notice file exists ? [OK]" echo -e "\t\tNotice file exists ? [OK]\n"
### Checking for RAT
echo "Running RAT Check Check"
mvn rat:check
echo "RAT Check Passed [OK]"
### Licensing Check ### Licensing Check
echo "Performing custom Licensing Check " echo "Performing custom Licensing Check "
numfilesWithNoLicense=`find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l`
if [ "$numfilesWithNoLicense" > "0" ]; then if [ "$numfilesWithNoLicense" -gt "0" ]; then
echo "There were some source files that did not have Apache License" echo "There were some source files that did not have Apache License"
find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)"
exit -1 exit -1
fi fi
echo "Licensing Check Passed [OK]" echo -e "\t\tLicensing Check Passed [OK]\n"
### Checking for RAT
echo "Running RAT Check"
bash -c "mvn apache-rat:check $REDIRECT"
echo -e "\t\tRAT Check Passed [OK]\n"
popd popd