diff --git a/scripts/release/cut_release_branch.sh b/scripts/release/cut_release_branch.sh index 7d506fb56..8e6e92307 100755 --- a/scripts/release/cut_release_branch.sh +++ b/scripts/release/cut_release_branch.sh @@ -35,9 +35,10 @@ function clean_up(){ if [[ $# -eq 1 && $1 = "-h" ]]; then echo "This script will update apache hudi(incubating) master branch with next release version and cut release branch for current development version." - echo "There are two params required:" + echo "There are 3 params required:" echo "--release=\${CURRENT_RELEASE_VERSION}" echo "--next_release=\${NEXT_RELEASE_VERSION}" + echo "--rc_num=\${RC_NUM}" exit else for param in "$@" @@ -48,9 +49,13 @@ else if [[ $param =~ --next_release\=([0-9]\.[0-9]*\.[0-9]) ]]; then NEXT_VERSION_IN_BASE_BRANCH=${BASH_REMATCH[1]} fi + if [[ $param =~ --rc_num\=([0-9]*) ]]; then + RC_NUM=${BASH_REMATCH[1]} + fi done fi -if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" ]]; then + +if [[ -z "$RELEASE" || -z "$NEXT_VERSION_IN_BASE_BRANCH" || -z "$RC_NUM" ]]; then echo "This sricpt needs to be ran with params, please run with -h to get more instructions." exit fi @@ -68,42 +73,19 @@ echo "next_release: ${NEXT_VERSION_IN_BASE_BRANCH}" echo "working master branch: ${MASTER_BRANCH}" echo "working release branch: ${RELEASE_BRANCH}" echo "local repo dir: ~/${LOCAL_CLONE_DIR}/${HUDI_ROOT_DIR}" +echo "RC_NUM: $RC_NUM echo "===============================================================" cd ~ if [[ -d ${LOCAL_CLONE_DIR} ]]; then rm -rf ${LOCAL_CLONE_DIR} fi + mkdir ${LOCAL_CLONE_DIR} cd ${LOCAL_CLONE_DIR} git clone ${GITHUB_REPO_URL} cd ${HUDI_ROOT_DIR} -# Update Notice.txt -mvn notice:generate - -echo "==============Update NOTICE.txt in master branch as following================" -git diff -echo "===============================================================" - -echo "Please make sure all changes above are expected. Do you confirm to commit?: [y|N]" -read confirmation -if [[ $confirmation != "y" ]]; then - echo "Exit without committing any changes on master branch." - clean_up - exit -fi - -git commit -am "Updating NOTICE.txt in master" --allow-empty - -# Pushing NOTICE.txt changes to master -if git push origin ${MASTER_BRANCH}; then - break -else - clean_up - exit -fi - # Now, create local release branch git branch ${RELEASE_BRANCH} @@ -139,7 +121,7 @@ fi # Checkout and update release branch - Add incubating and remove snapshot git checkout ${RELEASE_BRANCH} -mvn versions:set -DnewVersion=${RELEASE}-incubating +mvn versions:set -DnewVersion=${RELEASE}-incubating-rc${RC_NUM} echo "==================Current working branch=======================" echo ${RELEASE_BRANCH} diff --git a/scripts/release/validate_staged_release.sh b/scripts/release/validate_staged_release.sh index 0c9a63e26..62fa598ec 100755 --- a/scripts/release/validate_staged_release.sh +++ b/scripts/release/validate_staged_release.sh @@ -21,14 +21,36 @@ set -o errexit set -o nounset # print command before executing -set -o xtrace +#set -o xtrace CURR_DIR=`pwd` -if [[ `basename $CURR_DIR` != "release" ]] ; then - echo "You have to call the script from the release/ dir" +if [[ `basename $CURR_DIR` != "scripts" ]] ; then + echo "You have to call the script from the scripts/ dir" exit 1 fi +REDIRECT=' > /dev/null 2>&1' +if [[ $# -lt 2 ]]; then + echo "This script will validate source release candidate published in dist for apache hudi(incubating)" + echo "There are two params required:" + echo "--release=\${CURRENT_RELEASE_VERSION}" + echo "--rc_num=\${RC_NUM}" + exit +else + for param in "$@" + do + if [[ $param =~ --release\=([0-9]\.[0-9]*\.[0-9]) ]]; then + RELEASE_VERSION=${BASH_REMATCH[1]} + fi + if [[ $param =~ --rc_num\=([0-9]*) ]]; then + RC_NUM=${BASH_REMATCH[1]} + fi + if [[ $param =~ --verbose ]]; then + REDIRECT="" + fi + done +fi + if [ "$(uname)" == "Darwin" ]; then SHASUM="shasum -a 512" else @@ -38,7 +60,7 @@ fi # Get to a scratch dir RELEASE_TOOL_DIR=`pwd` -WORK_DIR=${RELEASE_TOOL_DIR}/validation_scratch_dir +WORK_DIR=/tmp/validation_scratch_dir_001 rm -rf $WORK_DIR mkdir $WORK_DIR pushd $WORK_DIR @@ -53,73 +75,75 @@ HUDI_REPO=hudi rm -rf $LOCAL_SVN_DIR mkdir $LOCAL_SVN_DIR cd $LOCAL_SVN_DIR -svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO} +bash -c "svn co ${ROOT_SVN_URL}/${DEV_REPO}/${HUDI_REPO} $REDIRECT" cd ${HUDI_REPO}/hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM} -$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512 +$SHASUM hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz > got.sha512 echo "Checking Checksum of Source Release" -diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512 -echo "Checksum Check of Source Release - [OK]" +diff -u hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.sha512 got.sha512 +echo -e "\t\tChecksum Check of Source Release - [OK]\n" # GPG Check echo "Checking Signature" -gpg --import ../KEYS -gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz -echo "Signature Check - [OK]" +bash -c "gpg --import ../KEYS $REDIRECT" +bash -c "gpg --verify hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz.asc hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT" +echo -e "\t\tSignature Check - [OK]\n" # Untar -tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz -pushd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM} +bash -c "tar -zxf hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM}.src.tgz $REDIRECT" +cd hudi-${RELEASE_VERSION}-incubating-rc${RC_NUM} ### BEGIN: Binary Files Check echo "Checking for binary files in source release" -numBinaryFiles=`find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/' | wc -l` +numBinaryFiles=`find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'text/' | wc -l | sed -e s'/ //g'` -if [ "$numBinaryFiles" > "0" ]; then - echo "There were non-text files in source release. Please check below\n" - find . -iname '*' | grep -v '.git' | xargs -I {} file -0 -I {} | grep -va directory | grep -va 'text/' +if [ "$numBinaryFiles" -gt "0" ]; then + echo -e "There were non-text files in source release. Please check below\n" + find . -iname '*' | xargs -I {} file -I {} | grep -va directory | grep -va 'text/' exit -1 fi - -echo "Binary Files in Source Release Check - [OK]" +echo -e "\t\tBinary Files in Source Release Check - [OK]\n" ### END: Binary Files Check ### Checking for DISCLAIMER +echo "Checking for DISCLAIMER" disclaimerFile="./DISCLAIMER" if [ ! -f "$disclaimerFile" ]; then echo "DISCLAIMER file missing" exit -1 fi -echo "DISCLAIMER file exists ? [OK]" +echo -e "\t\tDISCLAIMER file exists ? [OK]\n" ### Checking for LICENSE and NOTICE +echo "Checking for LICENSE and NOTICE" licenseFile="./LICENSE" noticeFile="./NOTICE" if [ ! -f "$licenseFile" ]; then echo "License file missing" exit -1 fi -echo "License file exists ? [OK]" +echo -e "\t\tLicense file exists ? [OK]" if [ ! -f "$noticeFile" ]; then echo "Notice file missing" exit -1 fi -echo "Notice file exists ? [OK]" - -### Checking for RAT -echo "Running RAT Check Check" -mvn rat:check -echo "RAT Check Passed [OK]" +echo -e "\t\tNotice file exists ? [OK]\n" ### Licensing Check echo "Performing custom Licensing Check " -numfilesWithNoLicense=`find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` -if [ "$numfilesWithNoLicense" > "0" ]; then +numfilesWithNoLicense=`find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v DISCLAIMER | grep -v KEYS | grep -v '.mailmap' | grep -v '.sqltemplate' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" | wc -l` +if [ "$numfilesWithNoLicense" -gt "0" ]; then echo "There were some source files that did not have Apache License" - find . -iname '*' | grep -v "\.git" | grep -v NOTICE | grep -v LICENSE | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" + find . -iname '*' -type f | grep -v NOTICE | grep -v LICENSE | grep -v '.json' | grep -v '.data' | grep -v DISCLAIMER | grep -v '.sqltemplate' | grep -v KEYS | grep -v '.mailmap' | xargs grep -L "Licensed to the Apache Software Foundation (ASF)" exit -1 fi -echo "Licensing Check Passed [OK]" +echo -e "\t\tLicensing Check Passed [OK]\n" + +### Checking for RAT +echo "Running RAT Check" +bash -c "mvn apache-rat:check $REDIRECT" +echo -e "\t\tRAT Check Passed [OK]\n" + popd