From 4aa0a139d2c659a097dd54dbce9cc1e7af9e3412 Mon Sep 17 00:00:00 2001 From: Duo Zhang Date: Sat, 22 Oct 2022 20:47:10 +0800 Subject: [PATCH] HBASE-27438 Make release scripts can generate CHANGES.md and RELEASENOTES.md for 3.x --- .../create-release/do-release-docker.sh | 1 + dev-support/create-release/prepend_changes.py | 39 +++++++++++++++++++ .../create-release/prepend_releasenotes.py | 39 +++++++++++++++++++ dev-support/create-release/release-build.sh | 24 +++++++++--- dev-support/create-release/release-util.sh | 39 ++++++++++++------- 5 files changed, 124 insertions(+), 18 deletions(-) create mode 100755 dev-support/create-release/prepend_changes.py create mode 100755 dev-support/create-release/prepend_releasenotes.py diff --git a/dev-support/create-release/do-release-docker.sh b/dev-support/create-release/do-release-docker.sh index b6874b3365d0..e95429f64910 100755 --- a/dev-support/create-release/do-release-docker.sh +++ b/dev-support/create-release/do-release-docker.sh @@ -209,6 +209,7 @@ SKIP_TAG=$SKIP_TAG RUNNING_IN_DOCKER=1 GIT_BRANCH=$GIT_BRANCH NEXT_VERSION=$NEXT_VERSION +PREV_VERSION=$PREV_VERSION RELEASE_VERSION=$RELEASE_VERSION RELEASE_TAG=$RELEASE_TAG GIT_REF=$GIT_REF diff --git a/dev-support/create-release/prepend_changes.py b/dev-support/create-release/prepend_changes.py new file mode 100755 index 000000000000..7076cb91aed7 --- /dev/null +++ b/dev-support/create-release/prepend_changes.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import re +import os + +if len(sys.argv) != 3: + print("usage: %s " % sys.argv[0]) + exit(1) + +pattern = re.compile(r'^## Release .+ - Unreleased .+$') +with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w: + line = prev_r.readline() + while line: + if pattern.match(line): + break + line = prev_r.readline() + for newline in new_r: + w.writelines(newline) + while line: + w.writelines(line) + line = prev_r.readline() +os.rename(sys.argv[2] + '.tmp', sys.argv[2]) diff --git a/dev-support/create-release/prepend_releasenotes.py b/dev-support/create-release/prepend_releasenotes.py new file mode 100755 index 000000000000..1dac1346d723 --- /dev/null +++ b/dev-support/create-release/prepend_releasenotes.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 +## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import re +import os + +if len(sys.argv) != 3: + print("usage: %s " % sys.argv[0]) + exit(1) + +pattern = re.compile(r'^# .+ Release Notes$') +with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w: + line = prev_r.readline() + while line: + if pattern.match(line): + break + line = prev_r.readline() + for newline in new_r: + w.writelines(newline) + while line: + w.writelines(line) + line = prev_r.readline() +os.rename(sys.argv[2] + '.tmp', sys.argv[2]) diff --git a/dev-support/create-release/release-build.sh b/dev-support/create-release/release-build.sh index 80f386a66c20..d471aa931e72 100755 --- a/dev-support/create-release/release-build.sh +++ b/dev-support/create-release/release-build.sh @@ -134,7 +134,7 @@ if [[ "$1" == "tag" ]]; then jira_fix_version="${PROJECT}-${RELEASE_VERSION}" fi shopt -u nocasematch - update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" + update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" "${PREV_VERSION}" cd "${PROJECT}" @@ -145,9 +145,22 @@ if [[ "$1" == "tag" ]]; then # Create release version maven_set_version "$RELEASE_VERSION" find . -name pom.xml -exec git add {} \; - git add RELEASENOTES.md CHANGES.md + # Always put CHANGES.md and RELEASENOTES.md to parent directory, so later we do not need to + # check their position when generating release data. We can not put them under the source code + # directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when + # generating src release tarball, we will reset the git repo + if [[ $(is_tracked "CHANGES.md") == 0 ]]; then + git add RELEASENOTES.md CHANGES.md + git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" + cp CHANGES.md ../ + cp RELEASENOTES.md ../ + else + # CHANGES.md is not tracked, should 3.x+ + git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG" + mv CHANGES.md ../ + mv RELEASENOTES.md ../ + fi - git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md" log "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH" git tag -s -m "Via create-release" "$RELEASE_TAG" @@ -237,8 +250,9 @@ if [[ "$1" == "publish-dist" ]]; then log "Copying release tarballs" cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/" - cp "${PROJECT}/CHANGES.md" "$svn_target/${DEST_DIR_NAME}/" - cp "${PROJECT}/RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/" + cp "CHANGES.md" "$svn_target/${DEST_DIR_NAME}/" + cp "RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/" + shopt -s nocasematch # Generate api report only if project is hbase for now. if [ "${PROJECT}" == "hbase" ]; then diff --git a/dev-support/create-release/release-util.sh b/dev-support/create-release/release-util.sh index 6d1dcd2ca4ec..6241a8538a08 100755 --- a/dev-support/create-release/release-util.sh +++ b/dev-support/create-release/release-util.sh @@ -37,6 +37,10 @@ fi # Maven Profiles for publishing snapshots and release to Maven Central and Dist PUBLISH_PROFILES=("-P" "apache-release,release") +# get the current directory, we want to use some python scripts to generate +# CHANGES.md and RELEASENOTES.md +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + function error { log "Error: $*" >&2 exit 1 @@ -183,7 +187,8 @@ function get_release_info { local RC_COUNT if [ "$REV" != 0 ]; then local PREV_REL_REV=$((REV - 1)) - PREV_REL_TAG="rel/${SHORT_VERSION}.${PREV_REL_REV}" + PREV_VERSION=${SHORT_VERSION}.${PREV_REL_REV} + PREV_REL_TAG="rel/${PREV_VERSION}" if git ls-remote --tags "$ASF_REPO" "$PREV_REL_TAG" | grep -q "refs/tags/${PREV_REL_TAG}$" ; then RC_COUNT=0 REV=$((REV + 1)) @@ -197,13 +202,16 @@ function get_release_info { else REV=$((REV + 1)) NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT" + # not easy to calculate it, just leave it as empty and let users provide it + PREV_VERSION="" RC_COUNT=0 fi fi RELEASE_VERSION="$(read_config "RELEASE_VERSION" "$RELEASE_VERSION")" NEXT_VERSION="$(read_config "NEXT_VERSION" "$NEXT_VERSION")" - export RELEASE_VERSION NEXT_VERSION + PREV_VERSION="$(read_config "PREV_VERSION" "$PREV_VERSION")" + export RELEASE_VERSION NEXT_VERSION PREV_VERSION RC_COUNT="$(read_config "RC_COUNT" "$RC_COUNT")" if [[ -z "${RELEASE_TAG}" ]]; then @@ -259,6 +267,7 @@ Release details: GIT_BRANCH: $GIT_BRANCH RELEASE_VERSION: $RELEASE_VERSION NEXT_VERSION: $NEXT_VERSION +PREV_VERSION: $PREV_VERSION RELEASE_TAG: $RELEASE_TAG $([[ "$GIT_REF" != "$RELEASE_TAG" ]] && printf "\n%s\n" "GIT_REF: $GIT_REF") API_DIFF_TAG: $API_DIFF_TAG ASF_USERNAME: $ASF_USERNAME @@ -528,6 +537,7 @@ function get_jira_name { function update_releasenotes { local project_dir="$1" local jira_fix_version="$2" + local previous_jira_fix_version="$3" local jira_project local timing_token timing_token="$(start_step)" @@ -547,11 +557,17 @@ function update_releasenotes { sed -i -e \ "/^## Release ${jira_fix_version}/,/^## Release/ {//!d; /^## Release ${jira_fix_version}/d;}" \ "${project_dir}/CHANGES.md" || true + else + # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist + curl --location --fail --silent --show-error --output ${project_dir}/CHANGES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/CHANGES.md" fi if [ -f "${project_dir}/RELEASENOTES.md" ]; then sed -i -e \ "/^# ${jira_project} ${jira_fix_version} Release Notes/,/^# ${jira_project}/{//!d; /^# ${jira_project} ${jira_fix_version} Release Notes/d;}" \ "${project_dir}/RELEASENOTES.md" || true + else + # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist + curl --location --fail --silent --show-error --output ${project_dir}/RELEASENOTES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/RELEASENOTES.md" fi # Yetus will not generate CHANGES if no JIRAs fixed against the release version @@ -566,21 +582,12 @@ function update_releasenotes { # The releasedocmaker call above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md. if [ -f "${project_dir}/CHANGES.md" ]; then - # To insert into project's CHANGES.md...need to cut the top off the - # CHANGELOG.X.X.X.md file removing license and first line and then - # insert it after the license comment closing where we have a - # DO NOT REMOVE marker text! - sed -i -e '/## Release/,$!d' "${changelog}" - sed -i -e '2,${/^# HBASE Changelog/d;}' "${project_dir}/CHANGES.md" - sed -i -e "/DO NOT REMOVE/r ${changelog}" "${project_dir}/CHANGES.md" + $SELF/prepend_changes.py "${changelog}" "${project_dir}/CHANGES.md" else mv "${changelog}" "${project_dir}/CHANGES.md" fi if [ -f "${project_dir}/RELEASENOTES.md" ]; then - # Similar for RELEASENOTES but slightly different. - sed -i -e '/Release Notes/,$!d' "${releasenotes}" - sed -i -e '2,${/^# RELEASENOTES/d;}' "${project_dir}/RELEASENOTES.md" - sed -i -e "/DO NOT REMOVE/r ${releasenotes}" "${project_dir}/RELEASENOTES.md" + $SELF/prepend_releasenotes.py "${releasenotes}" "${project_dir}/RELEASENOTES.md" else mv "${releasenotes}" "${project_dir}/RELEASENOTES.md" fi @@ -740,3 +747,9 @@ function maven_deploy { #inputs: function get_host_os() { uname -s | tr '[:lower:]' '[:upper:]' } + +function is_tracked() { + local file=$1 + git ls-files --error-unmatch "$file" &>/dev/null + return $? +}