HBASE-27438 Make release scripts can generate CHANGES.md and RELEASEN…

dev-support/create-release/do-release-docker.sh

@@ -209,6 +209,7 @@ SKIP_TAG=$SKIP_TAG
 RUNNING_IN_DOCKER=1
 GIT_BRANCH=$GIT_BRANCH
 NEXT_VERSION=$NEXT_VERSION
+PREV_VERSION=$PREV_VERSION
 RELEASE_VERSION=$RELEASE_VERSION
 RELEASE_TAG=$RELEASE_TAG
 GIT_REF=$GIT_REF

dev-support/create-release/prepend_changes.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import re
+import os
+
+if len(sys.argv) != 3:
+  print("usage: %s <NEW_CHANGES.md> <PREV_CHANGES.md>" % sys.argv[0])
+  exit(1)
+
+pattern = re.compile(r'^## Release .+ - Unreleased .+$')
+with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
+  line = prev_r.readline()
+  while line:
+    if pattern.match(line):
+      break
+    line = prev_r.readline()
+  for newline in new_r:
+    w.writelines(newline)
+  while line:
+    w.writelines(line)
+    line = prev_r.readline()
+os.rename(sys.argv[2] + '.tmp', sys.argv[2])

dev-support/create-release/prepend_releasenotes.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python3
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+import re
+import os
+
+if len(sys.argv) != 3:
+  print("usage: %s <NEW_RELEASENOTES.md> <PREV_RELEASENOTES.md>" % sys.argv[0])
+  exit(1)
+
+pattern = re.compile(r'^# .+ Release Notes$')
+with open(sys.argv[1], 'r', errors = 'ignore') as new_r, open(sys.argv[2], 'r', errors = 'ignore') as prev_r, open(sys.argv[2] + '.tmp', 'w') as w:
+  line = prev_r.readline()
+  while line:
+    if pattern.match(line):
+      break
+    line = prev_r.readline()
+  for newline in new_r:
+    w.writelines(newline)
+  while line:
+    w.writelines(line)
+    line = prev_r.readline()
+os.rename(sys.argv[2] + '.tmp', sys.argv[2])

dev-support/create-release/release-build.sh

@@ -134,7 +134,7 @@ if [[ "$1" == "tag" ]]; then
     jira_fix_version="${PROJECT}-${RELEASE_VERSION}"
   fi
   shopt -u nocasematch
-  update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}"
+  update_releasenotes "$(pwd)/${PROJECT}" "${jira_fix_version}" "${PREV_VERSION}"
 
   cd "${PROJECT}"
 
@@ -145,9 +145,22 @@ if [[ "$1" == "tag" ]]; then
   # Create release version
   maven_set_version "$RELEASE_VERSION"
   find . -name pom.xml -exec git add {} \;
-  git add RELEASENOTES.md CHANGES.md
+  # Always put CHANGES.md and RELEASENOTES.md to parent directory, so later we do not need to
+  # check their position when generating release data. We can not put them under the source code
+  # directory because for 3.x+, CHANGES.md and RELEASENOTES.md are not tracked so later when
+  # generating src release tarball, we will reset the git repo
+  if [[ $(is_tracked "CHANGES.md") == 0 ]]; then
+    git add RELEASENOTES.md CHANGES.md
+    git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
+    cp CHANGES.md ../
+    cp RELEASENOTES.md ../
+  else
+    # CHANGES.md is not tracked, should 3.x+
+    git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG"
+    mv CHANGES.md ../
+    mv RELEASENOTES.md ../
+  fi
 
-  git commit -s -m "Preparing ${PROJECT} release $RELEASE_TAG; tagging and updates to CHANGES.md and RELEASENOTES.md"
   log "Creating tag $RELEASE_TAG at the head of $GIT_BRANCH"
   git tag -s -m "Via create-release" "$RELEASE_TAG"
 
@@ -237,8 +250,9 @@ if [[ "$1" == "publish-dist" ]]; then
 
   log "Copying release tarballs"
   cp "${PROJECT}"-*.tar.* "$svn_target/${DEST_DIR_NAME}/"
-  cp "${PROJECT}/CHANGES.md" "$svn_target/${DEST_DIR_NAME}/"
-  cp "${PROJECT}/RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/"
+  cp "CHANGES.md" "$svn_target/${DEST_DIR_NAME}/"
+  cp "RELEASENOTES.md" "$svn_target/${DEST_DIR_NAME}/"
+
   shopt -s nocasematch
   # Generate api report only if project is hbase for now.
   if [ "${PROJECT}" == "hbase" ]; then

dev-support/create-release/release-util.sh

@@ -37,6 +37,10 @@ fi
 # Maven Profiles for publishing snapshots and release to Maven Central and Dist
 PUBLISH_PROFILES=("-P" "apache-release,release")
 
+# get the current directory, we want to use some python scripts to generate
+# CHANGES.md and RELEASENOTES.md
+SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 function error {
   log "Error: $*" >&2
   exit 1
@@ -183,7 +187,8 @@ function get_release_info {
     local RC_COUNT
     if [ "$REV" != 0 ]; then
       local PREV_REL_REV=$((REV - 1))
-      PREV_REL_TAG="rel/${SHORT_VERSION}.${PREV_REL_REV}"
+      PREV_VERSION=${SHORT_VERSION}.${PREV_REL_REV}
+      PREV_REL_TAG="rel/${PREV_VERSION}"
       if git ls-remote --tags "$ASF_REPO" "$PREV_REL_TAG" | grep -q "refs/tags/${PREV_REL_TAG}$" ; then
         RC_COUNT=0
         REV=$((REV + 1))
@@ -197,13 +202,16 @@ function get_release_info {
     else
       REV=$((REV + 1))
       NEXT_VERSION="${SHORT_VERSION}.${REV}-SNAPSHOT"
+      # not easy to calculate it, just leave it as empty and let users provide it
+      PREV_VERSION=""
       RC_COUNT=0
     fi
   fi
 
   RELEASE_VERSION="$(read_config "RELEASE_VERSION" "$RELEASE_VERSION")"
   NEXT_VERSION="$(read_config "NEXT_VERSION" "$NEXT_VERSION")"
-  export RELEASE_VERSION NEXT_VERSION
+  PREV_VERSION="$(read_config "PREV_VERSION" "$PREV_VERSION")"
+  export RELEASE_VERSION NEXT_VERSION PREV_VERSION
 
   RC_COUNT="$(read_config "RC_COUNT" "$RC_COUNT")"
   if [[ -z "${RELEASE_TAG}" ]]; then
@@ -259,6 +267,7 @@ Release details:
 GIT_BRANCH:      $GIT_BRANCH
 RELEASE_VERSION: $RELEASE_VERSION
 NEXT_VERSION:    $NEXT_VERSION
+PREV_VERSION:    $PREV_VERSION
 RELEASE_TAG:     $RELEASE_TAG $([[ "$GIT_REF" != "$RELEASE_TAG" ]] && printf "\n%s\n" "GIT_REF:         $GIT_REF")
 API_DIFF_TAG:    $API_DIFF_TAG
 ASF_USERNAME:    $ASF_USERNAME
@@ -528,6 +537,7 @@ function get_jira_name {
 function update_releasenotes {
   local project_dir="$1"
   local jira_fix_version="$2"
+  local previous_jira_fix_version="$3"
   local jira_project
   local timing_token
   timing_token="$(start_step)"
@@ -547,11 +557,17 @@ function update_releasenotes {
     sed -i -e \
         "/^## Release ${jira_fix_version}/,/^## Release/ {//!d; /^## Release ${jira_fix_version}/d;}" \
         "${project_dir}/CHANGES.md" || true
+  else
+    # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist
+    curl --location --fail --silent --show-error --output ${project_dir}/CHANGES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/CHANGES.md"
   fi
   if [ -f "${project_dir}/RELEASENOTES.md" ]; then
     sed -i -e \
         "/^# ${jira_project}  ${jira_fix_version} Release Notes/,/^# ${jira_project}/{//!d; /^# ${jira_project}  ${jira_fix_version} Release Notes/d;}" \
         "${project_dir}/RELEASENOTES.md" || true
+  else
+    # should be hbase 3.x, will copy CHANGES.md from archive.a.o/dist
+    curl --location --fail --silent --show-error --output ${project_dir}/RELEASENOTES.md "https://archive.apache.org/dist/hbase/${previous_jira_fix_version}/RELEASENOTES.md"
   fi
 
   # Yetus will not generate CHANGES if no JIRAs fixed against the release version
@@ -566,21 +582,12 @@ function update_releasenotes {
 
   # The releasedocmaker call above generates RELEASENOTES.X.X.X.md and CHANGELOG.X.X.X.md.
   if [ -f "${project_dir}/CHANGES.md" ]; then
-    # To insert into project's CHANGES.md...need to cut the top off the
-    # CHANGELOG.X.X.X.md file removing license and first line and then
-    # insert it after the license comment closing where we have a
-    # DO NOT REMOVE marker text!
-    sed -i -e '/## Release/,$!d' "${changelog}"
-    sed -i -e '2,${/^# HBASE Changelog/d;}' "${project_dir}/CHANGES.md"
-    sed -i -e "/DO NOT REMOVE/r ${changelog}" "${project_dir}/CHANGES.md"
+    $SELF/prepend_changes.py "${changelog}" "${project_dir}/CHANGES.md"
   else
     mv "${changelog}" "${project_dir}/CHANGES.md"
   fi
   if [ -f "${project_dir}/RELEASENOTES.md" ]; then
-    # Similar for RELEASENOTES but slightly different.
-    sed -i -e '/Release Notes/,$!d' "${releasenotes}"
-    sed -i -e '2,${/^# RELEASENOTES/d;}' "${project_dir}/RELEASENOTES.md"
-    sed -i -e "/DO NOT REMOVE/r ${releasenotes}" "${project_dir}/RELEASENOTES.md"
+    $SELF/prepend_releasenotes.py "${releasenotes}" "${project_dir}/RELEASENOTES.md"
   else
     mv "${releasenotes}" "${project_dir}/RELEASENOTES.md"
   fi
@@ -740,3 +747,9 @@ function maven_deploy { #inputs: <snapshot|release> <log_file_path>
 function get_host_os() {
   uname -s | tr '[:lower:]' '[:upper:]'
 }
+
+function is_tracked() {
+  local file=$1
+  git ls-files --error-unmatch "$file" &>/dev/null
+  return $?
+}