Merge pull request #141 from flatcar/kai/monorepo

Adjust release scripts to work with scripts as monorepo
flatcar · Apr 6, 2023 · 79ead6c · 79ead6c
2 parents 34a1687 + bb40300
commit 79ead6c
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 24 deletions.
diff --git a/mirror-repos-branch b/mirror-repos-branch
@@ -41,6 +41,12 @@ trap '{ export EXT="$?"; rm -rf "${REPOS_DIR}" && exit "${EXT}"; }' EXIT
 
 cd "${REPOS_DIR}"
 
+# Check if we are in the monorepo case
+[ ! -d scripts ] && git clone --recurse-submodules "git@github.com:flatcar/scripts"
+if [ "$(git -C scripts show "origin/${SRC_BUILD_BRANCH}":.gitmodules 2>/dev/null)" = "" ]; then
+  FLATCAR_REPOS=("scripts")
+fi
+
 for repo in "${FLATCAR_REPOS[@]}"; do
   # Note: systemd repo has neither flatcar-build-* branches nor flatcar-master
   # branch. So we should skip systemd completely.
@@ -62,7 +68,7 @@ for repo in "${FLATCAR_REPOS[@]}"; do
   fi
 
   echo "Mirroring a branch ${src_branch} to ${DST_BUILD_BRANCH}"
-  git checkout -B "${DST_BUILD_BRANCH}" "origin/${src_branch}"
+  git checkout --recurse-submodules --force -B "${DST_BUILD_BRANCH}" "origin/${src_branch}"
   git push ${PUSH_OPTIONS} origin "${DST_BUILD_BRANCH}"
 
   popd

diff --git a/show-changes b/show-changes
@@ -48,38 +48,68 @@ for section in security bugfixes changes updates; do
   esac
   echo
   for repo in coreos-overlay portage-stable scripts; do
-    if [ "${repo}" = scripts ] && [ ! -e "${repo}" ]; then
-      # Support the previous repo name for old checkouts
-      repo="flatcar-scripts"
-    fi
     OLDREF="${OLD}"
     NEWREF="${NEW}"
-    REPOPATH="${repo}"
-    # For the submodule detection, assume that the "scripts" repo name is ok
-    if [ "${repo}" != "scripts" ] && [ -d "scripts/sdk_container/src/third_party/${repo}" ]; then
+    OLDREPOPATH="${repo}"
+    NEWREPOPATH="${repo}"
+    OLDPREPEND=""
+    NEWPREPEND=""
+    if [ "${repo}" != "scripts" ]; then
       if [ "${FETCH}" = 1 ]; then
         git -C "scripts" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; }
       fi
-      # Find the pinned submodule refs because there may be no release tags inside the submodules
-      # Pipe to awk instead of using --object-only for git 2.35 support
-      OLDREF=$(git -C "scripts" ls-tree "${OLD}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }')
-      NEWREF=$(git -C "scripts" ls-tree "${NEW}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }')
-      REPOPATH="scripts/sdk_container/src/third_party/${repo}"
+      # Check if we are in the monorepo case where we can use the scripts ref or not
+      if [ "$(git -C scripts show "${OLD}":.gitmodules 2>/dev/null)" != "" ]; then
+        # Old version is not a monorepo but has submodules.
+        # Find the pinned submodule refs because there may be no release tags inside the submodules
+        # Pipe to awk instead of using --object-only for git 2.35 support
+        OLDREF=$(git -C "scripts" ls-tree "${OLD}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }')
+        # We can't assume anymore that the submodule repo is available under scripts/
+        if [ ! -d "${repo}" ]; then
+          git clone "git@github.com:flatcar/${repo}.git"
+        fi
+      else
+        OLDPREPEND="sdk_container/src/third_party/${repo}/"
+        OLDREPOPATH="scripts"
+      fi
+      if [ "$(git -C scripts show "${NEW}":.gitmodules 2>/dev/null)" != "" ]; then
+        # New version is not a monorepo but has submodules.
+        NEWREF=$(git -C "scripts" ls-tree "${NEW}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }')
+        if [ ! -d "${repo}" ]; then
+          git clone "git@github.com:flatcar/${repo}.git"
+        fi
+      else
+        NEWPREPEND="sdk_container/src/third_party/${repo}/"
+        NEWREPOPATH="scripts"
+      fi
     fi
     if [ "${FETCH}" = 1 ]; then
-      git -C "${REPOPATH}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; }
+      git -C "${OLDREPOPATH}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; }
+      git -C "${NEWREPOPATH}" fetch -t -f 2> /dev/null > /dev/null || { echo "Error: git fetch -t -f failed" ; exit 1 ; }
     fi
     if [ "${section}" = "security" ] && [ "${repo}" = "coreos-overlay" ]; then
-      FROM_KERNEL=$(git -C "${REPOPATH}" show "${OLDREF}":sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3)
-      TO_KERNEL=$(git -C "${REPOPATH}" show "${NEWREF}":sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3)
+      FROM_KERNEL=$(git -C "${OLDREPOPATH}" show "${OLDREF}":"${OLDPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3)
+      TO_KERNEL=$(git -C "${NEWREPOPATH}" show "${NEWREF}":"${NEWPREPEND}"sys-kernel/coreos-kernel/ | grep -m 1 'coreos-kernel-.*\.ebuild' | cut -d - -f 3 | cut -d . -f 1-3)
       if [ "${FROM_KERNEL}" != "${TO_KERNEL}" ]; then
         KERNEL_ENTRIES=$("${SCRIPTFOLDER}"/show-fixed-kernel-cves.py -f "${FROM_KERNEL}" -t "${TO_KERNEL}")
         if [ "${KERNEL_ENTRIES}" != "" ]; then
           echo "- Linux (${KERNEL_ENTRIES})"
         fi
       fi
     fi
-    git -C "${REPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${NEWREF}" -- "changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; }
+
+    # The assumption is that the old ref is really older, so we can assume that old would have submodules while new doesn't have them anymore
+    if [ "${OLDREPOPATH}" != "${NEWREPOPATH}" ] && [ "${NEWREPOPATH}" = "scripts" ]; then
+      # One patch before the ./checkout helper disappeared we still had submodules
+      LAST_SUBMOD_SCRIPTS_REF="$(git -C scripts rev-list -n 1 "${NEWREF}" -- checkout)~1"
+      LAST_SUBMOD_REF=$(git -C scripts ls-tree "${LAST_SUBMOD_SCRIPTS_REF}" "sdk_container/src/third_party/${repo}" | awk '{print $3 }')
+      # The patch that removed the submodule overrides README has the merge history
+      FIRST_MONO_REF=$(git -C scripts rev-list -n 1 "${NEWREF}" -- sdk_container/git-override/README.md)
+      git -C "${OLDREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${LAST_SUBMOD_REF}" -- "${OLDPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; }
+      git -C "${NEWREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${FIRST_MONO_REF}..${NEWREF}" -- "${NEWPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; }
+    else
+      git -C "${NEWREPOPATH}" difftool --no-prompt --extcmd='sh -c "cat \"$REMOTE\"" --' "${OLDREF}..${NEWREF}" -- "${NEWPREPEND}changelog/${section}/" | sort || { echo "Error: git difftool failed" ; exit 1 ; }
+    fi
     # The -x 'sh -c "cat \"$REMOTE\"" --' command assumes that new changes have their own changelog files,
     # and thus ignores the LOCAL file (which is the empty /dev/null) and prints out the REMOTE completly.
     # If an existing file got changed, we assume that this is just a correction for the old change but

diff --git a/tag-release b/tag-release
@@ -36,6 +36,17 @@ echo "SCRIPTS_REF=$SCRIPTS_REF OVERLAY_REF=$OVERLAY_REF PORTAGE_REF=$PORTAGE_REF
 REPOS="coreos-overlay portage-stable scripts"
 # scripts should be last as it binds the other two together as submodules, if they exist
 
+# Check if we are in the monorepo, in this case only work on "scripts"
+cd "$SCRIPTFOLDER/.."
+if [ ! -d "scripts" ]; then
+  git clone --recurse-submodules 'git@github.com:flatcar/scripts.git'
+fi
+git -C scripts fetch origin
+if [ "$(git -C scripts show "origin/$MAINT-$MAJOR":.gitmodules 2>/dev/null)" = "" ]; then
+  REPOS="scripts"
+fi
+
+
 for REPO in ${REPOS}; do
   echo "Preparing ${REPO}"
   cd "$SCRIPTFOLDER/.."
@@ -52,13 +63,17 @@ for REPO in ${REPOS}; do
   git tag -d "$TAG" || echo "No local tags deleted"
   git push --delete origin "$TAG" || echo "No remote tags deleted"
   # Check if we have to update the submodules while tagging
-  if [ "${REPO}" = "scripts" ] && git show "origin/$MAINT-$MAJOR":sdk_container/src > /dev/null 2>/dev/null ; then
+  if [ "${REPO}" = "scripts" ]; then
     if [ "${REF}" != "origin/$MAINT-$MAJOR" ]; then
       echo "Error: can't find the scripts branch to push the updated submodule to, you can't overwrite SCRIPTS_REF anymore"
       exit 1
     fi
-    echo "Checking out scripts branch $MAINT-$MAJOR to update submodules"
-    git checkout --recurse-submodules -B "$MAINT-$MAJOR" "origin/$MAINT-$MAJOR" || { echo "Error: could not checkout the right branch in your 'scripts' repo" ; exit 1 ; }
+    if [ "$(git status --porcelain)" != "" ]; then
+      echo "Error: uncommitted changes found which will be lost"
+      exit 1
+    fi
+    echo "Checking out scripts branch $MAINT-$MAJOR to update submodules and create version file"
+    git checkout --force --recurse-submodules -B "$MAINT-$MAJOR" "origin/$MAINT-$MAJOR" || { echo "Error: could not checkout the right branch in your 'scripts' repo" ; exit 1 ; }
     git pull --recurse-submodules || { echo "Error: could not pull the branch in your 'scripts' repo" ; exit 1 ; }
     if [ "$(git log HEAD.."origin/$MAINT-$MAJOR")" != "" ] || ! git diff --quiet "origin/$MAINT-$MAJOR" ; then
       echo "Error: local changes in your 'scripts' repo"
@@ -68,14 +83,16 @@ for REPO in ${REPOS}; do
     (
       source sdk_lib/sdk_container_common.sh
       source ci-automation/ci_automation_common.sh
-      cd "sdk_container/src/third_party/coreos-overlay"; git fetch --all --tags --force; cd -
-      cd "sdk_container/src/third_party/portage-stable"; git fetch --all --tags --force; cd -
-      update_submodules "${OVERLAY_REF}" "${PORTAGE_REF}"
+      if [ "$(git show "origin/$MAINT-$MAJOR":.gitmodules 2>/dev/null)" != "" ]; then
+        cd "sdk_container/src/third_party/coreos-overlay"; git fetch --all --tags --force; cd -
+        cd "sdk_container/src/third_party/portage-stable"; git fetch --all --tags --force; cd -
+        update_submodules "${OVERLAY_REF}" "${PORTAGE_REF}"
+      fi
       create_versionfile "${SDK_VERSION}" "${VERSION}"
       SIGN=1 update_and_push_version "${TAG}" true
     )
   else
-    # Tag the other repos or the LTS scripts repo
+    # Tag the other repos
     echo "Tagging ${REF} as ${TAG}"
     git tag -s "${TAG}" -m "${TAG}" "${REF}"
     echo "Pushing tag"