optimization: workflow 优化 (#121)

.github/workflows/auto_version_release.yml

@@ -0,0 +1,149 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Release version
+
+on:
+  push:
+    branches:
+      - V[0-9]+.[0-9]+.X-rc
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      # 构建环境的Python版本
+      PYTHON_VERSION: "3.7"
+      # yaml中版本的描述路径
+      VERSION_KW_P: "version"
+      # 描述app的yaml文件
+      APP_YAML: "app.yml"
+      # github 提交用户名
+      GITHUB_USERNAME: "github-actions"
+      # 开发日志所在目录
+      DEV_LOG_ROOT: "dev_log"
+      # 发布日志所在目录
+      RELEASE_LOG_ROOT: "release"
+      # tag 名称前缀
+      TAG_NAME_PREFIX: "V"
+      # release 名称前缀
+      RELEASE_NAME_PREFIX: "V"
+      # 开发分支名称后缀
+      DEV_BRANCH_SUFFIX: "-rc"
+      # 开发分支名称前缀
+      DEV_BRANCH_PREFIX: "V"
+
+    steps:
+    - id: checkout
+      name: Checkout
+      uses: actions/checkout@v2
+
+    - id: set-up-python
+      name: Set up Python ${{ env.PYTHON_VERSION }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+
+    - id: install-requirements
+      name: Install requirements
+      run: |
+        pip install PyYAML
+        pip install packaging
+        pip install ruamel.yaml
+
+    - id: generate-dev-log
+      name: Generate dev log
+      run: |
+        # 获取预发布版本
+        prerelease_version=$( python scripts/utils/op_yaml.py -f ${{ env.APP_YAML }} --keyword-path ${{ env.VERSION_KW_P }} --op get )
+        echo "🚀 prerelease_version -> $prerelease_version"
+
+        # 获取发布日志路径并更新发布日志
+        release_log_path=$( python scripts/workflows/release/upgrade_release_log.py -d ${{ env.DEV_LOG_ROOT }} -r ${{ env.RELEASE_LOG_ROOT }} )
+        echo "🌟 release_log_path -> $release_log_path"
+        release_log=$( cat "$release_log_path" )
+        echo "📒 release_log -> 👇👇👇 $release_log"
+
+        # 获取发布日志路径
+        release_log_path=$( echo ${{ env.RELEASE_LOG_ROOT }}/$(ls -a ${{ env.RELEASE_LOG_ROOT }} | grep "$prerelease_version"-) )
+
+        # 推送发布日志
+        git config --global user.email "${{ env.GITHUB_USERNAME }}@users.noreply.github.com"
+        git config --global user.name "${{ env.GITHUB_USERNAME }}"
+        git add .
+        git commit -m "docs: auto generate $prerelease_version release log"
+        git push origin ${{ github.ref }}
+        echo "✨️ main branch -> ${{ github.ref }} has been updated"
+
+        # 设置输出
+        echo "::set-output name=release_log_path::$(echo $release_log_path)"
+        echo "::set-output name=prerelease_version::$(echo $prerelease_version)"
+
+    - id: create-tag
+      name: Create tag
+      run: |
+        # 从上个步骤获取预发布版本号，拼接为标签名称
+        tag_name=$( echo "${{ env.TAG_NAME_PREFIX }}${{ steps.generate-dev-log.outputs.prerelease_version }}" )
+        release_log=$( cat ${{ steps.generate-dev-log.outputs.release_log_path }} )
+        echo "🏷️ tag -> $tag_name will be created"
+
+        # 创建并推送标签
+        # --cleanup=verbatim 修改默认的注释清理模式，保持完整的提交信息，默认的模式会将 # 开头的信息视为注释行
+        # 参考：https://stackoverflow.com/questions/2788092/start-a-git-commit-message-with-a-hashmark
+        # 参考：https://git.kernel.org/pub/scm/git/git.git/plain/Documentation/git-commit.txt
+        git tag -a "$tag_name" -m "$release_log" --cleanup=verbatim
+        git push origin "$tag_name"
+        echo "✨️ tag -> $tag_name has been created"
+
+        # 输出 tag_name
+        echo "::set-output name=tag_name::$(echo $tag_name)"
+
+    - id: create-release
+      name: Create release
+      uses: actions/create-release@latest
+      env:
+        GITHUB_TOKEN: ${{ secrets.PAT }}
+      with:
+        tag_name: ${{ steps.create-tag.outputs.tag_name }}
+        release_name: ${{ env.RELEASE_NAME_PREFIX }}${{ steps.generate-dev-log.outputs.prerelease_version }}
+        body_path: ${{ steps.generate-dev-log.outputs.release_log_path }}
+        draft: false
+        prerelease: true
+
+    - id: start-new-version
+      name: Start new version
+      run: |
+
+        # 版本已发布，此时取出的预发布版本是最新版本
+        latest_version=$( echo "${{ steps.generate-dev-log.outputs.prerelease_version }}" )
+        echo "🔥️ latest_version -> $latest_version"
+
+        next_version=$( python scripts/workflows/release/version_increment.py --version "$latest_version" )
+        echo "⬇️ next_version -> $next_version"
+
+        # 检出新开发分支
+        dev_branch_name=${{ env.DEV_BRANCH_PREFIX }}" )$( echo "${next_version}${{ env.DEV_BRANCH_SUFFIX }}" )
+        echo "🌿 dev_branch_name -> $dev_branch_name"
+        git checkout -b "$dev_branch_name"
+
+        # 开发分支写入预发布版本号
+        python scripts/utils/op_yaml.py -f ${{ env.APP_YAML }} --keyword-path ${{ env.VERSION_KW_P }} --op set --value "$next_version"
+
+        # 创建新开发版本的开发日志目录
+        next_version_dev_log_dir_path=$(echo "${{ env.DEV_LOG_ROOT }}/$next_version" )
+        echo "📖 next_version_dev_log_dir_path -> $next_version_dev_log_dir_path"
+        mkdir -p "$next_version_dev_log_dir_path"
+        touch "$next_version_dev_log_dir_path/.gitkeep"
+
+        # 推送到仓库
+        git add .
+        git commit -m "minor: start new version $next_version"
+        git push origin "$dev_branch_name"
+        echo "✨️ dev_branch -> $dev_branch_name has been created"
+
+    - id: celebrate
+      name: Celebrate
+      run: |
+        echo "🎉 Worth celebrating"
+        echo "🍻 All steps are successfully completed"
+        echo "👋 Goodbye!"

dev_log/2.1.354/crayon_202110301727.yaml

@@ -0,0 +1,3 @@
+---
+optimization:
+  - "workflow 优化 (#121)"

scripts/workflows/release/get_prerelease_version.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+
+
+import getopt
+import os
+import sys
+from typing import Dict, Union
+
+from packaging import version
+
+HELP_TEXT = """
+获取待发布的版本号
+通用参数：
+        [ -h, --help               [可选] "说明文档" ]
+        [ -d, --dev-log-root       [必选] "开发日志目录路径" ]
+"""
+
+
+def extract_params(argv) -> Dict[str, Union[str, bool, int, float]]:
+    try:
+        opts, args = getopt.getopt(argv, "hd:", ["dev-log-root=", "help"])
+    except getopt.GetoptError:
+        print(HELP_TEXT)
+        sys.exit(2)
+
+    sh_params = {"dev-log-root": None}
+    for opt, arg in opts:
+        if opt in ("h", "--help"):
+            print(HELP_TEXT)
+            sys.exit(2)
+
+        elif opt in ("-d", "--dev-log-root"):
+            sh_params["dev-log-root"] = arg
+
+    return sh_params
+
+
+def get_prerelease_version(dev_log_root: str) -> str:
+    version_ordered_list = sorted(os.listdir(dev_log_root), key=lambda v: version.parse(v))
+    version_pending_release = version_ordered_list[-1]
+    return version_pending_release
+
+
+if __name__ == "__main__":
+    params = extract_params(sys.argv[1:])
+    try:
+        print(get_prerelease_version(dev_log_root=params["dev-log-root"]))
+    except FileNotFoundError as err:
+        print(f"dev_log not found, err -> {err}")
+        sys.exit(1)

scripts/workflows/release/upgrade_release_log.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+import datetime
+import getopt
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, List, Union
+
+import yaml
+from get_prerelease_version import get_prerelease_version
+
+HELP_TEXT = """
+生成发布日志
+通用参数：
+        [ -h, --help                  [可选] "说明文档" ]
+        [ -d, --dev-log-root          [必选] "开发日志目录路径" ]
+        [ -r, --release-log-root      [必选] "发布日志路径" ]
+        [ -p, --release-md-path       [可选] "发布日志readme文件路径，默认取 {release-log-root}/release.md" ]
+        [ -v, --prerelease-version    [可选] "预发布版本，默认取dev-log-root最新的记录" ]
+"""
+
+
+def extract_params(argv) -> Dict[str, Union[str, bool, int, float]]:
+    try:
+        opts, args = getopt.getopt(
+            argv, "hd:r:v:p:", ["dev-log-root=", "release-log-root=", "prerelease-version", "release-md-path", "help"]
+        )
+    except getopt.GetoptError:
+        print(HELP_TEXT)
+        sys.exit(2)
+
+    sh_params = {"dev-log-root": None, "release-log-root": None, "prerelease-version": None, "release-md-path": None}
+    for opt, arg in opts:
+        if opt in ("h", "--help"):
+            print(HELP_TEXT)
+            sys.exit(2)
+
+        elif opt in ("-d", "--dev-log-root"):
+            sh_params["dev-log-root"] = arg
+
+        elif opt in ("-r", "--release-log-root"):
+            sh_params["release-log-root"] = arg
+
+        elif opt in ("-v", "--prerelease-version"):
+            sh_params["prerelease-version"] = arg
+
+        elif opt in ("-p", "--release-md-path"):
+            sh_params["release-md-path"] = arg
+    return sh_params
+
+
+if __name__ == "__main__":
+    params = extract_params(sys.argv[1:])
+    dev_log_root = params["dev-log-root"]
+    release_log_root = params["release-log-root"]
+    release_md_path = params["release-md-path"] or os.path.join(release_log_root, "readme.md")
+
+    # 获取等待发布的版本号
+    prerelease_version = params["prerelease-version"] or get_prerelease_version(dev_log_root=dev_log_root)
+    # 具体到某个版本的开发日志
+    dev_yaml_dir_path = os.path.join(dev_log_root, prerelease_version)
+    # 列举归档的yaml文件
+    dev_yaml_file_name_list = os.listdir(dev_yaml_dir_path)
+    # 根据pr类型对开发日志文本进行聚合
+    msgs_group_by_pr_type: Dict[str, List[str]] = defaultdict(list)
+
+    for dev_yaml_file_name in dev_yaml_file_name_list:
+        dev_yaml_file_path = os.path.join(dev_yaml_dir_path, dev_yaml_file_name)
+
+        try:
+            with open(file=dev_yaml_file_path, encoding="utf-8") as dev_yaml_fs:
+                dev_yaml = yaml.safe_load(dev_yaml_fs)
+            for pr_type, user_msgs in dev_yaml.items():
+                msgs_group_by_pr_type[pr_type].extend(user_msgs)
+        except Exception:
+            # 忽略解析错误的开发日志
+            continue
+
+    # 拼接日志
+    release_text = f"\n## {prerelease_version} - {datetime.date.today()} \n"
+    for pr_type, msgs in msgs_group_by_pr_type.items():
+        release_text += f"\n### {pr_type}: \n  * " + "\n  * ".join(msgs)
+
+    # 如果发布日志目录路径不存在，逐层进行创建，并且忽略已创建的层级（exist_ok）
+    if not os.path.exists(release_log_root):
+        os.makedirs(release_log_root, exist_ok=True)
+
+    release_md_title = "# Release\n"
+    if not os.path.exists(release_md_path):
+        os.makedirs(os.path.basename(release_md_path), exist_ok=True)
+        with open(file=release_md_path, mode="w+", encoding="utf-8") as release_md_fs:
+            release_md_fs.write(release_md_title)
+
+    # 标志位，记录是否已经写入
+    is_release_text_write = False
+    with open(file=release_md_path, mode="r", encoding="utf-8") as release_md_fs:
+        # 整体数据量不大，全部读取
+        all_release_text = release_md_fs.read()
+        # 判断日志是否已经写入
+        if release_text in all_release_text:
+            is_release_text_write = True
+
+    if not is_release_text_write:
+        with open(file=release_md_path, mode="w", encoding="utf-8") as release_md_fs:
+            new_all_release_text = (
+                release_md_title + release_text + "\n" + all_release_text.replace(release_md_title, "")
+            )
+            release_md_fs.write(new_all_release_text)
+
+    # 删除该版本的多余发布日志
+    for release_md_path_to_be_deleted in Path(release_log_root).glob(f"V{prerelease_version}*.md"):
+        os.remove(release_md_path_to_be_deleted)
+
+    # 另写一份发布日志到 version-date.md
+    version_release_md_path = os.path.join(release_log_root, f"V{prerelease_version}-{datetime.date.today()}.md")
+    # w -> overwrite
+    with open(file=version_release_md_path, mode="w", encoding="utf-8") as version_release_md_fs:
+        version_release_md_fs.write(release_text + "\n")
+
+    print(version_release_md_path)

scripts/workflows/release/version_increment.py

@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+
+import getopt
+import re
+import sys
+from typing import Dict, Union
+
+HELP_TEXT = """
+版本号递增
+通用参数：
+        [ -h, --help             [可选] "说明文档" ]
+        [ -v, --version          [必选] "开发日志目录路径" ]
+"""
+
+
+def extract_params(argv) -> Dict[str, Union[str, bool, int, float]]:
+    try:
+        opts, args = getopt.getopt(argv, "hv:", ["version=", "help"])
+    except getopt.GetoptError:
+        print(HELP_TEXT)
+        sys.exit(2)
+
+    sh_params = {"version": None}
+    for opt, arg in opts:
+        if opt in ("h", "--help"):
+            print(HELP_TEXT)
+            sys.exit(2)
+
+        elif opt in ("-v", "--version"):
+            sh_params["version"] = arg
+
+    return sh_params
+
+
+VERSION_SEP = "."
+VERSION_PATTERN_OBJ = re.compile(r"\d+\.\d+\.\d+")
+
+
+def version_increment(version_number: str) -> str:
+    if not VERSION_PATTERN_OBJ.match(version_number):
+        raise TypeError(f"version -> {version_number} invalid.")
+
+    major, minor, patch = version_number.split(VERSION_SEP)
+
+    return VERSION_SEP.join([major, minor, str(int(patch) + 1)])
+
+
+if __name__ == "__main__":
+    params = extract_params(sys.argv[1:])
+    try:
+        print(version_increment(version_number=params["version"]))
+    except TypeError as err:
+        print(f"err -> {err}")
+        sys.exit(1)