Merge pull request #50 from lkubb/more-automation

Fixes #41
Fixes #45

.github/workflows/test-action.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   Linux:
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 45
 
     strategy:
       fail-fast: false
@@ -20,10 +20,10 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
 
-      - name: Set up Python 3.12
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: 3.12
+          python-version: '3.10'
 
       - name: Install Nox
         run: |
@@ -40,77 +40,134 @@ jobs:
         run: |
           nox --force-color -e tests -- -vv tests/
 
-      # - name: Create CodeCov Flags
-      #   if: always()
-      #   id: codecov-flags
-      #   run: |
-      #     echo "flags=$(python -c "import sys; print('{},{},salt_{}'.format('${{ runner.os }}'.replace('-latest', ''), 'py{}{}'.format(*sys.version_info), '_'.join(str(v) for v in '${{ matrix.salt-version }}'.replace('==', '_').split('.'))))")" >> "$GITHUB_OUTPUT"
-
-      # - name: Upload Project Code Coverage
-      #   if: always()
-      #   continue-on-error: true
-      #   shell: bash
-      #   env:
-      #     CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      #     REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},project
-      #     REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-project
-      #     REPORT_PATH: artifacts/coverage-project.xml
-      #   run: |
-      #     if [ ! -f codecov.sh ]; then
-      #       n=0
-      #       until [ "$n" -ge 5 ]
-      #       do
-      #       if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then
-      #           break
-      #       fi
-      #         n=$((n+1))
-      #         sleep 15
-      #       done
-      #     fi
-      #     if [ -f codecov.sh ]; then
-      #       n=0
-      #       until [ "$n" -ge 5 ]
-      #       do
-      #         if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then
-      #             break
-      #         fi
-      #         n=$((n+1))
-      #         sleep 15
-      #       done
-      #     fi
-
-      # - name: Upload Tests Code Coverage
-      #   if: always()
-      #   continue-on-error: true
-      #   shell: bash
-      #   env:
-      #     CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
-      #     REPORT_FLAGS: ${{ steps.codecov-flags.outputs.flags }},tests
-      #     REPORT_NAME: ${{ runner.os }}-Py${{ matrix.python-version }}-Salt${{ matrix.salt-version }}-tests
-      #     REPORT_PATH: artifacts/coverage-tests.xml
-      #   run: |
-      #     if [ ! -f codecov.sh ]; then
-      #       n=0
-      #       until [ "$n" -ge 5 ]
-      #       do
-      #       if curl --max-time 30 -L https://codecov.io/bash --output codecov.sh; then
-      #           break
-      #       fi
-      #         n=$((n+1))
-      #         sleep 15
-      #       done
-      #     fi
-      #     if [ -f codecov.sh ]; then
-      #       n=0
-      #       until [ "$n" -ge 5 ]
-      #       do
-      #         if bash codecov.sh -R $(pwd) -n "${REPORT_NAME}" -f "${REPORT_PATH}" -F "${REPORT_FLAGS}"; then
-      #             break
-      #         fi
-      #         n=$((n+1))
-      #         sleep 15
-      #       done
-      #     fi
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: runtests-${{ runner.os }}.log
+          path: artifacts/runtests-*.log
+
+      - name: Set Exit Status
+        if: always()
+        run: |
+          mkdir exitstatus
+          echo "${{ job.status }}" > exitstatus/${{ github.job }}
+
+      - name: Upload Exit Status
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: exitstatus-${{ github.job }}
+          path: exitstatus
+          if-no-files-found: error
+
+  Windows:
+    runs-on: windows-latest
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # We need the full history for update tests
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Download libeay32.dll
+        run: |
+          PY_LOC="$(which python.exe)"
+          export PY_LOC
+          echo "${PY_LOC}"
+          PY_DIR="$(dirname "${PY_LOC}")"
+          export PY_DIR
+          echo "${PY_DIR}"
+          curl https://repo.saltproject.io/windows/dependencies/64/libeay32.dll --output "${PY_DIR}/libeay32.dll"
+          ls -l "${PY_DIR}"
+        shell: bash
+
+      - name: Install Nox
+        run: |
+          python -m pip install --upgrade pip
+          pip install nox
+
+      - name: Install Test Requirements
+        shell: bash
+        run: |
+          export PATH="/C/Program Files (x86)/Windows Kits/10/bin/10.0.18362.0/x64;$PATH"
+          nox --force-color -e tests --install-only
+
+      - name: Test
+        shell: bash
+        env:
+          SKIP_REQUIREMENTS_INSTALL: true
+        run: |
+          export PATH="/C/Program Files (x86)/Windows Kits/10/bin/10.0.18362.0/x64;$PATH"
+          nox --force-color -e tests -- -vv tests/
+
+      - name: Upload Logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: runtests-${{ runner.os }}.log
+          path: artifacts/runtests-*.log
+
+      - name: Set Exit Status
+        if: always()
+        run: |
+          mkdir exitstatus
+          echo "${{ job.status }}" > exitstatus/${{ github.job }}
+
+      - name: Upload Exit Status
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: exitstatus-${{ github.job }}
+          path: exitstatus
+          if-no-files-found: error
+
+  macOS:
+    runs-on: macOS-latest
+    timeout-minutes: 45
+
+    strategy:
+      fail-fast: false
+      max-parallel: 5
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          # We need the full history for update tests
+          fetch-depth: 0
+          fetch-tags: true
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install Nox
+        run: |
+          python -m pip install --upgrade pip
+          pip install nox
+
+      - name: Install Test Requirements
+        run: |
+          nox --force-color -e tests --install-only
+          brew install enchant
+
+      - name: Test
+        env:
+          SKIP_REQUIREMENTS_INSTALL: true
+        run: |
+          nox --force-color -e tests -- -vv tests/
 
       - name: Upload Logs
         if: always()

.gitignore

@@ -102,6 +102,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+!.envrc
 .env
 .venv
 env/

changelog/+devinit.added.md

@@ -0,0 +1 @@
+Added development environment initialization script to generated projects

changelog/+envrc.added.md

@@ -0,0 +1 @@
+Added a `.envrc` for direnv that runs the new initialization script, ensuring that development environments are present and in sync for all developers

changelog/+win.fixed.md

@@ -0,0 +1 @@
+Added Copier template test runs on macOS and Windows and fixed some template issues on Windows

changelog/41.fixed.md

@@ -0,0 +1 @@
+Fixed unwanted regeneration of deleted boilerplate during updates

changelog/45.added.md

@@ -0,0 +1 @@
+Automated most post-copy/update tasks like repo initialization, dev env setup and pre-commit installation and running

copier.yml

@@ -297,30 +297,39 @@ project_name_full:
 # ========================================
 
 _migrations:
+  # (Re)initialize the project and run pre-commit after updating
+  - >-
+    "{{ copier_python }}"
+    "{{ "{src}{sep}tasks{sep}initialize.py".format(src=_copier_conf.src_path, sep=_copier_conf.sep) }}"
+    migrate
   - version: 0.3.0
-    before:
-      # The minimum Python version was raised from 3.7 to 3.8. If the default answer
-      # is not changed before rendering the template, it fails.
-      - >
-          awk '/^python_requires:/ {sub("3.7", "3.8")}1' {{ _copier_conf.answers_file }} >
-          {{ _copier_conf.answers_file }}.tmp &&
-          mv {{ _copier_conf.answers_file }}.tmp {{ _copier_conf.answers_file }}
-
-# Once we require copier>=9.3.0, we can migrate to the new syntax:
-# _migrations:
-#   - version: 0.3.0
-#     when: '{{ _stage == "before" }}'
-#     command: >-
-#       awk '/^python_requires:/ {sub("3.7", "3.8")}1' {{ _copier_conf.answers_file }} >
-#       {{ _copier_conf.answers_file }}.tmp &&
-#       mv {{ _copier_conf.answers_file }}.tmp {{ _copier_conf.answers_file }}
+    when: '{{ _stage == "before" }}'
+    # The minimum Python version was raised from 3.7 to 3.8. If the default answer
+    # is not changed before rendering the template, it fails.
+    command: >-
+      awk '/^python_requires:/ {sub("3.7", "3.8")}1' {{ _copier_conf.answers_file }} >
+      {{ _copier_conf.answers_file }}.tmp &&
+      mv {{ _copier_conf.answers_file }}.tmp {{ _copier_conf.answers_file }}
+
+_tasks:
+  # Don't run this when updating (updating copies the template twice).
+  # This breaks when recopying.
+  # We should be able to simplify this check in the future to:
+  # if _copier_conf.operation == "copy".
+  - >-
+    {%- if
+      not _copier_conf.answers.last
+      and '.new_copy.' not in _copier_conf.dst_path.name
+      and '.old_copy.' not in _copier_conf.dst_path.name -%}
+        "{{ copier_python }}" "{{ "{src}{sep}tasks{sep}initialize.py".format(src=_copier_conf.src_path, sep=_copier_conf.sep) }}" init
+    {%- endif -%}
 
 # =====================================
 # | Copier settings for this template |
 # =====================================
 
-# We need multiselect functionality
-_min_copier_version: "9.1.0"
+# We need the new migration syntax
+_min_copier_version: "9.3.0"
 
 # The template root is found here
 _subdirectory: project
@@ -359,7 +368,14 @@ _message_after_copy: |
 
   Your Salt extension project "{{ project_name_full }}" has been created successfully! 🎉
 
-  You should continue with the first steps documented here:
+  At this point, your development environment should be ready to go.
+  If you have installed `direnv`, it can be activated automatically,
+  otherwise you need to source the Python virtual environment manually:
+
+    $ cd '{{ _copier_conf.dst_path }}'
+    $ source .venv/bin/activate
+
+  If you encounter any problems, try the manual steps documented here:
 
     ▶ https://salt-extensions.github.io/salt-extension-copier/topics/creation.html#first-steps
 

docs/topics/creation.md

@@ -30,9 +30,35 @@ If hosting the repository outside the organization, you can choose your provider
 
 (first-steps-target)=
 ## First steps
+Before hacking away on your new Salt extension, you need to initialize a Git repository and set up a development environment.
 
-To finalize your project setup, ensure you initialize the Git repository and Python virtual environment and install and run the `pre-commit` hooks.
+Contributors to existing Salt extension projects need to do the latter after cloning.
 
+(automatic-init-target)=
+### Automatic
+:::{versionadded} 0.4.0
+:::
+
+This process is automated completely in the following cases:
+
+* For maintainers: When creating/updating a project via Copier, unless `SKIP_INIT_MIGRATE=1` was set in the environment ([repo initialization](repo-init-target) + [dev env setup](dev-setup-target) + [pre-commit hook installation](hook-install-target) + running pre-commit).
+* For all developers: When `direnv` is installed and the project's `.envrc` is allowed to run ([dev env setup](dev-setup-target) + [pre-commit hook installation](hook-install-target)).
+
+:::{important}
+The automation either requires [`uv`](https://github.com/astral-sh/uv) or the Python version (MAJOR.MINOR) [listed here](https://github.com/saltstack/salt/blob/master/cicd/shared-gh-workflows-context.yml) to be available on your system, at the time of writing Python 3.10.
+:::
+
+:::{hint}
+Without `direnv`, you can still call the automation script manually after entering the project root directory:
+
+```bash
+python3 tools/initialize.py
+source .venv/bin/activate
+```
+:::
+
+### Manual
+(repo-init-target)=
 ### Initialize the repository
 ```bash
 git init -b main
@@ -44,22 +70,27 @@ Some automations assume your default branch is `main`. Ensure this is the case.
 
 (dev-setup-target)=
 ### Initialize the Python virtual environment
+:::{important}
+To create the virtualenv, it is recommended to use the same Python version (MAJOR.MINOR) as the one [listed here](https://github.com/saltstack/salt/blob/master/cicd/shared-gh-workflows-context.yml), at the time of writing Python 3.10.
+:::
+
 ```bash
-python -m venv venv
+python3.10 -m venv .venv
 source venv/bin/activate
 python -m pip install -e '.[tests,dev,docs]'
 ```
 
 This creates a virtual environment and installs relevant dependencies, including `nox` and `pre-commit`.
 
+(hook-install-target)=
 ### Install the `pre-commit` hook
 ```bash
 python -m pre_commit install --install-hooks
 ```
 
 This ensures `pre-commit` runs before each commit. It autoformats and lints your code and ensures the presence of necessary documentation files. To skip these checks temporarily, use `git commit --no-verify`.
 
-### First commit
+## First commit
 ```bash
 git add .
 git commit -m "Initial extension layout"

docs/topics/documenting/building.md

@@ -2,7 +2,7 @@
 # Building documentation
 
 :::{important}
-Ensure `nox` is installed. If you followed the [first steps](first-steps-target), you should be all set.
+Ensure `nox` is installed. If you executed the [first steps](first-steps-target) in some way, you should be all set.
 :::
 
 ## Prerequisites