Updates GHA config from plugin template

[noissue]

.ci/ansible/Containerfile.j2

@@ -1,4 +1,4 @@
-FROM {{ ci_base | default("pulp/pulp-ci-centos:" + pulp_container_tag) }}
+FROM {{ ci_base | default("ghcr.io/pulp/pulp-ci-centos:" + pulp_container_tag) }}
 
 # Add source directories to container
 {% for item in plugins %}

.ci/ansible/settings.py.j2

@@ -5,6 +5,9 @@ PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem"
 PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem"
 TOKEN_SERVER = "{{ pulp_scheme }}://pulp:{{ 443 if pulp_scheme == 'https' else 80 }}/token/"
 TOKEN_SIGNATURE_ALGORITHM = "ES256"
+CACHE_ENABLED = True
+REDIS_HOST = "localhost"
+REDIS_PORT = 6379
 
 {% if pulp_settings %}
 {% for key, value in pulp_settings.items() %}

.ci/ansible/smash-config.json

@@ -5,7 +5,8 @@
       "password"
     ],
     "selinux enabled": false,
-    "version": "3"
+    "version": "3",
+    "aiohttp_fixtures_origin": "172.18.0.1"
   },
   "hosts": [
     {

.ci/scripts/update_github.py

@@ -0,0 +1,20 @@
+# WARNING: DO NOT EDIT!
+#
+# This file was generated by plugin_template, and is managed by it. Please use
+# './plugin-template --github pulp_container' to update this file.
+#
+# For more info visit https://github.com/pulp/plugin_template
+
+import os
+from github import Github
+
+g = Github(os.environ.get("GITHUB_TOKEN"))
+repo = g.get_repo("pulp/pulp_container")
+
+GH_ISSUES = os.environ.get("GH_ISSUES")
+
+for issue in GH_ISSUES.split(","):
+    issue = repo.get_issue(int(issue))
+    if issue.state != "closed":
+        print(f"Closing issue: {issue.number}")
+        issue.edit(state="closed")

.ci/scripts/validate_commit_message.py

@@ -8,41 +8,36 @@
 import re
 import subprocess
 import sys
+import warnings
 from pathlib import Path
 
 
-import requests
+import os
+from github import Github
 
 
 NO_ISSUE = "[noissue]"
 CHANGELOG_EXTS = [".feature", ".bugfix", ".doc", ".removal", ".misc", ".deprecation"]
 
 
-KEYWORDS = ["fixes", "closes", "re", "ref"]
-STATUSES = ["NEW", "ASSIGNED", "POST", "MODIFIED"]
-REDMINE_URL = "https://pulp.plan.io"
+KEYWORDS = ["fixes", "closes"]
 
 sha = sys.argv[1]
-project = "pulp_container"
 message = subprocess.check_output(["git", "log", "--format=%B", "-n 1", sha]).decode("utf-8")
+g = Github(os.environ.get("GITHUB_TOKEN"))
+repo = g.get_repo("pulp/pulp_container")
 
 
 def __check_status(issue):
-    response = requests.get(f"{REDMINE_URL}/issues/{issue}.json")
-    response.raise_for_status()
-    bug_json = response.json()
-    status = bug_json["issue"]["status"]["name"]
-    if status not in STATUSES:
-        sys.exit(
-            "Error: issue #{issue} has invalid status of {status}. Status must be one of "
-            "{statuses}.".format(issue=issue, status=status, statuses=", ".join(STATUSES))
+    gi = repo.get_issue(int(issue))
+    if gi.pull_request:
+        sys.exit(f"Error: issue #{issue} is a pull request.")
+    if gi.closed_at and "cherry picked from commit" not in message:
+        warnings.warn(
+            "When backporting, use the -x flag to append a line that says "
+            "'(cherry picked from commit ...)' to the original commit message."
         )
-
-    if project:
-        project_id = bug_json["issue"]["project"]["id"]
-        project_json = requests.get(f"{REDMINE_URL}/projects/{project_id}.json").json()
-        if project_json["project"]["identifier"] != project:
-            sys.exit(f"Error: issue {issue} is not in the {project} project.")
+        sys.exit(f"Error: issue #{issue} is closed.")
 
 
 def __check_changelog(issue):
@@ -53,6 +48,8 @@ def __check_changelog(issue):
     for match in matches:
         if match.suffix not in CHANGELOG_EXTS:
             sys.exit(f"Invalid extension for changelog entry '{match}'.")
+        if match.suffix == ".feature" and "cherry picked from commit" in message:
+            sys.exit(f"Can not backport '{match}' as it is a feature.")
 
 
 print("Checking commit message for {sha}.".format(sha=sha[0:7]))

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,23 @@
+---
+name: 🐛 Bug report
+about: Create a report to help us improve
+title: ''
+labels: Issue, Triage-Needed
+assignees: ''
+
+---
+
+**Version**
+Please provide the versions of the pulpcore and pulp_container packages in use, and how they are installed.  If you are using Pulp via Katello, please provide the Katello version.
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Additional context**
+Add any other context about the problem here.  Please provide links to any previous discussions via Discourse or Bugzilla.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: ✨ Feature request
+about: Suggest an idea for this project
+title: ''
+labels: Feature, Triage-Needed
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context about the feature request here.

.github/ISSUE_TEMPLATE/task.md

@@ -0,0 +1,10 @@
+---
+name: 🗒️ Task
+about: Documentation, CI/CD, refactors, investigations
+title: ''
+labels: Task, Triage-Needed
+assignees: ''
+
+---
+
+

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-37-gf062376
+2021.08.26-71-g2d3b35c

.github/workflows/ci.yml

@@ -91,6 +91,10 @@ jobs:
           - TEST: docs
           - TEST: azure
           - TEST: s3
+    outputs:
+      deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }}
+      deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }}
+      deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }}
 
     steps:
       - uses: actions/checkout@v2
@@ -165,6 +169,7 @@ jobs:
           GITHUB_REPO_SLUG: ${{ github.repository }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+          REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }}
 
       - name: Setting secrets
         if: github.event_name != 'pull_request'
@@ -186,6 +191,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
 
+      - name: Extract Deprecations from Logs
+        id: deprecations
+        run: echo "::set-output name=deprecations-${{ matrix.env.TEST }}::$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0)"
+
       - name: Logs
         if: always()
         run: |
@@ -209,6 +218,8 @@ jobs:
           - TEST: upgrade
             FROM_PULPCORE_BRANCH: "3.11"
             FROM_PULP_CONTAINER_BRANCH: "2.3"
+    outputs:
+      deprecations-upgrade: ${{ steps.deprecations.outputs.deprecations-upgrade }}
 
     steps:
       - uses: actions/checkout@v2
@@ -286,6 +297,7 @@ jobs:
           GITHUB_REPO_SLUG: ${{ github.repository }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+          REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }}
 
       - name: Setting secrets
 
@@ -307,6 +319,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
 
+      - name: Extract Deprecations from Logs
+        id: deprecations
+        run: echo "::set-output name=deprecations-${{ matrix.env.TEST }}::$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0)"
+
       - name: Logs
         if: always()
         run: |
@@ -318,6 +334,23 @@ jobs:
           docker exec pulp ls -latr /etc/yum.repos.d/ || true
           docker exec pulp cat /etc/yum.repos.d/* || true
           docker exec pulp pip3 list
-
+  deprecations:
+    runs-on: ubuntu-latest
+    if: always()
+    needs: test
+    steps:
+      - name: Fail on deprecations
+        run: |
+          test -z "${{ needs.test.outputs.deprecations-pulp }}"
+          test -z "${{ needs.test.outputs.deprecations-azure }}"
+          test -z "${{ needs.test.outputs.deprecations-s3 }}"
+          test -z "${{ needs.test.outputs.deprecations-upgrade }}"
+      - name: Print deprecations
+        if: failure()
+        run: |
+          echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d
+          echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d
+          echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d
+          echo "${{ needs.test.outputs.deprecations-upgrade }}" | base64 -d
 
 

.github/workflows/nightly.yml

@@ -12,6 +12,7 @@ on:
     # * is a special character in YAML so you have to quote this string
     # runs at 3:00 UTC daily
     - cron: '00 3 * * *'
+  workflow_dispatch:
 
 jobs:
   test:
@@ -96,6 +97,7 @@ jobs:
           GITHUB_REPO_SLUG: ${{ github.repository }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+          REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }}
 
       - name: Setting secrets
 
@@ -140,7 +142,6 @@ jobs:
         with:
           name: ruby-client.tar
           path: ruby-client.tar
-
       - name: Upload built docs
         if: ${{ env.TEST == 'docs' }}
         uses: actions/upload-artifact@v2
@@ -251,6 +252,7 @@ jobs:
           GITHUB_REPO_SLUG: ${{ github.repository }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+          REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }}
 
       - name: Setting secrets
 

.github/workflows/release.yml

@@ -163,6 +163,7 @@ jobs:
           GITHUB_REPO_SLUG: ${{ github.repository }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           GITHUB_CONTEXT: ${{ github.event.pull_request.commits_url }}
+          REDIS_DISABLED: ${{ contains('', matrix.env.TEST) }}
 
       - name: Setting secrets
 
@@ -211,7 +212,6 @@ jobs:
         with:
           name: ruby-client.tar
           path: ruby-client.tar
-
       - name: Upload built docs
         if: ${{ env.TEST == 'docs' }}
         uses: actions/upload-artifact@v2
@@ -319,17 +319,18 @@ jobs:
       - name: Publish client to rubygems
         run: bash .github/workflows/scripts/publish_client_gem.sh
 
-      - name: Update Redmine
-        run: bash .ci/scripts/update_redmine.sh
+
+
+
 
       - name: Create release on GitHub
         run: bash .github/workflows/scripts/create_release_from_tag.sh ${{ github.event.inputs.release }}
 
       - name: Cleanup repository before making changelog PR
         run: rm -rf .lock generation pulp_container_client* *-client.tar pulp_container.tar todo web docs.tar
 
-      - name: Stage changelog for master branch
-        run: python .github/workflows/scripts/stage-changelog-for-master.py ${{ github.event.inputs.release }}
+      - name: Stage changelog for main branch
+        run: python .github/workflows/scripts/stage-changelog-for-default-branch.py ${{ github.event.inputs.release }}
 
       - name: Create Pull Request for Changelog
         uses: peter-evans/create-pull-request@v3

.github/workflows/scripts/before_install.sh

@@ -153,8 +153,6 @@ then
   echo "Failed to install amazon.aws"
   exit $s
 fi
-
-sed -i -e 's/DEBUG = False/DEBUG = True/' pulpcore/pulpcore/app/settings.py
 # Patch DJANGO_ALLOW_ASYNC_UNSAFE out of the pulpcore tasking_system
 # Don't let it fail. Be opportunistic.
 sed -i -e '/DJANGO_ALLOW_ASYNC_UNSAFE/d' pulpcore/pulpcore/tasking/entrypoint.py || true

.github/workflows/scripts/before_script.sh

@@ -34,6 +34,11 @@ if [[ "$TEST" == 'pulp' || "$TEST" == 'performance' || "$TEST" == 'upgrade' || "
   cmd_prefix dnf install -yq lsof which dnf-plugins-core
 fi
 
+if [[ "${REDIS_DISABLED:-false}" == true ]]; then
+  cmd_prefix bash -c "s6-svc -d /var/run/s6/services/redis"
+  echo "The Redis service was disabled for $TEST"
+fi
+
 if [[ -f $POST_BEFORE_SCRIPT ]]; then
   source $POST_BEFORE_SCRIPT
 fi

.github/workflows/scripts/check_commit.sh

@@ -15,6 +15,8 @@ set -euv
 echo ::group::REQUESTS
 pip3 install requests
 
+pip3 install pygithub
+
 echo ::endgroup::
 
 for sha in $(curl -H "Authorization: token $GITHUB_TOKEN" $GITHUB_CONTEXT | jq '.[].sha' | sed 's/"//g')

.github/workflows/scripts/docs-publisher.py

@@ -167,6 +167,27 @@ def main():
             exit_code = subprocess.call(rsync_command, cwd=docs_directory)
             if exit_code != 0:
                 raise RuntimeError("An error occurred while pushing docs.")
+        # publish to docs.pulpproject.org/en/3.y/
+        version_components = branch.split(".")
+        x_y_version = "{}.{}".format(version_components[0], version_components[1])
+        make_directory_with_rsync(["en", x_y_version])
+        remote_path_arg = "%s@%s:%sen/%s/" % (
+            USERNAME,
+            HOSTNAME,
+            SITE_ROOT,
+            x_y_version,
+        )
+        rsync_command = [
+            "rsync",
+            "-avzh",
+            "--delete",
+            "--omit-dir-times",
+            local_path_arg,
+            remote_path_arg,
+        ]
+        exit_code = subprocess.call(rsync_command, cwd=docs_directory)
+        if exit_code != 0:
+            raise RuntimeError("An error occurred while pushing docs.")
         # publish to docs.pulpproject.org/en/3.y.z/
         make_directory_with_rsync(["en", branch])
         remote_path_arg = "%s@%s:%sen/%s/" % (USERNAME, HOSTNAME, SITE_ROOT, branch)

.github/workflows/scripts/publish_docs.sh

@@ -19,7 +19,7 @@ chmod 600 ~/.ssh/pulp-infra
 echo "docs.pulpproject.org,8.43.85.236 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBGXG+8vjSQvnAkq33i0XWgpSrbco3rRqNZr0SfVeiqFI7RN/VznwXMioDDhc+hQtgVhd6TYBOrV07IMcKj+FAzg=" >> /home/runner/.ssh/known_hosts
 chmod 644 /home/runner/.ssh/known_hosts
 
-pip3 install -r doc_requirements.txt
+pip3 install packaging
 
 export PYTHONUNBUFFERED=1
 export DJANGO_SETTINGS_MODULE=pulpcore.app.settings