Update ansible galaxy roles upload script

docs/release_process.md

@@ -305,6 +305,92 @@ Using `release_helper.py` script:
 ./release_helper.py -c ~/secret.ini -r ComplianceAsCode/content cleanup --branch
 ```
 
+# Update Ansible Roles in Galaxy
+
+After the release is published, the Ansible roles in the RedHatOfficial organization should be
+updated to reflect the latest content.
+
+## Download the Latest Release
+
+- Download the latest released upstream scap-security-guide that contains the built playbooks:
+    - https://github.com/ComplianceAsCode/content/releases
+
+## Switch to the Release Tag
+
+- In your ComplianceAsCode/content repository, switch to the target tag of the release.
+    - For example:
+    ```bash
+    git checkout v0.1.79
+    ```
+
+## Verify the Product Allow List
+
+- Make sure the product allow list in the script contains all the correct RHEL major versions.
+    - If it wasn't updated already in the upstream, update it before proceeding.
+    - Reference: https://github.com/ComplianceAsCode/content/blob/f266b7fdac909a3ce84fdda61355437e00ed761b/utils/ansible_playbook_to_role.py#L65
+
+## Get a GitHub Token
+
+- Get a token from your GitHub account that can access the RedHatOfficial repos and has write permissions (the classic token should be easier to setup):
+    - https://github.com/settings/tokens
+
+## Set Up the Environment
+
+- Switch the environment to use the `ssg` Python environment and install the required dependencies:
+    ```bash
+    source .pyenv.sh
+    pip install "PyGithub>=1.58.2,<2.0"
+    ```
+
+## Run the Ansible Role Update Script
+
+- Run the following command to update the Ansible roles, replacing the token with your GitHub token:
+    ```bash
+    python utils/ansible_playbook_to_role.py --build-playbooks-dir <unzipped built ansible playbooks dir> --token <github_dev_token> --tag-release
+    ```
+
+> **_NOTE:_** It is also possible to use a GitHub user/password combination if the token is not
+provided.
+
+## Refresh Ansible Roles in Ansible Galaxy
+
+After the roles are updated in the [RedHatOfficial](https://github.com/RedHatOfficial/) organization, they need to be refreshed in
+Ansible Galaxy.
+
+### Get the Required Tokens
+
+- Get an Ansible Galaxy token by logging in with your GitHub account:
+    - https://galaxy.ansible.com/ui/token/
+- Get a GitHub token with read access to the [RedHatOfficial](https://github.com/RedHatOfficial/) organization or use the same as in previous steps:
+    - https://github.com/settings/tokens
+
+### Set the Environment Variables
+
+- Set both the `GALAXY_TOKEN` and `GITHUB_TOKEN` environment variables.
+    - For **bash**:
+    ```bash
+    export GALAXY_TOKEN=<your_galaxy_token>
+    export GITHUB_TOKEN=<your_github_token>
+    ```
+    - For **fish**:
+    ```fish
+    set -x GALAXY_TOKEN <your_galaxy_token>
+    set -x GITHUB_TOKEN <your_github_token>
+    ```
+
+### Run the Galaxy Update Script
+
+- Run the following command to refresh the Ansible roles in Ansible Galaxy:
+    ```bash
+    python utils/update_ansible_galaxy_roles.py
+    ```
+
+### Monitor the Import Status
+
+- It usually takes some time until all the roles are refreshed in Galaxy.
+- You can follow the import progress here:
+    - https://galaxy.ansible.com/ui/standalone/imports/
+
 # Announce It!
 
 - Announce the new release on the following communication channels:

utils/ansible_galaxy_meta_template.yml

@@ -16,4 +16,6 @@ galaxy_info:
 
   @GALAXY_TAGS@
 
-dependencies: []
+dependencies:
+  - community.general
+  - ansible.posix

utils/ansible_playbook_to_role.py

@@ -21,7 +21,7 @@
 try:
     from github import Github, InputGitAuthor, UnknownObjectException
 except ImportError:
-    print("Please install PyGithub, on Fedora it's in the python-PyGithub package.",
+    print("Please install PyGithub, you need a specific version of pygithub, install it through $ pip install \"PyGithub>=1.58.2,<2.0\"",
           file=sys.stderr)
     raise SystemExit(1)
 
@@ -31,7 +31,7 @@
     import ssg.yaml
     from ssg.utils import mkdir_p
 except ImportError:
-    print("Unable to find the ssg module. Please run 'source .pyenv'", file=sys.stderr)
+    print("Unable to find the ssg module. Please run 'source .pyenv.sh'", file=sys.stderr)
     raise SystemExit(1)
 
 
@@ -68,30 +68,7 @@ def dict_constructor(loader, node):
     "rhel10",
 ])
 
-PROFILE_ALLOWLIST = set([
-    "anssi_nt28_enhanced",
-    "anssi_nt28_high",
-    "anssi_nt28_intermediary",
-    "anssi_nt28_minimal",
-    "anssi_bp28_enhanced",
-    "anssi_bp28_high",
-    "anssi_bp28_intermediary",
-    "anssi_bp28_minimal",
-    "C2S",
-    "cis",
-    "cjis",
-    "hipaa",
-    "cui",
-    "ospp",
-    "pci-dss",
-    "rht-ccp",
-    "stig",
-    "rhvh-stig",
-    "rhvh-vpp",
-    "e8",
-    "ism_o",
-    "ism_o_secret",
-    "ism_o_top_secret",
+PROFILE_DENYLIST = set([
 ])
 
 
@@ -122,11 +99,14 @@ def create_empty_repositories(github_new_repos, github_org):
 
 
 def clone_and_init_repository(parent_dir, organization, repo):
-    os.system(
-        "git clone git@github.com:%s/%s" % (organization, repo))
-    os.system("ansible-galaxy init " + repo + " --force")
+    # 1. Initialize the Ansible role first (creates the directory)
+    os.system(f"ansible-galaxy init {repo}")
+
+    # 2. Change directory and initialize git
     os.chdir(repo)
     try:
+        os.system("git init --initial-branch=main")
+        os.system(f"git remote add origin git@github.com:{organization}/{repo}")
         os.system('git add .')
         os.system('git commit -a -m "Initial commit" --author "%s <%s>"'
                   % (GIT_COMMIT_AUTHOR_NAME, GIT_COMMIT_AUTHOR_EMAIL))
@@ -453,12 +433,10 @@ def _get_contents(self, path_name, branch='main'):
     def _remote_content(self, filepath):
         # We want the raw string to compare against _local_content
 
-        # New repos use main instead of master
-        branch = 'master'
-        if "rhel9" in self.remote_repo.full_name:
-            branch = 'main'
-        if "rhel10" in self.remote_repo.full_name:
-             branch = 'main'
+        # any version higher than rhel8 should use branch main
+        branch = 'main'
+        if "rhel8" in self.remote_repo.full_name:
+            branch = 'master'
 
         content, sha = self._get_contents(filepath, branch)
         return content, sha
@@ -511,8 +489,8 @@ def parse_args():
              "Defaults to {}.".format(ORGANIZATION_NAME))
     parser.add_argument(
         "--profile", "-p", default=[], action="append",
-        metavar="PROFILE", choices=PROFILE_ALLOWLIST,
-        help="What profiles to upload, if not specified, upload all that are applicable.")
+        metavar="PROFILE", choices=PROFILE_DENYLIST,
+        help="What profiles to prevent uploading, if not specified, upload all that are applicable.")
     parser.add_argument(
         "--product", "-r", default=[], action="append",
         metavar="PRODUCT", choices=PRODUCT_ALLOWLIST,
@@ -538,15 +516,15 @@ def locally_clone_and_init_repositories(organization, repo_list):
         shutil.rmtree(temp_dir)
 
 
-def select_roles_to_upload(product_allowlist, profile_allowlist,
+def select_roles_to_upload(product_allowlist, profile_denylist,
                            build_playbooks_dir):
     selected_roles = dict()
     for filename in sorted(os.listdir(build_playbooks_dir)):
         root, ext = os.path.splitext(filename)
         if ext == ".yml":
             # the format is product-playbook-profile.yml
             product, _, profile = root.split("-", 2)
-            if product in product_allowlist and profile in profile_allowlist:
+            if product in product_allowlist and profile not in profile_denylist:
                 role_name = "ansible-role-%s-%s" % (product, profile)
                 selected_roles[role_name] = (product, profile)
     return selected_roles
@@ -556,20 +534,15 @@ def main():
     args = parse_args()
 
     product_allowlist = set(PRODUCT_ALLOWLIST)
-    profile_allowlist = set(PROFILE_ALLOWLIST)
-
-    potential_roles = {
-        ("ansible-role-%s-%s" % (product, profile))
-        for product in product_allowlist for profile in profile_allowlist
-    }
+    profile_denylist = set(PROFILE_DENYLIST)
 
     if args.product:
         product_allowlist &= set(args.product)
     if args.profile:
-        profile_allowlist &= set(args.profile)
+        profile_denylist &= set(args.profile)
 
     selected_roles = select_roles_to_upload(
-        product_allowlist, profile_allowlist, args.build_playbooks_dir
+        product_allowlist, profile_denylist, args.build_playbooks_dir
     )
 
     if args.dry_run:
@@ -606,7 +579,7 @@ def main():
                 RoleGithubUpdater(repo, playbook_full_path).update_repository()
                 if args.tag_release:
                     update_repo_release(github, repo)
-            elif repo.name not in potential_roles:
+            elif "ansible-role-rhel" in repo.name:
                 print("Repo '%s' is not managed by this script. "
                       "It may need to be deleted, please verify and do that "
                       "manually!" % repo.name, file=sys.stderr)

utils/update_ansible_galaxy_roles.py

@@ -2,50 +2,52 @@
 import subprocess
 import sys
 
+try:
+    from github import Github
+except ImportError:
+    print("Please install PyGithub, you need a specific version of pygithub, "
+          "install it through $ pip install \"PyGithub>=1.58.2,<2.0\"",
+          file=sys.stderr)
+    sys.exit(1)
+
 # --- CONFIGURATION ---
 GITHUB_NAMESPACE = "RedHatOfficial"
 RHEL_ROLE_PREFIX = "ansible-role-rhel"
 ROLE_PREFIX = "ansible-role-"
 
-# The token must be exported in your terminal session before running the script:
-# export GALAXY_TOKEN="your_token_here"
+# The tokens must be exported in your terminal session before running the script:
+# export GALAXY_TOKEN="your_galaxy_token_here"
+# export GITHUB_TOKEN="your_github_token_here"
 GALAXY_TOKEN = os.getenv("GALAXY_TOKEN")
+GITHUB_TOKEN = os.getenv("GITHUB_TOKEN")
 
 
 if not GALAXY_TOKEN:
     print("ERROR: The GALAXY_TOKEN environment variable is not set.")
     print("Please set it (export GALAXY_TOKEN=\"...\") and try again.")
     sys.exit(1)
 
+if not GITHUB_TOKEN:
+    print("ERROR: The GITHUB_TOKEN environment variable is not set.")
+    print("Please set it (export GITHUB_TOKEN=\"...\") and try again.")
+    sys.exit(1)
+
 
 def get_role_repositories():
     """
-    Searches for the list of repositories in the namespace using the GitHub CLI (gh).
+    Retrieves the list of repositories in the namespace using PyGithub.
     """
     print(f"Searching for repositories in '{GITHUB_NAMESPACE}'")
 
-    # Command 'gh search repos' to search for repositories and extract only the name
-    command = [
-        "gh", "search", "repos",
-        f"org:{GITHUB_NAMESPACE}",
-        "--json", "name",
-        "--jq", ".[].name",
-        "--limit", "1000"
-    ]
+    github = Github(GITHUB_TOKEN)
 
     try:
-        # Execute the command and capture the output
-        result = subprocess.run(command, capture_output=True, text=True, check=True)
-        repos = result.stdout.strip().split('\n')
-        return [repo for repo in repos if repo]
-
-    except subprocess.CalledProcessError as e:
-        print("\nERROR running the 'gh' command. Check if GitHub CLI is installed and authenticated.")
-        print(f"Error details: {e.stderr}")
-        sys.exit(1)
-    except FileNotFoundError:
-        print("\nERROR: The 'gh' command (GitHub CLI) was not found.")
-        print("Ensure that GitHub CLI is installed and accessible in your PATH.")
+        github_org = github.get_organization(GITHUB_NAMESPACE)
+        repos = [repo.name for repo in github_org.get_repos()]
+        return repos
+
+    except Exception as e:
+        print(f"\nERROR accessing GitHub API: {e}")
         sys.exit(1)