diff --git a/docs/release_process.md b/docs/release_process.md index 325601da3c1d..a4783bb9ee58 100644 --- a/docs/release_process.md +++ b/docs/release_process.md @@ -305,6 +305,92 @@ Using `release_helper.py` script: ./release_helper.py -c ~/secret.ini -r ComplianceAsCode/content cleanup --branch ``` +# Update Ansible Roles in Galaxy + +After the release is published, the Ansible roles in the RedHatOfficial organization should be +updated to reflect the latest content. + +## Download the Latest Release + +- Download the latest released upstream scap-security-guide that contains the built playbooks: + - https://github.com/ComplianceAsCode/content/releases + +## Switch to the Release Tag + +- In your ComplianceAsCode/content repository, switch to the target tag of the release. + - For example: + ```bash + git checkout v0.1.79 + ``` + +## Verify the Product Allow List + +- Make sure the product allow list in the script contains all the correct RHEL major versions. + - If it wasn't updated already in the upstream, update it before proceeding. + - Reference: https://github.com/ComplianceAsCode/content/blob/f266b7fdac909a3ce84fdda61355437e00ed761b/utils/ansible_playbook_to_role.py#L65 + +## Get a GitHub Token + +- Get a token from your GitHub account that can access the RedHatOfficial repos and has write permissions (the classic token should be easier to setup): + - https://github.com/settings/tokens + +## Set Up the Environment + +- Switch the environment to use the `ssg` Python environment and install the required dependencies: + ```bash + source .pyenv.sh + pip install "PyGithub>=1.58.2,<2.0" + ``` + +## Run the Ansible Role Update Script + +- Run the following command to update the Ansible roles, replacing the token with your GitHub token: + ```bash + python utils/ansible_playbook_to_role.py --build-playbooks-dir --token --tag-release + ``` + +> **_NOTE:_** It is also possible to use a GitHub user/password combination if the token is not +provided. + +## Refresh Ansible Roles in Ansible Galaxy + +After the roles are updated in the [RedHatOfficial](https://github.com/RedHatOfficial/) organization, they need to be refreshed in +Ansible Galaxy. + +### Get the Required Tokens + +- Get an Ansible Galaxy token by logging in with your GitHub account: + - https://galaxy.ansible.com/ui/token/ +- Get a GitHub token with read access to the [RedHatOfficial](https://github.com/RedHatOfficial/) organization or use the same as in previous steps: + - https://github.com/settings/tokens + +### Set the Environment Variables + +- Set both the `GALAXY_TOKEN` and `GITHUB_TOKEN` environment variables. + - For **bash**: + ```bash + export GALAXY_TOKEN= + export GITHUB_TOKEN= + ``` + - For **fish**: + ```fish + set -x GALAXY_TOKEN + set -x GITHUB_TOKEN + ``` + +### Run the Galaxy Update Script + +- Run the following command to refresh the Ansible roles in Ansible Galaxy: + ```bash + python utils/update_ansible_galaxy_roles.py + ``` + +### Monitor the Import Status + +- It usually takes some time until all the roles are refreshed in Galaxy. +- You can follow the import progress here: + - https://galaxy.ansible.com/ui/standalone/imports/ + # Announce It! - Announce the new release on the following communication channels: diff --git a/utils/ansible_galaxy_meta_template.yml b/utils/ansible_galaxy_meta_template.yml index 00d6471c554d..27407e19079d 100644 --- a/utils/ansible_galaxy_meta_template.yml +++ b/utils/ansible_galaxy_meta_template.yml @@ -16,4 +16,6 @@ galaxy_info: @GALAXY_TAGS@ -dependencies: [] +dependencies: + - community.general + - ansible.posix diff --git a/utils/ansible_playbook_to_role.py b/utils/ansible_playbook_to_role.py index 831efebdf0dd..5ac3d9920e34 100755 --- a/utils/ansible_playbook_to_role.py +++ b/utils/ansible_playbook_to_role.py @@ -21,7 +21,7 @@ try: from github import Github, InputGitAuthor, UnknownObjectException except ImportError: - print("Please install PyGithub, on Fedora it's in the python-PyGithub package.", + print("Please install PyGithub, you need a specific version of pygithub, install it through $ pip install \"PyGithub>=1.58.2,<2.0\"", file=sys.stderr) raise SystemExit(1) @@ -31,7 +31,7 @@ import ssg.yaml from ssg.utils import mkdir_p except ImportError: - print("Unable to find the ssg module. Please run 'source .pyenv'", file=sys.stderr) + print("Unable to find the ssg module. Please run 'source .pyenv.sh'", file=sys.stderr) raise SystemExit(1) @@ -68,30 +68,7 @@ def dict_constructor(loader, node): "rhel10", ]) -PROFILE_ALLOWLIST = set([ - "anssi_nt28_enhanced", - "anssi_nt28_high", - "anssi_nt28_intermediary", - "anssi_nt28_minimal", - "anssi_bp28_enhanced", - "anssi_bp28_high", - "anssi_bp28_intermediary", - "anssi_bp28_minimal", - "C2S", - "cis", - "cjis", - "hipaa", - "cui", - "ospp", - "pci-dss", - "rht-ccp", - "stig", - "rhvh-stig", - "rhvh-vpp", - "e8", - "ism_o", - "ism_o_secret", - "ism_o_top_secret", +PROFILE_DENYLIST = set([ ]) @@ -122,11 +99,14 @@ def create_empty_repositories(github_new_repos, github_org): def clone_and_init_repository(parent_dir, organization, repo): - os.system( - "git clone git@github.com:%s/%s" % (organization, repo)) - os.system("ansible-galaxy init " + repo + " --force") + # 1. Initialize the Ansible role first (creates the directory) + os.system(f"ansible-galaxy init {repo}") + + # 2. Change directory and initialize git os.chdir(repo) try: + os.system("git init --initial-branch=main") + os.system(f"git remote add origin git@github.com:{organization}/{repo}") os.system('git add .') os.system('git commit -a -m "Initial commit" --author "%s <%s>"' % (GIT_COMMIT_AUTHOR_NAME, GIT_COMMIT_AUTHOR_EMAIL)) @@ -453,12 +433,10 @@ def _get_contents(self, path_name, branch='main'): def _remote_content(self, filepath): # We want the raw string to compare against _local_content - # New repos use main instead of master - branch = 'master' - if "rhel9" in self.remote_repo.full_name: - branch = 'main' - if "rhel10" in self.remote_repo.full_name: - branch = 'main' + # any version higher than rhel8 should use branch main + branch = 'main' + if "rhel8" in self.remote_repo.full_name: + branch = 'master' content, sha = self._get_contents(filepath, branch) return content, sha @@ -511,8 +489,8 @@ def parse_args(): "Defaults to {}.".format(ORGANIZATION_NAME)) parser.add_argument( "--profile", "-p", default=[], action="append", - metavar="PROFILE", choices=PROFILE_ALLOWLIST, - help="What profiles to upload, if not specified, upload all that are applicable.") + metavar="PROFILE", choices=PROFILE_DENYLIST, + help="What profiles to prevent uploading, if not specified, upload all that are applicable.") parser.add_argument( "--product", "-r", default=[], action="append", metavar="PRODUCT", choices=PRODUCT_ALLOWLIST, @@ -538,7 +516,7 @@ def locally_clone_and_init_repositories(organization, repo_list): shutil.rmtree(temp_dir) -def select_roles_to_upload(product_allowlist, profile_allowlist, +def select_roles_to_upload(product_allowlist, profile_denylist, build_playbooks_dir): selected_roles = dict() for filename in sorted(os.listdir(build_playbooks_dir)): @@ -546,7 +524,7 @@ def select_roles_to_upload(product_allowlist, profile_allowlist, if ext == ".yml": # the format is product-playbook-profile.yml product, _, profile = root.split("-", 2) - if product in product_allowlist and profile in profile_allowlist: + if product in product_allowlist and profile not in profile_denylist: role_name = "ansible-role-%s-%s" % (product, profile) selected_roles[role_name] = (product, profile) return selected_roles @@ -556,20 +534,15 @@ def main(): args = parse_args() product_allowlist = set(PRODUCT_ALLOWLIST) - profile_allowlist = set(PROFILE_ALLOWLIST) - - potential_roles = { - ("ansible-role-%s-%s" % (product, profile)) - for product in product_allowlist for profile in profile_allowlist - } + profile_denylist = set(PROFILE_DENYLIST) if args.product: product_allowlist &= set(args.product) if args.profile: - profile_allowlist &= set(args.profile) + profile_denylist &= set(args.profile) selected_roles = select_roles_to_upload( - product_allowlist, profile_allowlist, args.build_playbooks_dir + product_allowlist, profile_denylist, args.build_playbooks_dir ) if args.dry_run: @@ -606,7 +579,7 @@ def main(): RoleGithubUpdater(repo, playbook_full_path).update_repository() if args.tag_release: update_repo_release(github, repo) - elif repo.name not in potential_roles: + elif "ansible-role-rhel" in repo.name: print("Repo '%s' is not managed by this script. " "It may need to be deleted, please verify and do that " "manually!" % repo.name, file=sys.stderr) diff --git a/utils/update_ansible_galaxy_roles.py b/utils/update_ansible_galaxy_roles.py index 7a90f2c5785b..6b702b5ccd7f 100644 --- a/utils/update_ansible_galaxy_roles.py +++ b/utils/update_ansible_galaxy_roles.py @@ -2,14 +2,24 @@ import subprocess import sys +try: + from github import Github +except ImportError: + print("Please install PyGithub, you need a specific version of pygithub, " + "install it through $ pip install \"PyGithub>=1.58.2,<2.0\"", + file=sys.stderr) + sys.exit(1) + # --- CONFIGURATION --- GITHUB_NAMESPACE = "RedHatOfficial" RHEL_ROLE_PREFIX = "ansible-role-rhel" ROLE_PREFIX = "ansible-role-" -# The token must be exported in your terminal session before running the script: -# export GALAXY_TOKEN="your_token_here" +# The tokens must be exported in your terminal session before running the script: +# export GALAXY_TOKEN="your_galaxy_token_here" +# export GITHUB_TOKEN="your_github_token_here" GALAXY_TOKEN = os.getenv("GALAXY_TOKEN") +GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") if not GALAXY_TOKEN: @@ -17,35 +27,27 @@ print("Please set it (export GALAXY_TOKEN=\"...\") and try again.") sys.exit(1) +if not GITHUB_TOKEN: + print("ERROR: The GITHUB_TOKEN environment variable is not set.") + print("Please set it (export GITHUB_TOKEN=\"...\") and try again.") + sys.exit(1) + def get_role_repositories(): """ - Searches for the list of repositories in the namespace using the GitHub CLI (gh). + Retrieves the list of repositories in the namespace using PyGithub. """ print(f"Searching for repositories in '{GITHUB_NAMESPACE}'") - # Command 'gh search repos' to search for repositories and extract only the name - command = [ - "gh", "search", "repos", - f"org:{GITHUB_NAMESPACE}", - "--json", "name", - "--jq", ".[].name", - "--limit", "1000" - ] + github = Github(GITHUB_TOKEN) try: - # Execute the command and capture the output - result = subprocess.run(command, capture_output=True, text=True, check=True) - repos = result.stdout.strip().split('\n') - return [repo for repo in repos if repo] - - except subprocess.CalledProcessError as e: - print("\nERROR running the 'gh' command. Check if GitHub CLI is installed and authenticated.") - print(f"Error details: {e.stderr}") - sys.exit(1) - except FileNotFoundError: - print("\nERROR: The 'gh' command (GitHub CLI) was not found.") - print("Ensure that GitHub CLI is installed and accessible in your PATH.") + github_org = github.get_organization(GITHUB_NAMESPACE) + repos = [repo.name for repo in github_org.get_repos()] + return repos + + except Exception as e: + print(f"\nERROR accessing GitHub API: {e}") sys.exit(1)