From 66bd196ff6be7aa88a57939a4d8971de04886e56 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 24 Nov 2021 21:27:07 +0100 Subject: [PATCH 1/9] Split antsibull-build's single subcommand into prepare and rebuild-single. --- roles/build-release/tasks/build.yaml | 87 ++++++++++++++---------- src/antsibull/build_ansible_commands.py | 89 ++++++++++++++----------- src/antsibull/cli/antsibull_build.py | 28 ++++++-- 3 files changed, 122 insertions(+), 82 deletions(-) diff --git a/roles/build-release/tasks/build.yaml b/roles/build-release/tasks/build.yaml index 9b7c5e57..961d2c7e 100644 --- a/roles/build-release/tasks/build.yaml +++ b/roles/build-release/tasks/build.yaml @@ -18,45 +18,58 @@ _feature_freeze: "--feature-frozen" when: antsibull_ansible_version is regex("^\d+.\d+.\d+(b2|b3|rc1)$") -- name: Build a release with new dependencies - command: >- - poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build single {{ antsibull_ansible_version }} - --data-dir {{ antsibull_data_dir }} - --sdist-dir {{ antsibull_sdist_dir }} - --debian - {{ _feature_freeze | default('') }} - # Minimal failure tolerance to galaxy collection download errors - retries: 3 - delay: 5 - register: _build - until: _build.rc == 0 - args: - chdir: "{{ playbook_dir | dirname }}" - creates: "{{ antsibull_data_dir }}/{{ _deps_file }}" +- block: + - name: Create temporary collection cache directory + tempfile: + state: directory + suffix: .antsibull-collection-cache + register: collection_cache -- name: Remove existing release archive if it exists - file: - path: "{{ _release_archive }}" - state: absent - when: antsibull_force_rebuild | bool + - name: Prepare a release with new dependencies + command: >- + poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build prepare {{ antsibull_ansible_version }} + --collection-cache {{ collection_cache.path }} + --data-dir {{ antsibull_data_dir }} + {{ _feature_freeze | default('') }} + # Minimal failure tolerance to galaxy collection download errors + retries: 3 + delay: 5 + register: _build + until: _build.rc == 0 + args: + chdir: "{{ playbook_dir | dirname }}" + creates: "{{ antsibull_data_dir }}/{{ _deps_file }}" -# If the release archive is already there it won't be re-built if we run again -- name: Build a release with existing deps - command: >- - poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build rebuild-single {{ antsibull_ansible_version }} - --data-dir {{ antsibull_data_dir }} - --sdist-dir {{ antsibull_sdist_dir }} - --build-file {{ antsibull_build_file }} - --deps-file {{ _deps_file }} - --debian - # Minimal failure tolerance to galaxy collection download errors - retries: 3 - delay: 5 - register: _rebuild - until: _rebuild.rc == 0 - args: - chdir: "{{ playbook_dir | dirname }}" - creates: "{{ _release_archive }}" + - name: Remove existing release archive if it exists + file: + path: "{{ _release_archive }}" + state: absent + when: antsibull_force_rebuild | bool + + # If the release archive is already there it won't be re-built if we run again + - name: Build a release with existing deps + command: >- + poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build rebuild-single {{ antsibull_ansible_version }} + --collection-cache {{ collection_cache.path }} + --data-dir {{ antsibull_data_dir }} + --sdist-dir {{ antsibull_sdist_dir }} + --build-file {{ antsibull_build_file }} + --deps-file {{ _deps_file }} + --debian + # Minimal failure tolerance to galaxy collection download errors + retries: 3 + delay: 5 + register: _rebuild + until: _rebuild.rc == 0 + args: + chdir: "{{ playbook_dir | dirname }}" + creates: "{{ _release_archive }}" + + finally: + - name: Remove collection cache + file: + path: "{{ collection_cache.path }}" + state: absent # We can use this to test that the release on PyPi matches later (for example) - name: Record the sha256sum for the built tarball diff --git a/src/antsibull/build_ansible_commands.py b/src/antsibull/build_ansible_commands.py index aba439c6..093c74d1 100644 --- a/src/antsibull/build_ansible_commands.py +++ b/src/antsibull/build_ansible_commands.py @@ -207,7 +207,9 @@ def write_build_script(ansible_version: PypiVer, os.chmod(build_ansible_filename, mode=0o755) -def build_single_impl(dependency_data: DependencyFileData, add_release: bool = True) -> None: +def build_single_impl(dependency_data: DependencyFileData, + create_package: bool = True, + add_release: bool = True) -> None: app_ctx = app_context.app_ctx.get() # Determine included collection versions @@ -245,50 +247,59 @@ def build_single_impl(dependency_data: DependencyFileData, add_release: bool = T collection_cache=app_ctx.extra['collection_cache'], ansible_changelog=ansible_changelog) - # Create package and collections directories - package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}') - os.mkdir(package_dir, mode=0o700) - ansible_collections_dir = os.path.join(package_dir, 'ansible_collections') - os.mkdir(ansible_collections_dir, mode=0o700) - - # Write the ansible release info to the collections dir - write_release_py(app_ctx.extra['ansible_version'], ansible_collections_dir) - - # Install collections - # TODO: PY3.8: - # collections_to_install = [p for f in os.listdir(download_dir) - # if os.path.isfile(p := os.path.join(download_dir, f))] - collections_to_install = [] - for collection in os.listdir(download_dir): - path = os.path.join(download_dir, collection) - if os.path.isfile(path): - collections_to_install.append(path) - - asyncio.run(install_together(collections_to_install, ansible_collections_dir)) - - # Compose and write release notes + if create_package: + # Create package and collections directories + package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}') + os.mkdir(package_dir, mode=0o700) + ansible_collections_dir = os.path.join(package_dir, 'ansible_collections') + os.mkdir(ansible_collections_dir, mode=0o700) + + # Write the ansible release info to the collections dir + write_release_py(app_ctx.extra['ansible_version'], ansible_collections_dir) + + # Install collections + # TODO: PY3.8: + # collections_to_install = [p for f in os.listdir(download_dir) + # if os.path.isfile(p := os.path.join(download_dir, f))] + collections_to_install = [] + for collection in os.listdir(download_dir): + path = os.path.join(download_dir, collection) + if os.path.isfile(path): + collections_to_install.append(path) + + asyncio.run(install_together(collections_to_install, ansible_collections_dir)) + + # Compose and write release notes to destination directory release_notes = ReleaseNotes.build(changelog) - release_notes.write_changelog_to(package_dir) - release_notes.write_porting_guide_to(package_dir) - - # Write build scripts and files - write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir) - write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version, '', - package_dir, release_notes, app_ctx.extra['debian']) - if app_ctx.extra['debian']: - write_debian_directory(app_ctx.extra['ansible_version'], ansible_base_version, - package_dir) - make_dist(package_dir, app_ctx.extra['sdist_dir']) - - # Write changelog and porting guide also to destination directory - release_notes.write_changelog_to(app_ctx.extra['dest_data_dir']) - release_notes.write_porting_guide_to(app_ctx.extra['dest_data_dir']) + # Write changelog and porting guide also to destination directory + release_notes.write_changelog_to(app_ctx.extra['dest_data_dir']) + release_notes.write_porting_guide_to(app_ctx.extra['dest_data_dir']) + + if create_package: + # Write them also to package directory if we're creating the package + release_notes.write_changelog_to(package_dir) + release_notes.write_porting_guide_to(package_dir) + + # Write build scripts and files + write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir) + write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version, '', + package_dir, release_notes, app_ctx.extra['debian']) + if app_ctx.extra['debian']: + write_debian_directory(app_ctx.extra['ansible_version'], ansible_base_version, + package_dir) + make_dist(package_dir, app_ctx.extra['sdist_dir']) if add_release: ansible_changelog.changes.save() def build_single_command() -> int: + # This is deprecated; in the future users will have to first run prepare, and then + # rebuild-single. + return prepare_command(True) + + +def prepare_command(create_package: bool = False) -> int: app_ctx = app_context.app_ctx.get() build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file']) @@ -337,7 +348,7 @@ def build_single_command() -> int: str(ansible_base_version), {collection: str(version) for collection, version in included_versions.items()}) - build_single_impl(dependency_data) + build_single_impl(dependency_data, create_package=create_package) deps_filename = os.path.join(app_ctx.extra['dest_data_dir'], app_ctx.extra['deps_file']) deps_file = DepsFile(deps_filename) diff --git a/src/antsibull/cli/antsibull_build.py b/src/antsibull/cli/antsibull_build.py index e9bce1e2..d9fed0cf 100644 --- a/src/antsibull/cli/antsibull_build.py +++ b/src/antsibull/cli/antsibull_build.py @@ -24,7 +24,7 @@ ) from ..build_collection import build_collection_command # noqa: E402 from ..build_ansible_commands import ( # noqa: E402 - build_single_command, build_multiple_command, rebuild_single_command, + prepare_command, build_single_command, build_multiple_command, rebuild_single_command, ) from ..build_changelog import build_changelog # noqa: E402 from ..config import load_config # noqa: E402 @@ -38,6 +38,7 @@ DEFAULT_PIECES_FILE = f'{DEFAULT_FILE_BASE}.in' ARGS_MAP = {'new-ansible': new_ansible_command, + 'prepare': prepare_command, 'single': build_single_command, 'multiple': build_multiple_command, 'collection': build_collection_command, @@ -58,7 +59,7 @@ def _normalize_build_options(args: argparse.Namespace) -> None: def _normalize_build_write_data_options(args: argparse.Namespace) -> None: if args.command not in ( - 'new-ansible', 'single', 'rebuild-single', 'multiple', 'changelog', + 'new-ansible', 'prepare', 'single', 'rebuild-single', 'multiple', 'changelog', 'new-acd', 'build-single', 'build-multiple'): return @@ -103,15 +104,23 @@ def _normalize_new_release_options(args: argparse.Namespace) -> None: def _normalize_release_build_options(args: argparse.Namespace) -> None: flog = mlog.fields(func='_normalize_release_build_options') + if args.command == 'single': + flog.warning( + 'The single command is deprecated. Use `prepare` followed by `rebuild-single`' + ' instead.') + args.command = 'single' + if args.command == 'build-single': - flog.warning('The build-single command is deprecated. Use `single` instead.') + flog.warning( + 'The build-single command is deprecated. Use `prepare` followed by' + ' `rebuild-single` instead.') args.command = 'single' if args.command == 'build-multiple': flog.warning('The build-multiple command is deprecated. Use `multiple` instead.') args.command = 'multiple' - if args.command not in ('single', 'multiple', 'rebuild-single'): + if args.command not in ('prepare', 'single', 'multiple', 'rebuild-single'): return compat_version_part = ( @@ -136,7 +145,7 @@ def _normalize_release_build_options(args: argparse.Namespace) -> None: args.deps_file = f'{basename}-{args.ansible_version}.deps' - if args.command in ('single', 'multiple'): + if args.command in ('single', 'multiple', 'rebuild-single'): if not os.path.isdir(args.sdist_dir): raise InvalidArgumentError(f'{args.sdist_dir} must be an existing directory') @@ -234,10 +243,17 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: ' relative to --dest-data-dir. The default is' ' $BASENAME_OF_PIECES_FILE-X.Y.build') + prepare_parser = subparsers.add_parser('prepare', + parents=[build_write_data_parser, cache_parser, + build_step_parser, feature_freeze_parser], + description='Collect dependencies for an Ansible' + ' release') + build_single_parser = subparsers.add_parser('single', parents=[build_write_data_parser, cache_parser, build_step_parser, feature_freeze_parser], - description='Build a single-file Ansible') + description='Build a single-file Ansible' + ' [deprecated]') build_single_parser.add_argument('--sdist-dir', default='.', help='Directory to write the generated sdist tarball to') build_single_parser.add_argument('--debian', action='store_true', From 66c939d23f31489e7cc6e4e3ceb36559a262db25 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 24 Nov 2021 22:48:25 +0100 Subject: [PATCH 2/9] Reduce complexity. --- src/antsibull/cli/antsibull_build.py | 67 +++++++++++++--------------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/src/antsibull/cli/antsibull_build.py b/src/antsibull/cli/antsibull_build.py index d9fed0cf..9c55535d 100644 --- a/src/antsibull/cli/antsibull_build.py +++ b/src/antsibull/cli/antsibull_build.py @@ -52,6 +52,33 @@ } +def _normalize_commands(args: argparse.Namespace) -> None: + flog = mlog.fields(func='_normalize_commands') + + if args.command == 'new-acd': + flog.warning('The new-acd command is deprecated. Use `new-ansible` instead.') + args.command = 'new-ansible' + + if args.command == 'single': + flog.warning( + 'The single command is deprecated. Use `prepare` followed by `rebuild-single`' + ' instead.') + + if args.command == 'build-single': + flog.warning( + 'The build-single command is deprecated. Use `prepare` followed by' + ' `rebuild-single` instead.') + args.command = 'single' + + if args.command == 'build-multiple': + flog.warning('The build-multiple command is deprecated. Use `multiple` instead.') + args.command = 'multiple' + + if args.command == 'build-collection': + flog.warning('The build-collection command is deprecated. Use `collection` instead.') + args.command = 'collection' + + def _normalize_build_options(args: argparse.Namespace) -> None: if not os.path.isdir(args.data_dir): raise InvalidArgumentError(f'{args.data_dir} must be an existing directory') @@ -71,12 +98,6 @@ def _normalize_build_write_data_options(args: argparse.Namespace) -> None: def _normalize_new_release_options(args: argparse.Namespace) -> None: - flog = mlog.fields(func='_normalize_new_release_options') - - if args.command == 'new-acd': - flog.warning('The new-acd command is deprecated. Use `new-ansible` instead.') - args.command = 'new-ansible' - if args.command != 'new-ansible': return @@ -102,24 +123,6 @@ def _normalize_new_release_options(args: argparse.Namespace) -> None: def _normalize_release_build_options(args: argparse.Namespace) -> None: - flog = mlog.fields(func='_normalize_release_build_options') - - if args.command == 'single': - flog.warning( - 'The single command is deprecated. Use `prepare` followed by `rebuild-single`' - ' instead.') - args.command = 'single' - - if args.command == 'build-single': - flog.warning( - 'The build-single command is deprecated. Use `prepare` followed by' - ' `rebuild-single` instead.') - args.command = 'single' - - if args.command == 'build-multiple': - flog.warning('The build-multiple command is deprecated. Use `multiple` instead.') - args.command = 'multiple' - if args.command not in ('prepare', 'single', 'multiple', 'rebuild-single'): return @@ -160,12 +163,6 @@ def _normalize_release_rebuild_options(args: argparse.Namespace) -> None: def _normalize_collection_build_options(args: argparse.Namespace) -> None: - flog = mlog.fields(func='_normalize_collection_build_options') - - if args.command == 'build-collection': - flog.warning('The build-collection command is deprecated. Use `collection` instead.') - args.command = 'collection' - if args.command != 'collection': return @@ -243,11 +240,10 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: ' relative to --dest-data-dir. The default is' ' $BASENAME_OF_PIECES_FILE-X.Y.build') - prepare_parser = subparsers.add_parser('prepare', - parents=[build_write_data_parser, cache_parser, - build_step_parser, feature_freeze_parser], - description='Collect dependencies for an Ansible' - ' release') + subparsers.add_parser('prepare', + parents=[build_write_data_parser, cache_parser, + build_step_parser, feature_freeze_parser], + description='Collect dependencies for an Ansible release') build_single_parser = subparsers.add_parser('single', parents=[build_write_data_parser, cache_parser, @@ -305,6 +301,7 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: # Validation and coercion normalize_toplevel_options(args) + _normalize_commands(args) _normalize_build_options(args) _normalize_build_write_data_options(args) _normalize_new_release_options(args) From 4b66e449b931d3014d15b54aafd49a10147678e4 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Wed, 24 Nov 2021 22:58:14 +0100 Subject: [PATCH 3/9] Prepare no longer downloads collections, and thus also doesn't build changelog and porting guide RSTs. --- roles/build-release/tasks/build.yaml | 85 +++++------- src/antsibull/build_ansible_commands.py | 174 ++++++++++++------------ src/antsibull/cli/antsibull_build.py | 5 +- 3 files changed, 122 insertions(+), 142 deletions(-) diff --git a/roles/build-release/tasks/build.yaml b/roles/build-release/tasks/build.yaml index 961d2c7e..b96c1392 100644 --- a/roles/build-release/tasks/build.yaml +++ b/roles/build-release/tasks/build.yaml @@ -18,58 +18,43 @@ _feature_freeze: "--feature-frozen" when: antsibull_ansible_version is regex("^\d+.\d+.\d+(b2|b3|rc1)$") -- block: - - name: Create temporary collection cache directory - tempfile: - state: directory - suffix: .antsibull-collection-cache - register: collection_cache - - - name: Prepare a release with new dependencies - command: >- - poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build prepare {{ antsibull_ansible_version }} - --collection-cache {{ collection_cache.path }} - --data-dir {{ antsibull_data_dir }} - {{ _feature_freeze | default('') }} - # Minimal failure tolerance to galaxy collection download errors - retries: 3 - delay: 5 - register: _build - until: _build.rc == 0 - args: - chdir: "{{ playbook_dir | dirname }}" - creates: "{{ antsibull_data_dir }}/{{ _deps_file }}" - - - name: Remove existing release archive if it exists - file: - path: "{{ _release_archive }}" - state: absent - when: antsibull_force_rebuild | bool +- name: Prepare a release with new dependencies + command: >- + poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build prepare {{ antsibull_ansible_version }} + --data-dir {{ antsibull_data_dir }} + {{ _feature_freeze | default('') }} + # Minimal failure tolerance to galaxy collection download errors + retries: 3 + delay: 5 + register: _build + until: _build.rc == 0 + args: + chdir: "{{ playbook_dir | dirname }}" + creates: "{{ antsibull_data_dir }}/{{ _deps_file }}" - # If the release archive is already there it won't be re-built if we run again - - name: Build a release with existing deps - command: >- - poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build rebuild-single {{ antsibull_ansible_version }} - --collection-cache {{ collection_cache.path }} - --data-dir {{ antsibull_data_dir }} - --sdist-dir {{ antsibull_sdist_dir }} - --build-file {{ antsibull_build_file }} - --deps-file {{ _deps_file }} - --debian - # Minimal failure tolerance to galaxy collection download errors - retries: 3 - delay: 5 - register: _rebuild - until: _rebuild.rc == 0 - args: - chdir: "{{ playbook_dir | dirname }}" - creates: "{{ _release_archive }}" +- name: Remove existing release archive if it exists + file: + path: "{{ _release_archive }}" + state: absent + when: antsibull_force_rebuild | bool - finally: - - name: Remove collection cache - file: - path: "{{ collection_cache.path }}" - state: absent +# If the release archive is already there it won't be re-built if we run again +- name: Build a release with existing deps + command: >- + poetry run coverage run -p --source antsibull -m antsibull.cli.antsibull_build rebuild-single {{ antsibull_ansible_version }} + --data-dir {{ antsibull_data_dir }} + --sdist-dir {{ antsibull_sdist_dir }} + --build-file {{ antsibull_build_file }} + --deps-file {{ _deps_file }} + --debian + # Minimal failure tolerance to galaxy collection download errors + retries: 3 + delay: 5 + register: _rebuild + until: _rebuild.rc == 0 + args: + chdir: "{{ playbook_dir | dirname }}" + creates: "{{ _release_archive }}" # We can use this to test that the release on PyPi matches later (for example) - name: Record the sha256sum for the built tarball diff --git a/src/antsibull/build_ansible_commands.py b/src/antsibull/build_ansible_commands.py index 093c74d1..23739ba6 100644 --- a/src/antsibull/build_ansible_commands.py +++ b/src/antsibull/build_ansible_commands.py @@ -207,99 +207,17 @@ def write_build_script(ansible_version: PypiVer, os.chmod(build_ansible_filename, mode=0o755) -def build_single_impl(dependency_data: DependencyFileData, - create_package: bool = True, - add_release: bool = True) -> None: - app_ctx = app_context.app_ctx.get() - - # Determine included collection versions - ansible_base_version = PypiVer(dependency_data.ansible_base_version) - included_versions = { - collection: SemVer(version) - for collection, version in dependency_data.deps.items() - } - - with tempfile.TemporaryDirectory() as tmp_dir: - download_dir = os.path.join(tmp_dir, 'collections') - os.mkdir(download_dir, mode=0o700) - - # Download included collections - asyncio.run(download_collections(included_versions, app_ctx.galaxy_url, - download_dir, app_ctx.extra['collection_cache'])) - - # Get Ansible changelog, add new release - ansible_changelog = ChangelogData.ansible( - app_ctx.extra['data_dir'], app_ctx.extra['dest_data_dir']) - if add_release: - date = datetime.date.today() - ansible_changelog.add_ansible_release( - str(app_ctx.extra['ansible_version']), - date, - f'Release Date: {date}' - f'\n\n' - f'`Porting Guide `_') - - # Get changelog and porting guide data - changelog = get_changelog( - app_ctx.extra['ansible_version'], - deps_dir=app_ctx.extra['data_dir'], - deps_data=[dependency_data], - collection_cache=app_ctx.extra['collection_cache'], - ansible_changelog=ansible_changelog) - - if create_package: - # Create package and collections directories - package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}') - os.mkdir(package_dir, mode=0o700) - ansible_collections_dir = os.path.join(package_dir, 'ansible_collections') - os.mkdir(ansible_collections_dir, mode=0o700) - - # Write the ansible release info to the collections dir - write_release_py(app_ctx.extra['ansible_version'], ansible_collections_dir) - - # Install collections - # TODO: PY3.8: - # collections_to_install = [p for f in os.listdir(download_dir) - # if os.path.isfile(p := os.path.join(download_dir, f))] - collections_to_install = [] - for collection in os.listdir(download_dir): - path = os.path.join(download_dir, collection) - if os.path.isfile(path): - collections_to_install.append(path) - - asyncio.run(install_together(collections_to_install, ansible_collections_dir)) - - # Compose and write release notes to destination directory - release_notes = ReleaseNotes.build(changelog) - # Write changelog and porting guide also to destination directory - release_notes.write_changelog_to(app_ctx.extra['dest_data_dir']) - release_notes.write_porting_guide_to(app_ctx.extra['dest_data_dir']) - - if create_package: - # Write them also to package directory if we're creating the package - release_notes.write_changelog_to(package_dir) - release_notes.write_porting_guide_to(package_dir) - - # Write build scripts and files - write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir) - write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version, '', - package_dir, release_notes, app_ctx.extra['debian']) - if app_ctx.extra['debian']: - write_debian_directory(app_ctx.extra['ansible_version'], ansible_base_version, - package_dir) - make_dist(package_dir, app_ctx.extra['sdist_dir']) - - if add_release: - ansible_changelog.changes.save() - - def build_single_command() -> int: # This is deprecated; in the future users will have to first run prepare, and then # rebuild-single. - return prepare_command(True) + result = prepare_command() + if result != 0: + return result + return rebuild_single_command() -def prepare_command(create_package: bool = False) -> int: + +def prepare_command() -> int: app_ctx = app_context.app_ctx.get() build_filename = os.path.join(app_ctx.extra['data_dir'], app_ctx.extra['build_file']) @@ -348,7 +266,17 @@ def prepare_command(create_package: bool = False) -> int: str(ansible_base_version), {collection: str(version) for collection, version in included_versions.items()}) - build_single_impl(dependency_data, create_package=create_package) + # Get Ansible changelog, add new release + ansible_changelog = ChangelogData.ansible( + app_ctx.extra['data_dir'], app_ctx.extra['dest_data_dir']) + date = datetime.date.today() + ansible_changelog.add_ansible_release( + str(app_ctx.extra['ansible_version']), + date, + f'Release Date: {date}' + f'\n\n' + f'`Porting Guide `_') + ansible_changelog.changes.save() deps_filename = os.path.join(app_ctx.extra['dest_data_dir'], app_ctx.extra['deps_file']) deps_file = DepsFile(deps_filename) @@ -367,7 +295,73 @@ def rebuild_single_command() -> int: deps_file = DepsFile(deps_filename) dependency_data = deps_file.parse() - build_single_impl(dependency_data, add_release=False) + # Determine included collection versions + ansible_base_version = PypiVer(dependency_data.ansible_base_version) + included_versions = { + collection: SemVer(version) + for collection, version in dependency_data.deps.items() + } + + with tempfile.TemporaryDirectory() as tmp_dir: + download_dir = os.path.join(tmp_dir, 'collections') + os.mkdir(download_dir, mode=0o700) + + # Download included collections + asyncio.run(download_collections(included_versions, app_ctx.galaxy_url, + download_dir, app_ctx.extra['collection_cache'])) + + # Get Ansible changelog, add new release + ansible_changelog = ChangelogData.ansible( + app_ctx.extra['data_dir'], app_ctx.extra['dest_data_dir']) + + # Get changelog and porting guide data + changelog = get_changelog( + app_ctx.extra['ansible_version'], + deps_dir=app_ctx.extra['data_dir'], + deps_data=[dependency_data], + collection_cache=app_ctx.extra['collection_cache'], + ansible_changelog=ansible_changelog) + + # Create package and collections directories + package_dir = os.path.join(tmp_dir, f'ansible-{app_ctx.extra["ansible_version"]}') + os.mkdir(package_dir, mode=0o700) + ansible_collections_dir = os.path.join(package_dir, 'ansible_collections') + os.mkdir(ansible_collections_dir, mode=0o700) + + # Write the ansible release info to the collections dir + write_release_py(app_ctx.extra['ansible_version'], ansible_collections_dir) + + # Install collections + # TODO: PY3.8: + # collections_to_install = [p for f in os.listdir(download_dir) + # if os.path.isfile(p := os.path.join(download_dir, f))] + collections_to_install = [] + for collection in os.listdir(download_dir): + path = os.path.join(download_dir, collection) + if os.path.isfile(path): + collections_to_install.append(path) + + asyncio.run(install_together(collections_to_install, ansible_collections_dir)) + + # Compose and write release notes to destination directory + release_notes = ReleaseNotes.build(changelog) + release_notes.write_changelog_to(package_dir) + release_notes.write_porting_guide_to(package_dir) + + # Write changelog and porting guide also to destination directory + release_notes.write_changelog_to(app_ctx.extra['dest_data_dir']) + release_notes.write_porting_guide_to(app_ctx.extra['dest_data_dir']) + + # Write build scripts and files + write_build_script(app_ctx.extra['ansible_version'], ansible_base_version, package_dir) + write_python_build_files(app_ctx.extra['ansible_version'], ansible_base_version, '', + package_dir, release_notes, app_ctx.extra['debian']) + if app_ctx.extra['debian']: + write_debian_directory(app_ctx.extra['ansible_version'], ansible_base_version, + package_dir) + + # Create source distribution + make_dist(package_dir, app_ctx.extra['sdist_dir']) return 0 diff --git a/src/antsibull/cli/antsibull_build.py b/src/antsibull/cli/antsibull_build.py index 9c55535d..bd564d89 100644 --- a/src/antsibull/cli/antsibull_build.py +++ b/src/antsibull/cli/antsibull_build.py @@ -241,8 +241,9 @@ def parse_args(program_name: str, args: List[str]) -> argparse.Namespace: ' $BASENAME_OF_PIECES_FILE-X.Y.build') subparsers.add_parser('prepare', - parents=[build_write_data_parser, cache_parser, - build_step_parser, feature_freeze_parser], + parents=[ + build_write_data_parser, build_step_parser, feature_freeze_parser, + ], description='Collect dependencies for an Ansible release') build_single_parser = subparsers.add_parser('single', From 6d634e1bb0a3ebe0e44f07a563bfcaa14a96bcbf Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 7 Dec 2021 07:43:39 +0100 Subject: [PATCH 4/9] Quiet 'Found no release_summary section' warning. --- pyproject.toml | 2 +- src/antsibull/changelog.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c6f2e4dd..3a1d9c86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ aiocontextvars = {version = "*", python = "~3.6"} aiofiles = "*" aiohttp = ">= 3.0.0" ansible-pygments = "*" -antsibull-changelog = ">= 0.7.0" +antsibull-changelog = ">= 0.10.0" asyncio-pool = "*" docutils = "*" importlib-metadata = {version = "*", python = "<3.8"} diff --git a/src/antsibull/changelog.py b/src/antsibull/changelog.py index 93ce1a22..3ff82a67 100644 --- a/src/antsibull/changelog.py +++ b/src/antsibull/changelog.py @@ -102,7 +102,8 @@ def concatenate(cls, changelogs: t.List['ChangelogData']) -> 'ChangelogData': def add_ansible_release(self, version: str, date: datetime.date, release_summary: str) -> None: add_release(self.config, self.changes, [], [], version, - codename=None, date=date, update_existing=True) + codename=None, date=date, update_existing=True, + show_release_summary_warning=False) release_date = self.changes.releases[version] if 'changes' not in release_date: release_date['changes'] = {} From 5f9ca8b30308e72915aa48719bc73db2a8f2cea8 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 7 Dec 2021 07:47:45 +0100 Subject: [PATCH 5/9] Add comment. --- src/antsibull/build_ansible_commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/antsibull/build_ansible_commands.py b/src/antsibull/build_ansible_commands.py index 23739ba6..42ebf1e6 100644 --- a/src/antsibull/build_ansible_commands.py +++ b/src/antsibull/build_ansible_commands.py @@ -278,6 +278,7 @@ def prepare_command() -> int: f'`Porting Guide `_') ansible_changelog.changes.save() + # Write dependency file deps_filename = os.path.join(app_ctx.extra['dest_data_dir'], app_ctx.extra['deps_file']) deps_file = DepsFile(deps_filename) deps_file.write( From 9ef1f41747940de4b793aded22cc10e6719b0d4f Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 7 Dec 2021 07:52:21 +0100 Subject: [PATCH 6/9] Update build documentation. --- docs/build-ansible.rst | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/docs/build-ansible.rst b/docs/build-ansible.rst index 9b3e1bbd..15f62d53 100644 --- a/docs/build-ansible.rst +++ b/docs/build-ansible.rst @@ -67,9 +67,13 @@ All alpha releases and the first beta # Generate the list of compatible versions. antsibull-build new-ansible 3.0.0a1 --data-dir ansible-build-data/3 + # Prepare the ansible release + # (This generates the dependency file and updates changelog.yaml) + antsibull-build prepare 3.0.0a1 --data-dir ansible-build-data/3 + # Create the ansible release # (This generates a single tarball for ansible with a dep on the ansible-base package) - antsibull-build single 3.0.0a1 --data-dir ansible-build-data/3 --sdist-dir built --debian + antsibull-build rebuild-single 3.0.0a1 --data-dir ansible-build-data/3 --sdist-dir built --debian Setup for the first beta release @@ -102,9 +106,13 @@ Beta2 up to and including rc1 rm -rf built mkdir built + # Prepare the ansible release + # (This generates the dependency file and updates changelog.yaml) + antsibull-build prepare 3.0.0b2 --feature-frozen --data-dir ansible-build-data/3 + # Create the ansible release # (This generates a single tarball for ansible with a dep on the ansible-base package) - antsibull-build single 3.0.0b2 --feature-frozen --data-dir ansible-build-data/3 --sdist-dir built --debian + antsibull-build rebuild-single 3.0.0b2 --data-dir ansible-build-data/3 --sdist-dir built --debian Any subsequent rcs and final @@ -115,7 +123,7 @@ Any subsequent rcs and final # Copy the previous rc's .deps file to the new rc version cp ansible-build-data/3/ansible-3.0.0rc1.deps ansible-build-data/3/ansible-3.0.0rc2.deps - # We do not run antsibull-build single because the compatible collection version information + # We do not run antsibull-build prepare because the compatible collection version information # is now set until final. # * Change the _ansible_version field to the new version # * If ansible-base needs a version update, change it in the .build and .deps file. @@ -137,19 +145,19 @@ New minor releases (3.Y.0) rm -rf built mkdir built - # Create the ansible release - # (This generates a single tarball for ansible with a dep on the ansible-base package) - antsibull-build single 3.1.0 --data-dir ansible-build-data/3 --sdist-dir built --debian + # Prepare the ansible release + # (This generates the dependency file and updates changelog.yaml) + antsibull-build prepare 3.1.0 --data-dir ansible-build-data/3 # Until we get separate versions for ansible-base working correctly: # https://github.com/ansible-community/antsibull/issues/187 - # We'll need to update the ansible-base version manually and then rebuild the release. Follow - # these steps after running antsibull-build single above: + # We'll need to update the ansible-core version manually. Follow + # these steps after running antsibull-build prepare above: # vim ansible-build-data/3/ansible-3.1.0.deps # Change the ansible-base version information in here to the latest compatible version on pypi - rm -rf built - mkdir built + # Create the ansible release + # (This generates a single tarball for ansible with a dep on the ansible-base package) antsibull-build rebuild-single 3.1.0 --data-dir ansible-build-data/3 --build-file ansible-3.build --deps-file ansible-3.1.0.deps --sdist-dir built --debian @@ -192,8 +200,8 @@ We want to sync docs and releases. So the first thing to do is to alert the doc ``#ansible-docs`` that we're making a release (they should know ahead of time if they're watching the schedule too). -* In patch releases, check the porting guide for unwanted breaking changes (collections that are new - in this patch release are allowed to have breaking changes but existing collections should not.) +* In minor/patch releases, check the porting guide for unwanted (breaking) changes (collections that are + new in this patch release are allowed to have breaking changes but existing collections should not.) * Fixing this requires manually changing the .deps file and re-running rebuild-single (and then pinging the collection maintainer to find out what should happen for the next release.) From 81f5efc9117ffe29bb904e3a89e68da329cf4e5e Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 7 Dec 2021 08:04:06 +0100 Subject: [PATCH 7/9] Deprecate 'multiple' command. --- src/antsibull/cli/antsibull_build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/antsibull/cli/antsibull_build.py b/src/antsibull/cli/antsibull_build.py index bd564d89..d7dade43 100644 --- a/src/antsibull/cli/antsibull_build.py +++ b/src/antsibull/cli/antsibull_build.py @@ -71,8 +71,11 @@ def _normalize_commands(args: argparse.Namespace) -> None: args.command = 'single' if args.command == 'build-multiple': - flog.warning('The build-multiple command is deprecated. Use `multiple` instead.') + flog.warning( + 'The build-multiple command is deprecated and will be removed in a future release.') args.command = 'multiple' + elif args.command == 'multiple': + flog.warning('The multiple command is deprecated and will be removed in a future release.') if args.command == 'build-collection': flog.warning('The build-collection command is deprecated. Use `collection` instead.') From 8f8b887390808a6a395eab6027331643c4f97554 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 7 Dec 2021 17:41:04 +0100 Subject: [PATCH 8/9] Update docs/build-ansible.rst Co-authored-by: David Moreau Simard --- docs/build-ansible.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build-ansible.rst b/docs/build-ansible.rst index 15f62d53..0ed76c4d 100644 --- a/docs/build-ansible.rst +++ b/docs/build-ansible.rst @@ -157,7 +157,7 @@ New minor releases (3.Y.0) # Change the ansible-base version information in here to the latest compatible version on pypi # Create the ansible release - # (This generates a single tarball for ansible with a dep on the ansible-base package) + # (This generates a single tarball for ansible with a dep on the ansible-core package) antsibull-build rebuild-single 3.1.0 --data-dir ansible-build-data/3 --build-file ansible-3.build --deps-file ansible-3.1.0.deps --sdist-dir built --debian From 718d6817183d428b9b4cf6625def18daf76c474d Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Tue, 7 Dec 2021 17:42:18 +0100 Subject: [PATCH 9/9] More -base to -core in that doc. --- docs/build-ansible.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/build-ansible.rst b/docs/build-ansible.rst index 0ed76c4d..4499d2e7 100644 --- a/docs/build-ansible.rst +++ b/docs/build-ansible.rst @@ -72,7 +72,7 @@ All alpha releases and the first beta antsibull-build prepare 3.0.0a1 --data-dir ansible-build-data/3 # Create the ansible release - # (This generates a single tarball for ansible with a dep on the ansible-base package) + # (This generates a single tarball for ansible with a dep on the ansible-core package) antsibull-build rebuild-single 3.0.0a1 --data-dir ansible-build-data/3 --sdist-dir built --debian @@ -111,7 +111,7 @@ Beta2 up to and including rc1 antsibull-build prepare 3.0.0b2 --feature-frozen --data-dir ansible-build-data/3 # Create the ansible release - # (This generates a single tarball for ansible with a dep on the ansible-base package) + # (This generates a single tarball for ansible with a dep on the ansible-core package) antsibull-build rebuild-single 3.0.0b2 --data-dir ansible-build-data/3 --sdist-dir built --debian @@ -126,7 +126,7 @@ Any subsequent rcs and final # We do not run antsibull-build prepare because the compatible collection version information # is now set until final. # * Change the _ansible_version field to the new version - # * If ansible-base needs a version update, change it in the .build and .deps file. + # * If ansible-core needs a version update, change it in the .build and .deps file. # * If any collections have been granted an update exception, change the range manually in the # .build and .deps file. # vim ansible-build-data/3/ansible-3.build @@ -149,12 +149,12 @@ New minor releases (3.Y.0) # (This generates the dependency file and updates changelog.yaml) antsibull-build prepare 3.1.0 --data-dir ansible-build-data/3 - # Until we get separate versions for ansible-base working correctly: + # Until we get separate versions for ansible-core working correctly: # https://github.com/ansible-community/antsibull/issues/187 # We'll need to update the ansible-core version manually. Follow # these steps after running antsibull-build prepare above: # vim ansible-build-data/3/ansible-3.1.0.deps - # Change the ansible-base version information in here to the latest compatible version on pypi + # Change the ansible-core version information in here to the latest compatible version on pypi # Create the ansible release # (This generates a single tarball for ansible with a dep on the ansible-core package) @@ -240,7 +240,7 @@ Update the topic in the #ansible channel with the new version TODO ==== -* Right now the script assumes ansible-base and ansible will have the same version. This is true - for 2.10 and possibly for 3 but in the longer term ansible-base major releases are going to +* Right now the script assumes ansible-core and ansible will have the same version. This is true + for 2.10 and possibly for 3 but in the longer term ansible-core major releases are going to slow down while ansible releases may speed up slightly. We'll need to adapt the script to handle these diverged versions.