Skip to content

Commit

Permalink
Make provider doc preparation a bit more fun :) (#23629)
Browse files Browse the repository at this point in the history 
Previously you had to manually add versions when changelog was
modified. But why not to get a bit more fun and get the versions
bumped automatically based on your assesment when reviewing the
provideers rather than after looking at the generated changelog.
  • Loading branch information
@potiuk
potiuk authored May 10, 2022
1 parent 863b257 commit cfa95af
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 55 deletions.
129 changes: 84 additions & 45 deletions dev/provider_packages/prepare_provider_packages.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

import jsonschema
import rich_click as click
import semver as semver
from github import Github, Issue, PullRequest, UnknownObjectException
from packaging.version import Version
from rich.console import Console
Expand Down Expand Up @@ -895,34 +896,31 @@ def print_changes_table(changes_table):


def get_all_changes_for_package(
versions: List[str],
provider_package_id: str,
source_provider_package_path: str,
verbose: bool,
) -> Tuple[bool, Optional[Union[List[List[Change]], Change]], str]:
"""
Retrieves all changes for the package.
:param versions: list of versions
:param provider_package_id: provider package id
:param source_provider_package_path: path where package is located
:param verbose: whether to print verbose messages
"""
current_version = versions[0]
provider_details = get_provider_details(provider_package_id)
current_version = provider_details.versions[0]
current_tag_no_suffix = get_version_tag(current_version, provider_package_id)
if verbose:
console.print(f"Checking if tag '{current_tag_no_suffix}' exist.")
if not subprocess.call(
get_git_tag_check_command(current_tag_no_suffix),
cwd=source_provider_package_path,
cwd=provider_details.source_provider_package_path,
stderr=subprocess.DEVNULL,
):
if verbose:
console.print(f"The tag {current_tag_no_suffix} exists.")
# The tag already exists
changes = subprocess.check_output(
get_git_log_command(verbose, HEAD_OF_HTTPS_REMOTE, current_tag_no_suffix),
cwd=source_provider_package_path,
cwd=provider_details.source_provider_package_path,
text=True,
)
if changes:
Expand All @@ -936,7 +934,7 @@ def get_all_changes_for_package(
try:
changes_since_last_doc_only_check = subprocess.check_output(
get_git_log_command(verbose, HEAD_OF_HTTPS_REMOTE, last_doc_only_hash),
cwd=source_provider_package_path,
cwd=provider_details.source_provider_package_path,
text=True,
)
if not changes_since_last_doc_only_check:
Expand All @@ -956,14 +954,9 @@ def get_all_changes_for_package(

console.print(f"[yellow]The provider {provider_package_id} has changes since last release[/]")
console.print()
console.print(
"[yellow]Please update version in "
f"'airflow/providers/{provider_package_id.replace('-','/')}/'"
"provider.yaml'[/]\n"
)
console.print("[yellow]Or mark the changes as doc-only[/]")
console.print(f"[bright_blue]Provider: {provider_package_id}[/]\n")
changes_table, array_of_changes = convert_git_changes_to_table(
"UNKNOWN",
f"NEXT VERSION AFTER + {provider_details.versions[0]}",
changes,
base_url="https://github.com/apache/airflow/commit/",
markdown=False,
Expand All @@ -975,21 +968,21 @@ def get_all_changes_for_package(
return False, None, ""
if verbose:
console.print("The tag does not exist. ")
if len(versions) == 1:
if len(provider_details.versions) == 1:
console.print(
f"The provider '{provider_package_id}' has never been released but it is ready to release!\n"
)
else:
console.print(f"New version of the '{provider_package_id}' package is ready to be released!\n")
next_version_tag = HEAD_OF_HTTPS_REMOTE
changes_table = ''
current_version = versions[0]
current_version = provider_details.versions[0]
list_of_list_of_changes: List[List[Change]] = []
for version in versions[1:]:
for version in provider_details.versions[1:]:
version_tag = get_version_tag(version, provider_package_id)
changes = subprocess.check_output(
get_git_log_command(verbose, next_version_tag, version_tag),
cwd=source_provider_package_path,
cwd=provider_details.source_provider_package_path,
text=True,
)
changes_table_for_version, array_of_changes_for_version = convert_git_changes_to_table(
Expand All @@ -1001,7 +994,7 @@ def get_all_changes_for_package(
current_version = version
changes = subprocess.check_output(
get_git_log_command(verbose, next_version_tag),
cwd=source_provider_package_path,
cwd=provider_details.source_provider_package_path,
text=True,
)
changes_table_for_version, array_of_changes_for_version = convert_git_changes_to_table(
Expand Down Expand Up @@ -1127,16 +1120,48 @@ def confirm(message: str, answer: Optional[str] = None) -> bool:
given_answer = answer.lower() if answer is not None else ""
while given_answer not in ["y", "n", "q", "yes", "no", "quit"]:
console.print(f"[yellow]{message}[y/n/q]?[/] ", end='')
given_answer = input("").lower()
try:
given_answer = input("").lower()
except KeyboardInterrupt:
given_answer = "q"
if given_answer.lower() in ["q", "quit"]:
# Returns 65 in case user decided to quit
sys.exit(65)
return given_answer in ["y", "yes"]


def mark_latest_changes_as_documentation_only(
provider_details: ProviderPackageDetails, latest_change: Change
):
class TypeOfChange(Enum):
DOCUMENTATION = "d"
BUGFIX = "b"
FEATURE = "f"
BREAKING_CHANGE = "x"
SKIP = "s"


def get_type_of_changes() -> TypeOfChange:
"""
Ask user to specify type of changes (case-insensitive).
:return: Type of change.
"""
given_answer = ""
while given_answer not in [*[t.value for t in TypeOfChange], "q"]:
console.print(
"[yellow]Type of change (d)ocumentation, (b)ugfix, (f)eature, (x)breaking "
"change, (s)kip, (q)uit [d/b/f/x/s/q]?[/] ",
end='',
)
try:
given_answer = input("").lower()
except KeyboardInterrupt:
given_answer = 'q'
if given_answer == "q":
# Returns 65 in case user decided to quit
sys.exit(65)
return TypeOfChange(given_answer)


def mark_latest_changes_as_documentation_only(provider_package_id: str, latest_change: Change):
provider_details = get_provider_details(provider_package_id=provider_package_id)
console.print(
f"Marking last change: {latest_change.short_hash} and all above changes since the last release "
"as doc-only changes!"
Expand All @@ -1149,6 +1174,24 @@ def mark_latest_changes_as_documentation_only(
sys.exit(66)


def add_new_version(type_of_change: TypeOfChange, provider_package_id: str):
provider_details = get_provider_details(provider_package_id)
version = provider_details.versions[0]
v = semver.VersionInfo.parse(version)
if type_of_change == TypeOfChange.BREAKING_CHANGE:
v = v.bump_major()
elif type_of_change == TypeOfChange.FEATURE:
v = v.bump_minor()
elif type_of_change == TypeOfChange.BUGFIX:
v = v.bump_patch()
provider_yaml_path = Path(get_source_package_path(provider_package_id)) / "provider.yaml"
original_text = provider_yaml_path.read_text()
new_text = re.sub(r'versions:', f'versions:\n - {v}', original_text, 1)
provider_yaml_path.write_text(new_text)
console.print()
console.print(f"[bright_blue]Bumped version to {v}")


def update_release_notes(
provider_package_id: str,
version_suffix: str,
Expand All @@ -1167,21 +1210,7 @@ def update_release_notes(
:returns False if the package should be skipped, True if everything generated properly
"""
verify_provider_package(provider_package_id)
provider_details = get_provider_details(provider_package_id)
provider_info = get_provider_info_from_provider_yaml(provider_package_id)
current_release_version = provider_details.versions[0]
jinja_context = get_provider_jinja_context(
provider_info=provider_info,
provider_details=provider_details,
current_release_version=current_release_version,
version_suffix=version_suffix,
)
proceed, latest_change, changes = get_all_changes_for_package(
provider_details.versions,
provider_package_id,
provider_details.source_provider_package_path,
verbose,
)
proceed, latest_change, changes = get_all_changes_for_package(provider_package_id, verbose)
if not force:
if proceed:
if not confirm("Provider marked for release. Proceed", answer=answer):
Expand All @@ -1194,17 +1223,29 @@ def update_release_notes(
console.print()
return False
else:
if confirm("Are those changes documentation-only?", answer=answer):
type_of_change = get_type_of_changes()
if type_of_change == TypeOfChange.DOCUMENTATION:
if isinstance(latest_change, Change):
mark_latest_changes_as_documentation_only(provider_details, latest_change)
mark_latest_changes_as_documentation_only(provider_package_id, latest_change)
else:
raise ValueError(
"Expected only one change to be present to mark changes "
f"in provider {provider_package_id} as docs-only. "
f"Received {len(latest_change)}."
)
return False

elif type_of_change == TypeOfChange.SKIP:
return False
elif type_of_change in [TypeOfChange.BUGFIX, TypeOfChange.FEATURE, TypeOfChange.BREAKING_CHANGE]:
add_new_version(type_of_change, provider_package_id)
proceed, latest_change, changes = get_all_changes_for_package(provider_package_id, verbose)
provider_details = get_provider_details(provider_package_id)
provider_info = get_provider_info_from_provider_yaml(provider_package_id)
jinja_context = get_provider_jinja_context(
provider_info=provider_info,
provider_details=provider_details,
current_release_version=provider_details.versions[0],
version_suffix=version_suffix,
)
jinja_context["DETAILED_CHANGES_RST"] = changes
jinja_context["DETAILED_CHANGES_PRESENT"] = len(changes) > 0
update_commits_rst(
Expand Down Expand Up @@ -1707,9 +1748,7 @@ def _update_changelog(package_id: str, verbose: bool) -> bool:
)
changelog_path = os.path.join(provider_details.source_provider_package_path, "CHANGELOG.rst")
proceed, changes, _ = get_all_changes_for_package(
provider_details.versions,
package_id,
provider_details.source_provider_package_path,
verbose,
)
if not proceed:
Expand Down
15 changes: 5 additions & 10 deletions scripts/in_container/run_prepare_provider_documentation.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,22 +45,18 @@ function run_prepare_documentation() {
skipped_documentation+=("${provider_package}")
continue
echo "${COLOR_YELLOW}Skipping provider package '${provider_package}'${COLOR_RESET}"
fi
if [[ ${res} == "65" ]]; then
elif [[ ${res} == "65" ]]; then
echo "${COLOR_RED}Exiting as the user chose to quit!${COLOR_RESET}"
exit 1
fi
if [[ ${res} == "128" ]]; then
elif [[ ${res} == "128" ]]; then
echo "${COLOR_RED}Exiting as there wes a serious error during processing '${provider_package}'${COLOR_RESET}"
error_documentation+=("${provider_package}")
exit 1
fi
if [[ ${res} == "66" ]]; then
elif [[ ${res} == "66" ]]; then
echo "${COLOR_YELLOW}Provider package '${provider_package}' marked as documentation-only!${COLOR_RESET}"
doc_only_documentation+=("${provider_package}")
continue
fi
if [[ ${res} != "0" ]]; then
elif [[ ${res} != "0" ]]; then
echo "${COLOR_RED}Error when generating provider package '${provider_package}'${COLOR_RESET}"
error_documentation+=("${provider_package}")
continue
Expand All @@ -75,8 +71,7 @@ function run_prepare_documentation() {
skipped_documentation+=("${provider_package}")
continue
echo "${COLOR_YELLOW}Skipping provider package '${provider_package}'${COLOR_RESET}"
fi
if [[ ${res} == "65" ]]; then
elif [[ ${res} == "65" ]]; then
echo "${COLOR_RED}Exiting as the user chose to quit!${COLOR_RESET}"
exit 1
fi
Expand Down

0 comments on commit cfa95af

Please sign in to comment.