Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve git repo and change detection logic #177

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/build-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,8 @@ jobs:
with:
pr_number: ${{ steps.findPr.outputs.pr }}
message: |
New release `${{ steps.findPr.outputs.pr }}.${{ github.run_number }}` deployed at [Test Pypi](https://test.pypi.org/project/mpyl/)
New release `${{ steps.findPr.outputs.pr }}.${{ github.run_number }}` deployed at [Test Pypi](https://test.pypi.org/project/mpyl/).
Install with `pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple mpyl==${{ steps.findPr.outputs.pr }}.${{ github.run_number }}`
reactions: rocket
comment_tag: execution

Expand Down
9 changes: 6 additions & 3 deletions Jenkinsfile
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ pipeline {
stage('Build') {
environment {
DOCKER_REGISTRY = credentials('91751de6-20aa-4b12-8459-6e16094a233a')
GIT_CREDENTIALS = credentials('e8bc2c24-e461-4dae-9122-e8ae8bd7ec07')
GITHUB_TOKEN = credentials('github-pat-mpyl-vandebronjenkins')
MPYL_GITHUB_APP_PRIVATE_KEY = credentials('mpyl_pipeline_github_app_private_key')
SLACK_TOKEN = credentials('JENKINS_MPYL_APP_OAUTH_TOKEN')
Expand All @@ -33,11 +34,13 @@ pipeline {
}
steps {
script {
sh("git tag -d \$(git tag -l)")

def gitconfig = scm.userRemoteConfigs.getAt(0)
git(branch: params.MPYL_CONFIG_BRANCH, credentialsId: gitconfig.getCredentialsId(), url: 'https://github.com/Vandebron/mpyl_config.git')
def config = readFile('mpyl_config.yml')
git(branch: env.BRANCH_NAME, credentialsId: gitconfig.getCredentialsId(), url: gitconfig.getUrl())
writeFile(file: 'mpyl_config.yml', text: config)

def content = sh(script: "curl -s https://api.github.com/repos/Vandebron/mpyl_config/contents/mpyl_config.yml -H 'Authorization: token $GIT_CREDENTIALS_PSW' -H 'Accept: application/vnd.github.v3.raw'", returnStdout: true)
writeFile(file: 'mpyl_config.yml', text: content)
withKubeConfig([credentialsId: 'jenkins-rancher-service-account-kubeconfig-test']) {
wrap([$class: 'BuildUser']) {
sh "pipenv clean"
Expand Down
3 changes: 3 additions & 0 deletions README-usage.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,9 @@ eval (env _MPYL_COMPLETE=fish_source mpyl)
```

#### YAML auto completion

![Schema based autocompletion](documentation_images/autocompletion.gif)

###### Intellij IDEA or PyCharm
Go to: `Preferences | Languages & Frameworks | Schemas and DTDs | JSON Schema Mappings`
- Add new schema
Expand Down
2 changes: 2 additions & 0 deletions src/mpyl/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from .cli.meta_info import get_version
from .cli.meta_info import version
from .cli.projects import projects
from .cli.repository import repository
from .utilities.pyaml_env import parse_config
from .utilities.repo import RepoConfig, Repository

Expand All @@ -27,6 +28,7 @@ def add_commands():
main_group.add_command(build)
main_group.add_command(version)
main_group.add_command(health)
main_group.add_command(repository)


def main():
Expand Down
96 changes: 71 additions & 25 deletions src/mpyl/cli/build.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""Commands related to build"""
import asyncio
import shutil
import sys
from pathlib import Path
from typing import Optional

Expand Down Expand Up @@ -36,7 +37,9 @@
BUILD_ARTIFACTS_FOLDER,
)
from ..project import load_project
from ..reporting.formatting.markdown import run_result_to_markdown
from ..reporting.formatting.markdown import (
execution_plan_as_markdown,
)
from ..steps.models import RunProperties
from ..steps.run import RunResult
from ..utilities.github import GithubConfig
Expand Down Expand Up @@ -135,51 +138,88 @@ def run(obj: CliContext, ci, all_, tag): # pylint: disable=invalid-name


@build.command(help="The status of the current local branch from MPyL's perspective")
@click.option(
"-f",
"--fix-pr",
is_flag=True,
help="Attempt to make the current branch fit for a PR build",
)
@click.pass_obj
def status(obj: CliContext):
def status(obj: CliContext, fix_pr):
try:
upgrade_check = asyncio.wait_for(warn_if_update(obj.console), timeout=3)
__print_status(obj)
__print_status(obj, fix_pr)
asyncio.get_event_loop().run_until_complete(upgrade_check)
except asyncio.exceptions.TimeoutError:
pass


def __print_status(obj: CliContext):
def __print_status(obj: CliContext, fix_pr: bool = False):
run_properties = RunProperties.from_configuration(obj.run_properties, obj.config)
branch = run_properties.versioning.branch
if not branch:
branch = obj.repo.get_branch
ci_branch = run_properties.versioning.branch

if ci_branch and not obj.repo.get_branch:
obj.console.print("Current branch is detached.")
if fix_pr:
obj.console.print(f"Checking out: `git checkout -b {ci_branch}`")
obj.repo.checkout(ci_branch)
else:
obj.console.log(
f"Branch not specified in `{DEFAULT_RUN_PROPERTIES_FILE_NAME}`. Branch determined via git: {branch}"
Markdown(
f"Branch not specified at `build.versioning.branch` in _{DEFAULT_RUN_PROPERTIES_FILE_NAME}_, "
f"falling back to git: _{obj.repo.get_branch}_"
)
)

if branch and obj.repo.main_branch == obj.repo.get_branch:
branch = obj.repo.get_branch
main_branch = obj.repo.main_branch

if branch == main_branch:
obj.console.log(f"On main branch ({branch}), cannot determine build status")
return

tag = obj.repo.get_tag if not branch else None
version = run_properties.versioning
revision = "Tag" if tag else "Branch"
revision = version.revision or obj.repo.get_sha
base_revision = obj.repo.base_revision
obj.console.print(
Markdown(
f"**{revision}:** `{version.branch or version.tag}` at `{version.revision}`"
f"**{'Tag' if tag else 'Branch'}:** `{branch or version.tag}` at `{revision}`. "
f"Base `{main_branch}` {f'at `{base_revision}`' if base_revision else 'not present (grafted).'}"
)
)

if obj.repo.main_branch_pulled:
changes = (
obj.repo.changes_in_branch_including_local()
if branch
else obj.repo.changes_in_merge_commit()
)
build_set = find_build_set(obj.repo, changes, False)
result = RunResult(run_properties=run_properties, run_plan=build_set)
if result.run_plan:
if not base_revision:
fetch = f"`git fetch origin {main_branch}:refs/remotes/origin/{main_branch}`"
if fix_pr:
obj.console.print(
Markdown("**Execution plan:** \n" + run_result_to_markdown(result))
f"Main branch {main_branch} not present locally. Fetching: {fetch}`"
)
obj.repo.fetch_main_branch()
else:
obj.console.print("No changes detected, nothing to do.")
obj.console.print(
Markdown(
f"Cannot determine what to build, since this branch has no base. "
f"Did you {fetch}?"
)
)
return

changes = (
obj.repo.changes_in_branch_including_local()
if ci_branch is None
else (
obj.repo.changes_in_merge_commit() if tag else obj.repo.changes_in_branch()
)
)
build_set = find_build_set(obj.repo, changes, False)
result = RunResult(run_properties=run_properties, run_plan=build_set)
if result.run_plan:
obj.console.print(
Markdown("**Execution plan:** \n" + execution_plan_as_markdown(result))
)
else:
obj.console.print("No changes detected, nothing to do.")


class Pipeline(ParamType):
Expand Down Expand Up @@ -321,9 +361,15 @@ def jenkins( # pylint: disable=too-many-arguments
):
try:
upgrade_check = asyncio.wait_for(warn_if_update(ctx.obj.console), timeout=5)
selected_pipeline = (
pipeline if pipeline else ctx.obj.config["jenkins"]["defaultPipeline"]
)
if "jenkins" not in ctx.obj.config:
ctx.obj.console.print(
"No Jenkins configuration found in config file. "
"Please add a `jenkins` section to your MPyL config file."
)
sys.exit(0)
jenkins_config = ctx.obj.config["jenkins"]

selected_pipeline = pipeline if pipeline else jenkins_config["defaultPipeline"]
pipeline_parameters = {"TEST": "true", "VERSION": test} if test else {}
if arguments:
pipeline_parameters["BUILD_PARAMS"] = " ".join(arguments)
Expand Down
2 changes: 1 addition & 1 deletion src/mpyl/cli/commands/build/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""CLI support for Jenkins multibranch pipeline"""
"""CI build related command group"""
import os
import shutil
from dataclasses import dataclass
Expand Down
4 changes: 2 additions & 2 deletions src/mpyl/cli/commands/build/mpyl.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@ def get_build_plan(
)
logger.info(f"Running with {params}")
if branch:
if repo.main_branch_pulled:
if repo.base_revision:
logger.info(
f"Branch `{repo.main_branch}` already present locally. Skipping pull."
)
else:
logger.info(f"Pulling `{repo.main_branch}` from {repo.get_remote_url}")
logger.info(f"Pulling `{repo.main_branch}` from {repo.remote_url}")
pull_result = repo.fetch_main_branch()
logger.info(f"Pulled `{pull_result[0].remote_ref_path.strip()}` to local")
changes = (
Expand Down
3 changes: 2 additions & 1 deletion src/mpyl/cli/commands/health/checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ def __check_jenkins(console: HealthConsole):
success=True,
)
except KeyError as exc:
console.check(f"Jenkins config not valid: {exc}", success=False)
console.check(f"Jenkins not (correctly) configured: {exc}", success=False)
return

gh_is_installed = shutil.which("gh")
if gh_is_installed:
Expand Down
2 changes: 1 addition & 1 deletion src/mpyl/cli/commands/projects/__init__.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
"""Project related command implementations"""
"""MPyL project definition related command group"""
1 change: 1 addition & 0 deletions src/mpyl/cli/commands/repository/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""CVS repository related command group"""
Empty file.
111 changes: 111 additions & 0 deletions src/mpyl/cli/repository.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
"""Commands related to the CSV (git) repository"""
import click
from rich.markdown import Markdown

from . import (
CliContext,
CONFIG_PATH_HELP,
create_console_logger,
)
from ..constants import DEFAULT_CONFIG_FILE_NAME, DEFAULT_RUN_PROPERTIES_FILE_NAME
from ..steps.models import RunProperties
from ..utilities.pyaml_env import parse_config
from ..utilities.repo import Repository, RepoConfig


@click.group("repo")
@click.option(
"--config",
"-c",
required=True,
type=click.Path(exists=True),
help=CONFIG_PATH_HELP,
envvar="MPYL_CONFIG_PATH",
default=DEFAULT_CONFIG_FILE_NAME,
)
@click.option(
"--properties",
"-p",
required=False,
type=click.Path(exists=False),
help="Path to run properties",
envvar="MPYL_RUN_PROPERTIES_PATH",
default=DEFAULT_RUN_PROPERTIES_FILE_NAME,
show_default=True,
)
@click.option("--verbose", "-v", is_flag=True, default=False)
@click.pass_context
def repository(ctx, config, properties, verbose):
"""Manage CVS (git) repositories"""
console = create_console_logger(local=False, verbose=verbose)
ctx.obj = CliContext(
config=(parse_config(config)),
repo=ctx.with_resource(
Repository(config=RepoConfig.from_config(parse_config(config)))
),
console=console,
verbose=verbose,
run_properties=(parse_config(properties)),
)


@repository.command(help="The status of the current local repository")
@click.pass_obj
def status(obj: CliContext):
"""Print the status of the current repository"""
run_properties = RunProperties.from_configuration(obj.run_properties, obj.config)
console = obj.console
repo = obj.repo

console.print(
Markdown(f"Repository tracking [{repo.remote_url}]({repo.remote_url})")
if repo.remote_url
else Markdown("Repository not tracking any remote origin")
)
ci_branch = run_properties.versioning.branch

console.print(
Markdown(
f"Branch as specified in _{DEFAULT_RUN_PROPERTIES_FILE_NAME}_: `{ci_branch}`"
if ci_branch
else f"No branch specified at `build.versioning.branch` in _{DEFAULT_RUN_PROPERTIES_FILE_NAME}_"
)
)
if not repo.has_valid_head:
console.log(
Markdown(
f"Current branch `{repo.get_branch}` does **not** point to a valid reference."
)
)
return

console.log(
Markdown(
f"Current branch: `{repo.get_branch}` at `{repo.get_sha}`"
if repo.get_branch
else "Head of current branch is detached"
)
)
base_revision = repo.base_revision
console.line(1)
console.log(Markdown(f"Configured base branch: `{repo.main_origin_branch}`"))
if repo.get_branch == repo.main_branch:
console.log(f"On main branch ({repo.main_branch})")
return
if base_revision:
console.log(
Markdown(
f"Locally `{repo.main_origin_branch}` is at _{base_revision.hexsha}_ by _{base_revision.author}_ at "
f"{base_revision.committed_datetime}"
)
)
changes = obj.repo.changes_between(base_revision.hexsha, repo.get_sha)
console.log(
f"{len(changes)} commits between `{repo.main_origin_branch}` and `{repo.get_branch}`"
)
else:
console.log(
Markdown(
f"Revision for `{repo.main_origin_branch}` not found. Cannot diff with base"
)
)
Binary file added src/mpyl/documentation_images/autocompletion.gif
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
9 changes: 6 additions & 3 deletions src/mpyl/reporting/formatting/markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,12 @@ def markdown_for_stage(run_result: RunResult, stage: Stage):


def run_result_to_markdown(run_result: RunResult) -> str:
result: str = f"{run_result.status_line} \n"
status_line: str = f"{run_result.status_line} \n"
return status_line + execution_plan_as_markdown(run_result)


def execution_plan_as_markdown(run_result):
result = ""
exception = run_result.exception
if exception:
result += f"For _{exception.executor}_ on _{exception.project_name}_ at _{exception.stage}_ \n"
Expand All @@ -109,10 +114,8 @@ def run_result_to_markdown(run_result: RunResult) -> str:
failed = run_result.failed_result
result += f"For _{failed.project.name}_ at _{failed.stage}_ \n"
result += f"\n\n{run_result.failed_result.output.message}\n\n"

for stage in Stage:
result += markdown_for_stage(run_result, stage)

return result


Expand Down
Loading