Improve git repo and change detection logic

.github/workflows/build-package.yml

@@ -153,7 +153,8 @@ jobs:
         with:
           pr_number: ${{ steps.findPr.outputs.pr }}
           message: |
-            New release `${{ steps.findPr.outputs.pr }}.${{ github.run_number }}` deployed at [Test Pypi](https://test.pypi.org/project/mpyl/)
+            New release `${{ steps.findPr.outputs.pr }}.${{ github.run_number }}` deployed at [Test Pypi](https://test.pypi.org/project/mpyl/). 
+            Install with `pip install -i https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple mpyl==${{ steps.findPr.outputs.pr }}.${{ github.run_number }}`
           reactions: rocket
           comment_tag: execution
 

Jenkinsfile

@@ -24,6 +24,7 @@ pipeline {
         stage('Build') {
            environment {
                 DOCKER_REGISTRY = credentials('91751de6-20aa-4b12-8459-6e16094a233a')
+                GIT_CREDENTIALS = credentials('e8bc2c24-e461-4dae-9122-e8ae8bd7ec07')
                 GITHUB_TOKEN = credentials('github-pat-mpyl-vandebronjenkins')
                 MPYL_GITHUB_APP_PRIVATE_KEY = credentials('mpyl_pipeline_github_app_private_key')
                 SLACK_TOKEN = credentials('JENKINS_MPYL_APP_OAUTH_TOKEN')
@@ -33,11 +34,13 @@ pipeline {
             }
             steps {
                 script {
+                    sh("git tag -d \$(git tag -l)")
+
                     def gitconfig = scm.userRemoteConfigs.getAt(0)
-                    git(branch: params.MPYL_CONFIG_BRANCH, credentialsId: gitconfig.getCredentialsId(), url: 'https://github.com/Vandebron/mpyl_config.git')
-                    def config = readFile('mpyl_config.yml')
                     git(branch: env.BRANCH_NAME, credentialsId: gitconfig.getCredentialsId(), url: gitconfig.getUrl())
-                    writeFile(file: 'mpyl_config.yml', text: config)
+
+                    def content = sh(script: "curl -s https://api.github.com/repos/Vandebron/mpyl_config/contents/mpyl_config.yml -H 'Authorization: token $GIT_CREDENTIALS_PSW' -H 'Accept: application/vnd.github.v3.raw'", returnStdout: true)
+                    writeFile(file: 'mpyl_config.yml', text: content)
                     withKubeConfig([credentialsId: 'jenkins-rancher-service-account-kubeconfig-test']) {
                         wrap([$class: 'BuildUser']) {
                             sh "pipenv clean"

README-usage.md

@@ -85,6 +85,9 @@ eval (env _MPYL_COMPLETE=fish_source mpyl)
 ```
 
 #### YAML auto completion 
+
+![Schema based autocompletion](documentation_images/autocompletion.gif)
+
 ###### Intellij IDEA or PyCharm
 Go to: `Preferences | Languages & Frameworks | Schemas and DTDs | JSON Schema Mappings`
 - Add new schema

src/mpyl/__init__.py

@@ -13,6 +13,7 @@
 from .cli.meta_info import get_version
 from .cli.meta_info import version
 from .cli.projects import projects
+from .cli.repository import repository
 from .utilities.pyaml_env import parse_config
 from .utilities.repo import RepoConfig, Repository
 
@@ -27,6 +28,7 @@ def add_commands():
     main_group.add_command(build)
     main_group.add_command(version)
     main_group.add_command(health)
+    main_group.add_command(repository)
 
 
 def main():

src/mpyl/cli/build.py

@@ -1,6 +1,7 @@
 """Commands related to build"""
 import asyncio
 import shutil
+import sys
 from pathlib import Path
 from typing import Optional
 
@@ -36,7 +37,9 @@
     BUILD_ARTIFACTS_FOLDER,
 )
 from ..project import load_project
-from ..reporting.formatting.markdown import run_result_to_markdown
+from ..reporting.formatting.markdown import (
+    execution_plan_as_markdown,
+)
 from ..steps.models import RunProperties
 from ..steps.run import RunResult
 from ..utilities.github import GithubConfig
@@ -135,51 +138,88 @@ def run(obj: CliContext, ci, all_, tag):  # pylint: disable=invalid-name
 
 
 @build.command(help="The status of the current local branch from MPyL's perspective")
+@click.option(
+    "-f",
+    "--fix-pr",
+    is_flag=True,
+    help="Attempt to make the current branch fit for a PR build",
+)
 @click.pass_obj
-def status(obj: CliContext):
+def status(obj: CliContext, fix_pr):
     try:
         upgrade_check = asyncio.wait_for(warn_if_update(obj.console), timeout=3)
-        __print_status(obj)
+        __print_status(obj, fix_pr)
         asyncio.get_event_loop().run_until_complete(upgrade_check)
     except asyncio.exceptions.TimeoutError:
         pass
 
 
-def __print_status(obj: CliContext):
+def __print_status(obj: CliContext, fix_pr: bool = False):
     run_properties = RunProperties.from_configuration(obj.run_properties, obj.config)
-    branch = run_properties.versioning.branch
-    if not branch:
-        branch = obj.repo.get_branch
+    ci_branch = run_properties.versioning.branch
+
+    if ci_branch and not obj.repo.get_branch:
+        obj.console.print("Current branch is detached.")
+        if fix_pr:
+            obj.console.print(f"Checking out: `git checkout -b {ci_branch}`")
+            obj.repo.checkout(ci_branch)
+    else:
         obj.console.log(
-            f"Branch not specified in `{DEFAULT_RUN_PROPERTIES_FILE_NAME}`. Branch determined via git: {branch}"
+            Markdown(
+                f"Branch not specified at `build.versioning.branch` in _{DEFAULT_RUN_PROPERTIES_FILE_NAME}_, "
+                f"falling back to git: _{obj.repo.get_branch}_"
+            )
         )
 
-    if branch and obj.repo.main_branch == obj.repo.get_branch:
+    branch = obj.repo.get_branch
+    main_branch = obj.repo.main_branch
+
+    if branch == main_branch:
         obj.console.log(f"On main branch ({branch}), cannot determine build status")
         return
+
     tag = obj.repo.get_tag if not branch else None
     version = run_properties.versioning
-    revision = "Tag" if tag else "Branch"
+    revision = version.revision or obj.repo.get_sha
+    base_revision = obj.repo.base_revision
     obj.console.print(
         Markdown(
-            f"**{revision}:** `{version.branch or version.tag}` at `{version.revision}`"
+            f"**{'Tag' if tag else 'Branch'}:** `{branch or version.tag}` at `{revision}`. "
+            f"Base `{main_branch}` {f'at `{base_revision}`' if base_revision else 'not present (grafted).'}"
         )
     )
 
-    if obj.repo.main_branch_pulled:
-        changes = (
-            obj.repo.changes_in_branch_including_local()
-            if branch
-            else obj.repo.changes_in_merge_commit()
-        )
-        build_set = find_build_set(obj.repo, changes, False)
-        result = RunResult(run_properties=run_properties, run_plan=build_set)
-        if result.run_plan:
+    if not base_revision:
+        fetch = f"`git fetch origin {main_branch}:refs/remotes/origin/{main_branch}`"
+        if fix_pr:
             obj.console.print(
-                Markdown("**Execution plan:**  \n" + run_result_to_markdown(result))
+                f"Main branch {main_branch} not present locally. Fetching: {fetch}`"
             )
+            obj.repo.fetch_main_branch()
         else:
-            obj.console.print("No changes detected, nothing to do.")
+            obj.console.print(
+                Markdown(
+                    f"Cannot determine what to build, since this branch has no base. "
+                    f"Did you {fetch}?"
+                )
+            )
+            return
+
+    changes = (
+        obj.repo.changes_in_branch_including_local()
+        if ci_branch is None
+        else (
+            obj.repo.changes_in_merge_commit() if tag else obj.repo.changes_in_branch()
+        )
+    )
+    build_set = find_build_set(obj.repo, changes, False)
+    result = RunResult(run_properties=run_properties, run_plan=build_set)
+    if result.run_plan:
+        obj.console.print(
+            Markdown("**Execution plan:**  \n" + execution_plan_as_markdown(result))
+        )
+    else:
+        obj.console.print("No changes detected, nothing to do.")
 
 
 class Pipeline(ParamType):
@@ -321,9 +361,15 @@ def jenkins(  # pylint: disable=too-many-arguments
 ):
     try:
         upgrade_check = asyncio.wait_for(warn_if_update(ctx.obj.console), timeout=5)
-        selected_pipeline = (
-            pipeline if pipeline else ctx.obj.config["jenkins"]["defaultPipeline"]
-        )
+        if "jenkins" not in ctx.obj.config:
+            ctx.obj.console.print(
+                "No Jenkins configuration found in config file. "
+                "Please add a `jenkins` section to your MPyL config file."
+            )
+            sys.exit(0)
+        jenkins_config = ctx.obj.config["jenkins"]
+
+        selected_pipeline = pipeline if pipeline else jenkins_config["defaultPipeline"]
         pipeline_parameters = {"TEST": "true", "VERSION": test} if test else {}
         if arguments:
             pipeline_parameters["BUILD_PARAMS"] = " ".join(arguments)

src/mpyl/cli/commands/build/__init__.py

@@ -1,4 +1,4 @@
-"""CLI support for Jenkins multibranch pipeline"""
+"""CI build related command group"""
 import os
 import shutil
 from dataclasses import dataclass

src/mpyl/cli/commands/build/mpyl.py

@@ -56,12 +56,12 @@ def get_build_plan(
     )
     logger.info(f"Running with {params}")
     if branch:
-        if repo.main_branch_pulled:
+        if repo.base_revision:
             logger.info(
                 f"Branch `{repo.main_branch}` already present locally. Skipping pull."
             )
         else:
-            logger.info(f"Pulling `{repo.main_branch}` from {repo.get_remote_url}")
+            logger.info(f"Pulling `{repo.main_branch}` from {repo.remote_url}")
             pull_result = repo.fetch_main_branch()
             logger.info(f"Pulled `{pull_result[0].remote_ref_path.strip()}` to local")
         changes = (

src/mpyl/cli/commands/health/checks.py

@@ -79,7 +79,8 @@ def __check_jenkins(console: HealthConsole):
             success=True,
         )
     except KeyError as exc:
-        console.check(f"Jenkins config not valid: {exc}", success=False)
+        console.check(f"Jenkins not (correctly) configured: {exc}", success=False)
+        return
 
     gh_is_installed = shutil.which("gh")
     if gh_is_installed:

src/mpyl/cli/commands/projects/__init__.py

@@ -1 +1 @@
-"""Project related command implementations"""
+"""MPyL project definition related command group"""

src/mpyl/cli/commands/repository/__init__.py

@@ -0,0 +1 @@
+"""CVS repository related command group"""

src/mpyl/cli/repository.py

@@ -0,0 +1,111 @@
+"""Commands related to the CSV (git) repository"""
+import click
+from rich.markdown import Markdown
+
+from . import (
+    CliContext,
+    CONFIG_PATH_HELP,
+    create_console_logger,
+)
+from ..constants import DEFAULT_CONFIG_FILE_NAME, DEFAULT_RUN_PROPERTIES_FILE_NAME
+from ..steps.models import RunProperties
+from ..utilities.pyaml_env import parse_config
+from ..utilities.repo import Repository, RepoConfig
+
+
+@click.group("repo")
+@click.option(
+    "--config",
+    "-c",
+    required=True,
+    type=click.Path(exists=True),
+    help=CONFIG_PATH_HELP,
+    envvar="MPYL_CONFIG_PATH",
+    default=DEFAULT_CONFIG_FILE_NAME,
+)
+@click.option(
+    "--properties",
+    "-p",
+    required=False,
+    type=click.Path(exists=False),
+    help="Path to run properties",
+    envvar="MPYL_RUN_PROPERTIES_PATH",
+    default=DEFAULT_RUN_PROPERTIES_FILE_NAME,
+    show_default=True,
+)
+@click.option("--verbose", "-v", is_flag=True, default=False)
+@click.pass_context
+def repository(ctx, config, properties, verbose):
+    """Manage CVS (git) repositories"""
+    console = create_console_logger(local=False, verbose=verbose)
+    ctx.obj = CliContext(
+        config=(parse_config(config)),
+        repo=ctx.with_resource(
+            Repository(config=RepoConfig.from_config(parse_config(config)))
+        ),
+        console=console,
+        verbose=verbose,
+        run_properties=(parse_config(properties)),
+    )
+
+
+@repository.command(help="The status of the current local repository")
+@click.pass_obj
+def status(obj: CliContext):
+    """Print the status of the current repository"""
+    run_properties = RunProperties.from_configuration(obj.run_properties, obj.config)
+    console = obj.console
+    repo = obj.repo
+
+    console.print(
+        Markdown(f"Repository tracking [{repo.remote_url}]({repo.remote_url})")
+        if repo.remote_url
+        else Markdown("Repository not tracking any remote origin")
+    )
+    ci_branch = run_properties.versioning.branch
+
+    console.print(
+        Markdown(
+            f"Branch as specified in _{DEFAULT_RUN_PROPERTIES_FILE_NAME}_: `{ci_branch}`"
+            if ci_branch
+            else f"No branch specified at `build.versioning.branch` in _{DEFAULT_RUN_PROPERTIES_FILE_NAME}_"
+        )
+    )
+    if not repo.has_valid_head:
+        console.log(
+            Markdown(
+                f"Current branch `{repo.get_branch}` does **not** point to a valid reference."
+            )
+        )
+        return
+
+    console.log(
+        Markdown(
+            f"Current branch: `{repo.get_branch}` at `{repo.get_sha}`"
+            if repo.get_branch
+            else "Head of current branch is detached"
+        )
+    )
+    base_revision = repo.base_revision
+    console.line(1)
+    console.log(Markdown(f"Configured base branch: `{repo.main_origin_branch}`"))
+    if repo.get_branch == repo.main_branch:
+        console.log(f"On main branch ({repo.main_branch})")
+        return
+    if base_revision:
+        console.log(
+            Markdown(
+                f"Locally `{repo.main_origin_branch}` is at _{base_revision.hexsha}_ by _{base_revision.author}_ at "
+                f"{base_revision.committed_datetime}"
+            )
+        )
+        changes = obj.repo.changes_between(base_revision.hexsha, repo.get_sha)
+        console.log(
+            f"{len(changes)} commits between `{repo.main_origin_branch}` and `{repo.get_branch}`"
+        )
+    else:
+        console.log(
+            Markdown(
+                f"Revision for `{repo.main_origin_branch}` not found. Cannot diff with base"
+            )
+        )

src/mpyl/reporting/formatting/markdown.py

@@ -100,7 +100,12 @@ def markdown_for_stage(run_result: RunResult, stage: Stage):
 
 
 def run_result_to_markdown(run_result: RunResult) -> str:
-    result: str = f"{run_result.status_line}  \n"
+    status_line: str = f"{run_result.status_line}  \n"
+    return status_line + execution_plan_as_markdown(run_result)
+
+
+def execution_plan_as_markdown(run_result):
+    result = ""
     exception = run_result.exception
     if exception:
         result += f"For _{exception.executor}_ on _{exception.project_name}_ at _{exception.stage}_ \n"
@@ -109,10 +114,8 @@ def run_result_to_markdown(run_result: RunResult) -> str:
         failed = run_result.failed_result
         result += f"For _{failed.project.name}_ at _{failed.stage}_ \n"
         result += f"\n\n{run_result.failed_result.output.message}\n\n"
-
     for stage in Stage:
         result += markdown_for_stage(run_result, stage)
-
     return result