diff --git a/.github/workflows/update_api_docs.yml b/.github/workflows/update_api_docs.yml new file mode 100644 index 0000000000..a3f08fda79 --- /dev/null +++ b/.github/workflows/update_api_docs.yml @@ -0,0 +1,81 @@ +name: Update API docs + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +jobs: + update_api: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + working-directory: ./docs/api_docs + steps: + - name: Check out repository 🛎️ + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: "3.10" + + - name: Install nebari and docs dependencies + run: | + python -m pip install --upgrade pip + pip install -e "../..[docs]" + + - name: Generate new API docs + run: | + echo "generate the new docs" + pydoc-markdown -I "../../" -p nebari --render-toc > api_temp.md + echo "make modifications to clean up" + python api_doc_mods.py --autogen_path api_temp.md --outpath api_doc.md + + - name: Lint the new docs + continue-on-error: true + run: | + echo "run linting and formatting" + pre-commit run --files api_doc.md + echo "run it again to prove its clean now" + pre-commit run --files api_doc.md + + - name: Check for changes + run: | + if git diff --exit-code; then + echo "changes_exist=true" >> $GITHUB_ENV + else + echo "changes_exist=false" >> $GITHUB_ENV + fi + + - name: Look for changes to cli files + uses: tj-actions/verify-changed-files@v12 + id: verify-changed-files + with: + files: | + docs/api_docs/api_doc.md + + - name: Create Pull Request in code repo + id: create_pull_request + uses: peter-evans/create-pull-request@v4 + if: steps.verify-changed-files.outputs.files_changed == 'true' + with: + token: ${{ secrets.NEBARI_SENSEI_API_DOCS_PR_OPENER }} + commit-message: Update report + committer: GitHub + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: false + branch: auto_api_doc_update + delete-branch: true + title: '[AUTO] Update API doc' + body: | + Update API doc + - Auto-generated by [create-pull-request][1] + + [1]: https://github.com/peter-evans/create-pull-request + labels: | + "area: documentation 📖" + draft: false diff --git a/.gitignore b/.gitignore index db9bca57d6..68665a09c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # docs .nox _build +docs/api_docs/api_temp.md # setuptools scm nebari/_version.py diff --git a/docs/.gitignore b/docs/.gitignore index 565e212662..c334ed175e 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -70,3 +70,6 @@ yarn-error.log* app/__pycache__ docs/users.pdf + +# docs +docs/api_docs/api_temp.md diff --git a/docs/api_docs/README.md b/docs/api_docs/README.md new file mode 100644 index 0000000000..aa3a06028c --- /dev/null +++ b/docs/api_docs/README.md @@ -0,0 +1,28 @@ +# Nebari API docs + +The Nebari API docs are generated from this repo, then merged into the nebari-docs repo via PR opened by GitHub Actions. + +## Environment + +From the root of the repo: + +```bash +pip install -e ".[docs]" +``` + +## Workflow + +The docs may be built manually through the following workflow, or executed as part of an autoupdate via GH Actions CI. + +The workflow described below will create the API docs and run a formatting script to do a bit of cleanup. + +```bash +# work out of the api_docs directory in the nebari repo +cd docs/api_docs/ +# auto generate the api docs +pydoc-markdown -I "../../" -p nebari --render-toc > api_temp.md +# make formatting modifications and copy into the docs repo +python api_doc_mods.py --autogen_path api_temp.md --outpath api_doc.md +# run linting and formatting +pre-commit run --files api_doc.md +``` diff --git a/docs/api_docs/api_doc.md b/docs/api_docs/api_doc.md new file mode 100644 index 0000000000..d421af4eb0 --- /dev/null +++ b/docs/api_docs/api_doc.md @@ -0,0 +1,535 @@ +______________________________________________________________________ + +## title: API Documentation description: Reference to API docs + +\<--- DO NOT EDIT, THIS PAGE IS AUTOGENERATED ---># API Documentation + +## Table of Contents + + + +# nebari.destroy + + + +# nebari.initialize + + + +# nebari.render + + + +#### render_contents + +```python +def render_contents(config: Dict) +``` + +Dynamically generated contents from Nebari configuration + + + +#### gen_gitignore + +```python +def gen_gitignore(config) +``` + +Generate `.gitignore` file. Add files as needed. + + + +#### gen_cicd + +```python +def gen_cicd(config) +``` + +Use cicd schema to generate workflow files based on the `ci_cd` key in the `config`. + +For more detail on schema: GiHub-Actions - nebari/providers/cicd/github.py GitLab-CI - nebari/providers/cicd/gitlab.py + + + +#### inspect_files + +```python +def inspect_files(source_dirs: str, + output_dirs: str, + source_base_dir: str, + output_base_dir: str, + ignore_filenames: List[str] = None, + ignore_directories: List[str] = None, + deleted_paths: List[str] = None, + contents: Dict[str, str] = None) +``` + +Return created, updated and untracked files by computing a checksum over the provided directory + +**Arguments**: + +- `source_dirs` _str_ - The source dir used as base for comparssion +- `output_dirs` _str_ - The destination dir which will be matched with +- `source_base_dir` _str_ - Relative base path to source directory +- `output_base_dir` _str_ - Relative base path to output directory +- `ignore_filenames` _list\[str\]_ - Filenames to ignore while comparing for changes +- `ignore_directories` _list\[str\]_ - Directories to ignore while comparing for changes +- `deleted_paths` _list\[str\]_ - Paths that if exist in output directory should be deleted +- `contents` _dict_ - filename to content mapping for dynamically generated files + + + +#### hash_file + +```python +def hash_file(file_path: str) +``` + +Get the hex digest of the given file + +**Arguments**: + +- `file_path` _str_ - path to file + + + +#### set_env_vars_in_config + +```python +def set_env_vars_in_config(config) +``` + +For values in the config starting with 'NEBARI_SECRET_XXX' the environment variables are searched for the pattern XXX +and the config value is modified. This enables setting secret values that should not be directly stored in the config +file. + +NOTE: variables are most likely written to a file somewhere upon render. In order to further reduce risk of exposure of +any of these variables you might consider preventing storage of the terraform render output. + + + +# nebari.cli + + + +# nebari.cli.keycloak + + + +#### add_user + +```python +@app_keycloak.command(name="adduser") +def add_user(add_users: Tuple[str, str] = typer.Option( + ..., "--user", help="Provide both: "), + config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + )) +``` + +Add a user to Keycloak. User will be automatically added to the \[italic\]analyst\[/italic\] group. + + + +#### list_users + +```python +@app_keycloak.command(name="listusers") +def list_users(config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", +)) +``` + +List the users in Keycloak. + + + +# nebari.cli.init + + + +#### handle_init + +```python +def handle_init(inputs: InitInputs) +``` + +Take the inputs from the `nebari init` command, render the config and write it to a local yaml file. + + + +#### check_cloud_provider_creds + +```python +def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str) +``` + +Validate that the necessary cloud credentials have been set as environment variables. + + + +#### check_auth_provider_creds + +```python +def check_auth_provider_creds(ctx: typer.Context, auth_provider: str) +``` + +Validating the the necessary auth provider credentials have been set as environment variables. + + + +#### check_project_name + +```python +def check_project_name(ctx: typer.Context, project_name: str) +``` + +Validate the project_name is acceptable. Depends on `cloud_provider`. + + + +#### guided_init_wizard + +```python +def guided_init_wizard(ctx: typer.Context, guided_init: str) +``` + +Guided Init Wizard is a user-friendly questionnaire used to help generate the `nebari-config.yaml`. + + + +# nebari.cli.main + + + +## OrderCommands Objects + +```python +class OrderCommands(TyperGroup) +``` + + + +#### list_commands + +```python +def list_commands(ctx: Context) +``` + +Return list of commands in the order appear. + + + +#### init + +```python +@app.command() +def init(cloud_provider: str = typer.Argument( + "local", + help=f"options: {enum_to_list(ProviderEnum)}", + callback=check_cloud_provider_creds, + is_eager=True, +), + guided_init: bool = typer.Option( + False, + help=GUIDED_INIT_MSG, + callback=guided_init_wizard, + is_eager=True, + ), + project_name: str = typer.Option( + ..., + "--project-name", + "--project", + "-p", + callback=check_project_name, + ), + domain_name: str = typer.Option( + ..., + "--domain-name", + "--domain", + "-d", + ), + namespace: str = typer.Option("dev", ), + auth_provider: str = typer.Option( + "password", + help=f"options: {enum_to_list(AuthenticationEnum)}", + callback=check_auth_provider_creds, + ), + auth_auto_provision: bool = typer.Option(False, ), + repository: str = typer.Option(None, ), + repository_auto_provision: bool = typer.Option(False, ), + ci_provider: str = typer.Option( + None, + help=f"options: {enum_to_list(CiEnum)}", + ), + terraform_state: str = typer.Option( + "remote", help=f"options: {enum_to_list(TerraformStateEnum)}"), + kubernetes_version: str = typer.Option("latest", ), + ssl_cert_email: str = typer.Option( + None, + callback=check_ssl_cert_email, + ), + disable_prompt: bool = typer.Option( + False, + is_eager=True, + )) +``` + +Create and initialize your \[purple\]nebari-config.yaml\[/purple\] file. + +This command will create and initialize your \[purple\]nebari-config.yaml\[/purple\] :sparkles: + +This file contains all your Nebari cluster configuration details and, is used as input to later commands such as +\[green\]nebari render\[/green\], \[green\]nebari deploy\[/green\], etc. + +If you're new to Nebari, we recommend you use the Guided Init wizard. To get started simply run: + +``` + [green]nebari init --guided-init[/green] +``` + + + +#### validate + +```python +@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) +def validate(config: str = typer.Option( + ..., + "--config", + "-c", + help= + "nebari configuration yaml file path, please pass in as -c/--config flag", +), + enable_commenting: bool = typer.Option( + False, + "--enable-commenting", + help="Toggle PR commenting on GitHub Actions")) +``` + +Validate the values in the \[purple\]nebari-config.yaml\[/purple\] file are acceptable. + + + +#### render + +```python +@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) +def render( + output: str = typer.Option( + "./", + "-o", + "--output", + help="output directory", + ), + config: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration yaml file path", + ), + dry_run: bool = typer. + Option( + False, + "--dry-run", + help= + "simulate rendering files without actually writing or updating any files", + )) +``` + +Dynamically render the Terraform scripts and other files from your \[purple\]nebari-config.yaml\[/purple\] file. + + + +#### deploy + +```python +@app.command() +def deploy( + config: str = typer.Option( + ..., + "--config", + "-c", + help="nebari configuration yaml file path", + ), + output: str = typer.Option( + "./", + "-o", + "--output", + help="output directory", + ), + dns_provider: str = typer.Option( + False, + "--dns-provider", + help="dns provider to use for registering domain name mapping", + ), + dns_auto_provision: bool = typer. + Option( + False, + "--dns-auto-provision", + help= + "Attempt to automatically provision DNS, currently only available for `cloudflare`", + ), + disable_prompt: bool = typer.Option( + False, + "--disable-prompt", + help="Disable human intervention", + ), + disable_render: bool = typer.Option( + False, + "--disable-render", + help="Disable auto-rendering in deploy stage", + ), + disable_checks: bool = typer.Option( + False, + "--disable-checks", + help="Disable the checks performed after each stage", + ), + skip_remote_state_provision: bool = typer. + Option( + False, + "--skip-remote-state-provision", + help= + "Skip terraform state deployment which is often required in CI once the terraform remote state bootstrapping phase is complete", + )) +``` + +Deploy the Nebari cluster from your \[purple\]nebari-config.yaml\[/purple\] file. + + + +#### destroy + +```python +@app.command() +def destroy( + config: str = typer.Option(..., + "-c", + "--config", + help="nebari configuration file path"), + output: str = typer.Option( + "./", + "-o", + "--output", + help="output directory", + ), + disable_render: bool = typer.Option( + False, + "--disable-render", + help="Disable auto-rendering before destroy", + ), + disable_prompt: bool = typer. + Option( + False, + "--disable-prompt", + help= + "Destroy entire Nebari cluster without confirmation request. Suggested for CI use.", + )) +``` + +Destroy the Nebari cluster from your \[purple\]nebari-config.yaml\[/purple\] file. + + + +#### cost + +```python +@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) +def cost(path: str = typer.Option( + None, + "-p", + "--path", + help= + "Pass the path of your stages directory generated after rendering Nebari configurations before deployment", +), + dashboard: bool = typer.Option( + True, + "-d", + "--dashboard", + help="Enable the cost dashboard", + ), + file: str = typer.Option( + None, + "-f", + "--file", + help="Specify the path of the file to store the cost report", + ), + currency: str = typer.Option( + "USD", + "-c", + "--currency", + help="Specify the currency code to use in the cost report", + ), + compare: bool = typer.Option( + False, + "-cc", + "--compare", + help="Compare the cost report to a previously generated report", + )) +``` + +Estimate the cost of deploying Nebari based on your \[purple\]nebari-config.yaml\[/purple\]. +\[italic\]Experimental.\[/italic\] + +\[italic\]This is still only experimental using Infracost under the hood. The estimated value is a base cost and does +not include usage costs.\[/italic\] + + + +#### upgrade + +```python +@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) +def upgrade( + config: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", + ), + attempt_fixes: bool = typer. + Option( + False, + "--attempt-fixes", + help= + "Attempt to fix the config for any incompatibilities between your old and new Nebari versions.", + )) +``` + +Upgrade your \[purple\]nebari-config.yaml\[/purple\] from pre-0.4.0 to 0.4.0. + +Due to several breaking changes that came with the 0.4.0 release, this utility is available to help update your +\[purple\]nebari-config.yaml\[/purple\] to comply with the introduced changes. See the project +\[green\]RELEASE.md\[/green\] for details. + + + +#### support + +```python +@app.command(rich_help_panel=SECOND_COMMAND_GROUP_NAME) +def support(config_filename: str = typer.Option( + ..., + "-c", + "--config", + help="nebari configuration file path", +), + output: str = typer.Option( + "./nebari-support-logs.zip", + "-o", + "--output", + help="output filename", + )) +``` + +Support tool to write all Kubernetes logs locally and compress them into a zip file. + +The Nebari team recommends k9s to manage and inspect the state of the cluster. However, this command occasionally +helpful for debugging purposes should the logs need to be shared. diff --git a/docs/api_docs/api_doc_mods.py b/docs/api_docs/api_doc_mods.py new file mode 100644 index 0000000000..c31fc8a51b --- /dev/null +++ b/docs/api_docs/api_doc_mods.py @@ -0,0 +1,117 @@ +from copy import deepcopy +from pathlib import Path + +import click + + +def insert_lines(source_list: list, idx: int, lines: list) -> None: + """Inject a list of lines at a given index""" + lines_r = deepcopy(lines) + lines_r.reverse() + [source_list.insert(idx, line) for line in lines_r] + + +def modify_autogen_api_docs(autogen_path: Path, outpath: Path) -> None: + """Modifications to the auto-generated API docs in order to make them + presentable. + + Args: + autogen_path (Path): path to the autogenerated API docs (*.md) + outpath (Path): output path for the modified docs (*.md) + """ + if not isinstance(autogen_path, Path): + autogen_path = Path(autogen_path) + + if not isinstance(outpath, Path): + outpath = Path(outpath) + + with open(autogen_path) as f: + text = f.readlines() + + header_info = [ + "---\n", + "title: API Documentation\n", + "description: Reference to API docs\n", + "---\n", + "<--- DO NOT EDIT, THIS PAGE IS AUTOGENERATED --->", + "# API Documentation\n", + "## Table of Contents\n", + ] + + # remove Table of contents header + del text[0] + + # insert new header + insert_lines(text, 0, header_info) + + # fix header flags (otherwise page nav won't work) + for idx, line in enumerate(text): + if line[0:7] == "# nebari.": + text[idx] = "##" + line + elif line == "# nebari\n": + text[idx] = "## nebari\n" + + keep_docs = [ + '"nebari.initialize', + '"nebari.cli.deploy', + '"nebari.cli.destroy', + '"nebari.cli.initialize', + '"nebari.cli.render', + '"nebari.cli.support', + '"nebari.cli', + '"nebari.cli.upgrade', + '"nebari.cli.validate', + '"nebari.render', + '"nebari.destroy', + ] + + id_idx = [] + keep_idx = [] + for idx, line in enumerate(text): + if line.startswith("