ENH - Switch to ruff and pre-commit.ci (#1602)

.github/workflows/run-pre-commit.yaml → .github/workflows/run-precommit.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - develop
       - release/\d{4}.\d{1,2}.\d{1,2}
   pull_request:
 
@@ -15,11 +16,14 @@ jobs:
       run:
         shell: bash -l {0}
     steps:
-      - name: Checkout repository
+      - name: Checkout repository 🔔
         uses: actions/checkout@v3
 
-      - name: Install pre-commit
+      # https://github.com/pre-commit/action to enable cache
+      - uses: pre-commit/action@v3.0.0
+
+      - name: Install pre-commit 📦
         run: pip install pre-commit
 
-      - name: Run pre-commit
-        run: pre-commit run --all
+      - name: Run terraform pre-commit ⚡️
+        run: pre-commit run terraform_fmt

.gitignore

@@ -44,3 +44,5 @@ data*
 nebari-config.yaml
 
 .vscode/
+
+/.ruff_cache

.pre-commit-config.yaml

@@ -11,6 +11,13 @@
 # These pre-commit hooks are run as CI.
 #
 # NOTE: if it can be avoided, add configs/args in pyproject.toml, setup.cfg or below instead of creating a new `.config.file`.
+# https://pre-commit.ci/#configuration
+ci:
+  autoupdate_schedule: monthly
+  autofix_commit_msg: |
+    [pre-commit.ci] Apply automatic pre-commit fixes
+  # this does not work on pre-commit ci
+  skip: [terraform_fmt]
 
 repos:
   # general
@@ -44,16 +51,11 @@ repos:
     hooks:
       - id: black
 
-  - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: v0.0.178
     hooks:
-      - id: flake8
-        args:
-          [
-            "--builtins=c",
-            "--ignore=E203,E266,E501,W503",
-            "--exclude=[.git,__pycache__,docs/source/conf.py,nebari/template,build,dist,docs,home]",
-          ]
+      - id: ruff
+        args: ["--fix"]
 
   - repo: https://github.com/pycqa/isort
     rev: 5.10.1
@@ -66,7 +68,7 @@ repos:
 
   # terraform
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.76.0
+    rev: v1.77.0
     hooks:
       - id: terraform_fmt
         args:

docs/ext/substitute.py

@@ -1,12 +1,12 @@
 """
 This is a hard replace as soon as the source file is read, so no respect for any markup at all
-Simply forces replace of ||QHUB_VERSION|| with the qhub_version_string in conf.py
+Simply forces replace of ||QHUB_VERSION|| with the qhub_version_string in conf.py.
 """
 
 
 def dosubs(app, docname, source):
     """
-    Replace QHUB_VERSION with the qhub version
+    Replace QHUB_VERSION with the qhub version.
     """
     if app.config.qhub_version_string != "":
         src = source[0]

nebari/cli/init.py

@@ -90,7 +90,6 @@ def handle_init(inputs: InitInputs):
 
 def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str):
     """Validate that the necessary cloud credentials have been set as environment variables."""
-
     if ctx.params.get("disable_prompt"):
         return cloud_provider
 
@@ -197,7 +196,6 @@ def check_cloud_provider_creds(ctx: typer.Context, cloud_provider: str):
 
 def check_auth_provider_creds(ctx: typer.Context, auth_provider: str):
     """Validating the the necessary auth provider credentials have been set as environment variables."""
-
     if ctx.params.get("disable_prompt"):
         return auth_provider
 
@@ -254,7 +252,6 @@ def check_auth_provider_creds(ctx: typer.Context, auth_provider: str):
 
 def check_project_name(ctx: typer.Context, project_name: str):
     """Validate the project_name is acceptable. Depends on `cloud_provider`."""
-
     project_name_convention(
         project_name.lower(), {"provider": ctx.params["cloud_provider"]}
     )

nebari/cli/main.py

@@ -405,7 +405,7 @@ def cost(
     ),
 ):
     """
-    Estimate the cost of deploying Nebari based on your [purple]nebari-config.yaml[/purple]. [italic]Experimental.[/italic]
+    Estimate the cost of deploying Nebari based on your [purple]nebari-config.yaml[/purple]. [italic]Experimental.[/italic].
 
     [italic]This is still only experimental using Infracost under the hood.
     The estimated value is a base cost and does not include usage costs.[/italic]
@@ -470,7 +470,6 @@ def support(
     The Nebari team recommends k9s to manage and inspect the state of the cluster.
     However, this command occasionally helpful for debugging purposes should the logs need to be shared.
     """
-
     kube_config.load_kube_config()
 
     v1 = client.CoreV1Api()

nebari/cost.py

@@ -27,7 +27,7 @@
 
 def _check_infracost():
     """
-    Check if infracost is installed
+    Check if infracost is installed.
     """
     try:
         subprocess.check_output(["infracost", "--version"])
@@ -38,7 +38,7 @@ def _check_infracost():
 
 def _check_infracost_api_key():
     """
-    Check if infracost API key is configured
+    Check if infracost API key is configured.
     """
     try:
         subprocess.check_output(["infracost", "configure", "get", "api_key"])
@@ -49,7 +49,7 @@ def _check_infracost_api_key():
 
 def _set_currency_code(currency_code):
     """
-    Specify the currency code for infracost
+    Specify the currency code for infracost.
     """
     try:
         subprocess.check_output(
@@ -62,7 +62,7 @@ def _set_currency_code(currency_code):
 
 def _run_infracost(path):
     """
-    Run infracost on the given path and return the JSON output
+    Run infracost on the given path and return the JSON output.
     """
     try:
         process = subprocess.Popen(
@@ -86,7 +86,7 @@ def _run_infracost(path):
 
 def _enable_infracost_dashboard():
     """
-    Enable infracost dashboard
+    Enable infracost dashboard.
     """
     try:
         subprocess.check_output(
@@ -99,7 +99,7 @@ def _enable_infracost_dashboard():
 
 def _disable_infracost_dashboard():
     """
-    Disable infracost dashboard
+    Disable infracost dashboard.
     """
     try:
         subprocess.check_output(
@@ -112,7 +112,7 @@ def _disable_infracost_dashboard():
 
 def infracost_diff(path, compare_to_path):
     """
-    Compare the infracost report of the given path to a previous infracost report
+    Compare the infracost report of the given path to a previous infracost report.
     """
     try:
         process = subprocess.Popen(
@@ -145,7 +145,7 @@ def infracost_report(path, dashboard, file, currency_code, compare):
     """
     Generate a report of the infracost cost of the given path
     args:
-        path: path to the nebari stages directory
+        path: path to the nebari stages directory.
     """
     console = Console()
     # If path is not provided, use the current directory with `stages` subdirectory

nebari/deploy.py

@@ -31,7 +31,7 @@ def provision_01_terraform_state(stage_outputs, config):
 
 
 def provision_02_infrastructure(stage_outputs, config, disable_checks=False):
-    """Generalized method to provision infrastructure
+    """Generalized method to provision infrastructure.
 
     After successful deployment the following properties are set on
     `stage_outputs[directory]`.

nebari/provider/cloud/amazon_web_services.py

@@ -27,7 +27,6 @@ def zones(region):
 @functools.lru_cache()
 def kubernetes_versions(region="us-west-2"):
     """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest."""
-
     # AWS SDK (boto3) currently doesn't offer an intuitive way to list available kubernetes version. This implementation grabs kubernetes versions for specific EKS addons. It will therefore always be (at the very least) a subset of all kubernetes versions still supported by AWS.
     if not os.getenv("AWS_DEFAULT_REGION"):
         os.environ["AWS_DEFAULT_REGION"] = region

nebari/provider/cloud/azure_cloud.py

@@ -29,7 +29,6 @@ def initiate_container_service_client():
 @functools.lru_cache()
 def kubernetes_versions(region="Central US"):
     """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest."""
-
     client = initiate_container_service_client()
     azure_location = region.replace(" ", "").lower()
 

nebari/provider/cloud/digital_ocean.py

@@ -44,7 +44,6 @@ def regions():
 # keep `region` parameter
 def kubernetes_versions(region=None):
     """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest."""
-
     supported_kubernetes_versions = sorted(
         [_["slug"] for _ in _kubernetes_options()["options"]["versions"]]
     )

nebari/provider/cloud/google_cloud.py

@@ -33,7 +33,6 @@ def zones(project, region):
 @functools.lru_cache()
 def kubernetes_versions(region):
     """Return list of available kubernetes supported by cloud provider. Sorted from oldest to latest."""
-
     output = subprocess.check_output(
         [
             "gcloud",

nebari/provider/terraform.py

@@ -31,7 +31,7 @@ def deploy(
     input_vars: Dict[str, Any] = None,
     state_imports: List = None,
 ):
-    """Execute a given terraform directory
+    """Execute a given terraform directory.
 
     Parameters:
       directory: directory in which to run terraform operations on

nebari/render.py

@@ -149,8 +149,7 @@ def render_template(output_directory, config_filename, force=False, dry_run=Fals
 
 
 def render_contents(config: Dict):
-    """Dynamically generated contents from Nebari configuration"""
-
+    """Dynamically generated contents from Nebari configuration."""
     contents = {
         **tf_objects.stage_01_terraform_state(config),
         **tf_objects.stage_02_infrastructure(config),
@@ -185,7 +184,6 @@ def gen_gitignore(config):
     Generate `.gitignore` file.
     Add files as needed.
     """
-
     from inspect import cleandoc
 
     filestoignore = """
@@ -239,7 +237,7 @@ def inspect_files(
     deleted_paths: List[str] = None,
     contents: Dict[str, str] = None,
 ):
-    """Return created, updated and untracked files by computing a checksum over the provided directory
+    """Return created, updated and untracked files by computing a checksum over the provided directory.
 
     Args:
         source_dirs (str): The source dir used as base for comparssion
@@ -302,7 +300,7 @@ def list_files(
 
 
 def hash_file(file_path: str):
-    """Get the hex digest of the given file
+    """Get the hex digest of the given file.
 
     Args:
         file_path (str): path to file

nebari/schema.py

@@ -396,7 +396,7 @@ class Profiles(Base):
 
     @validator("jupyterlab")
     def check_default(cls, v, values):
-        """Check if only one default value is present"""
+        """Check if only one default value is present."""
         default = [attrs["default"] for attrs in v if "default" in attrs]
         if default.count(True) > 1:
             raise TypeError(

nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/dask-gateway/files/gateway_config.py

@@ -300,7 +300,7 @@ def deep_merge(d1, d2):
     'b': {'c': 1, 'z': [5, 6]},
     'e': {'f': {'g': {}}},
     'm': 1,
-    }
+    }.
 
     >>> value_2 = {
         'a': [3, 4],

nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py

@@ -8,7 +8,7 @@
 
 
 def base_profile_home_mounts(username):
-    """Configure the home directory mount for user
+    """Configure the home directory mount for user.
 
     Ensure that user directory exists and user has permissions to
     read/write/execute.
@@ -72,7 +72,7 @@ def base_profile_home_mounts(username):
 
 
 def base_profile_shared_mounts(groups):
-    """Configure the group directory mounts for user
+    """Configure the group directory mounts for user.
 
     Ensure that {shared}/{group} directory exists and user has
     permissions to read/write/execute. Kubernetes does not allow the
@@ -133,7 +133,7 @@ def base_profile_shared_mounts(groups):
 
 def profile_conda_store_mounts(username, groups):
     """Configure the conda_store environment directories mounts for
-    user
+    user.
 
     Ensure that {shared}/{group} directory exists and user has
     permissions to read/write/execute.
@@ -304,7 +304,7 @@ def configure_user(username, groups, uid=1000, gid=100):
 
 
 def render_profile(profile, username, groups, keycloak_profilenames):
-    """Render each profile for user
+    """Render each profile for user.
 
     If profile is not available for given username, groups returns
     None. Otherwise profile is transformed into kubespawner profile.
@@ -407,7 +407,7 @@ def deep_merge(d1, d2):
     'b': {'c': 1, 'z': [5, 6]},
     'e': {'f': {'g': {}}},
     'm': 1,
-    }
+    }.
 
     >>> value_2 = {
         'a': [3, 4],

nebari/upgrade.py

@@ -123,7 +123,7 @@ def requires_nebari_version_field(self):
 
     def upgrade_step(self, config, start_version, config_filename, *args, **kwargs):
         """
-        Perform the upgrade from start_version to self.version
+        Perform the upgrade from start_version to self.version.
 
         Generally, this will be in-place in config, but must also return config dict.
 
@@ -135,7 +135,6 @@ def upgrade_step(self, config, start_version, config_filename, *args, **kwargs):
         It should normally be left as-is for all upgrades. Use _version_specific_upgrade below
         for any actions that are only required for the particular upgrade you are creating.
         """
-
         finish_version = self.get_version()
         __rounded_finish_version__ = ".".join(
             [str(c) for c in rounded_ver_parse(finish_version)]
@@ -206,7 +205,7 @@ def _version_specific_upgrade(
         self, config, start_version, config_filename, *args, **kwargs
     ):
         """
-        Override this method in subclasses if you need to do anything specific to your version
+        Override this method in subclasses if you need to do anything specific to your version.
         """
         return config
 

nebari/version.py

@@ -1,4 +1,4 @@
-"""a backport for the nebari version references"""
+"""a backport for the nebari version references."""
 
 import re