Merge pull request #1387 from consideRatio/pr/add-black

pre-commit: add and apply black, a Python autoformatter

.flake8

@@ -1,5 +1,4 @@
 [flake8]
-ignore = W605,W503,W504,H306,H238,H301,H202
-max-line-length = 120
-per-file-ignores =
-    test/test_packages.py:E501
+max-line-length = 88
+select = C,E,F,W,B,B950
+extend-ignore = E203, E501

.pre-commit-config.yaml

@@ -3,47 +3,58 @@ ci:
   skip: [hadolint-docker]
 
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+  # Autoformat: Python code
+  - repo: https://github.com/ambv/black
+    rev: 21.6b0
     hooks:
-      - id: check-yaml
-        files: .*\.(yaml|yml)$
+      - id: black
+        args: [--target-version=py39]
+
+  # Autoformat: YAML, JSON, Markdown, etc.
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.3.2
+    hooks:
+      - id: prettier
+
+  # Lint: Dockerfile
   - repo: https://github.com/hadolint/hadolint.git
     rev: v2.5.0
     hooks:
       - id: hadolint-docker
         # FIXME: remove after https://github.com/hadolint/hadolint/issues/628 is resolved
         entry: hadolint/hadolint:v2.5.0 hadolint
+
+  # Lint: YAML
   - repo: https://github.com/adrienverge/yamllint.git
     rev: v1.26.1
     hooks:
       - id: yamllint
         args: ["-d {extends: relaxed, rules: {line-length: disable}}", "-s"]
         files: \.(yaml|yml)$
+
+  # Lint: Bash scripts
   - repo: https://github.com/openstack-dev/bashate.git
     rev: 2.0.0
     hooks:
       - id: bashate
         args: ["--ignore=E006"]
+
+  # Lint: Shell scripts
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.7.2.1
     hooks:
       - id: shellcheck
         args: ["-x"]
+
+  # Lint: Python
   - repo: https://github.com/PyCQA/flake8
     rev: 3.9.2
     hooks:
       - id: flake8
-  - repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v1.5.7
-    hooks:
-      - id: autopep8
+
+  # Lint: Markdown
   - repo: https://github.com/igorshubovych/markdownlint-cli
     rev: v0.27.1
     hooks:
       - id: markdownlint
         args: ["--fix"]
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.3.2
-    hooks:
-      - id: prettier

all-spark-notebook/test/test_spark_notebooks.py

@@ -22,10 +22,12 @@ def test_nbconvert(container, test_file):
     output_dir = "/tmp"
     timeout_ms = 600
     LOGGER.info(f"Test that {test_file} notebook can be executed ...")
-    command = "jupyter nbconvert --to markdown " + \
-        f"--ExecutePreprocessor.timeout={timeout_ms} " + \
-        f"--output-dir {output_dir} " + \
-        f"--execute {cont_data_dir}/{test_file}.ipynb"
+    command = (
+        "jupyter nbconvert --to markdown "
+        + f"--ExecutePreprocessor.timeout={timeout_ms} "
+        + f"--output-dir {output_dir} "
+        + f"--execute {cont_data_dir}/{test_file}.ipynb"
+    )
     c = container.run(
         volumes={host_data_dir: {"bind": cont_data_dir, "mode": "ro"}},
         tty=True,

base-notebook/jupyter_notebook_config.py

@@ -8,17 +8,17 @@
 import stat
 
 c = get_config()  # noqa: F821
-c.NotebookApp.ip = '0.0.0.0'
+c.NotebookApp.ip = "0.0.0.0"
 c.NotebookApp.port = 8888
 c.NotebookApp.open_browser = False
 
 # https://github.com/jupyter/notebook/issues/3130
 c.FileContentsManager.delete_to_trash = False
 
 # Generate a self-signed certificate
-if 'GEN_CERT' in os.environ:
+if "GEN_CERT" in os.environ:
     dir_name = jupyter_data_dir()
-    pem_file = os.path.join(dir_name, 'notebook.pem')
+    pem_file = os.path.join(dir_name, "notebook.pem")
     try:
         os.makedirs(dir_name)
     except OSError as exc:  # Python >2.5
@@ -28,28 +28,37 @@
             raise
 
     # Generate an openssl.cnf file to set the distinguished name
-    cnf_file = os.path.join(os.getenv('CONDA_DIR', '/usr/lib'), 'ssl', 'openssl.cnf')
+    cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf")
     if not os.path.isfile(cnf_file):
-        with open(cnf_file, 'w') as fh:
-            fh.write('''\
+        with open(cnf_file, "w") as fh:
+            fh.write(
+                """\
 [req]
 distinguished_name = req_distinguished_name
 [req_distinguished_name]
-''')
+"""
+            )
 
     # Generate a certificate if one doesn't exist on disk
-    subprocess.check_call(['openssl', 'req', '-new',
-                           '-newkey', 'rsa:2048',
-                           '-days', '365',
-                           '-nodes', '-x509',
-                           '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated',
-                           '-keyout', pem_file,
-                           '-out', pem_file])
+    subprocess.check_call(
+        [
+            "openssl",
+            "req",
+            "-new",
+            "-newkey=rsa:2048",
+            "-days=365",
+            "-nodes",
+            "-x509",
+            "-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated",
+            f"-keyout={pem_file}",
+            f"-out={pem_file}",
+        ]
+    )
     # Restrict access to the file
     os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR)
     c.NotebookApp.certfile = pem_file
 
 # Change default umask for all subprocesses of the notebook server if set in
 # the environment
-if 'NB_UMASK' in os.environ:
-    os.umask(int(os.environ['NB_UMASK'], 8))
+if "NB_UMASK" in os.environ:
+    os.umask(int(os.environ["NB_UMASK"], 8))

base-notebook/test/test_container_options.py

@@ -11,59 +11,55 @@
 def test_cli_args(container, http_client):
     """Container should respect notebook server command line args
     (e.g., disabling token security)"""
-    c = container.run(
-        command=["start-notebook.sh", "--NotebookApp.token=''"]
-    )
-    resp = http_client.get('http://localhost:8888')
+    c = container.run(command=["start-notebook.sh", "--NotebookApp.token=''"])
+    resp = http_client.get("http://localhost:8888")
     resp.raise_for_status()
-    logs = c.logs(stdout=True).decode('utf-8')
+    logs = c.logs(stdout=True).decode("utf-8")
     LOGGER.debug(logs)
-    assert 'login_submit' not in resp.text
+    assert "login_submit" not in resp.text
 
 
-@pytest.mark.filterwarnings('ignore:Unverified HTTPS request')
+@pytest.mark.filterwarnings("ignore:Unverified HTTPS request")
 def test_unsigned_ssl(container, http_client):
     """Container should generate a self-signed SSL certificate
     and notebook server should use it to enable HTTPS.
     """
-    container.run(
-        environment=['GEN_CERT=yes']
-    )
+    container.run(environment=["GEN_CERT=yes"])
     # NOTE: The requests.Session backing the http_client fixture does not retry
     # properly while the server is booting up. An SSL handshake error seems to
     # abort the retry logic. Forcing a long sleep for the moment until I have
     # time to dig more.
     time.sleep(5)
-    resp = http_client.get('https://localhost:8888', verify=False)
+    resp = http_client.get("https://localhost:8888", verify=False)
     resp.raise_for_status()
-    assert 'login_submit' in resp.text
+    assert "login_submit" in resp.text
 
 
 def test_uid_change(container):
     """Container should change the UID of the default user."""
     c = container.run(
         tty=True,
-        user='root',
-        environment=['NB_UID=1010'],
-        command=['start.sh', 'bash', '-c', 'id && touch /opt/conda/test-file']
+        user="root",
+        environment=["NB_UID=1010"],
+        command=["start.sh", "bash", "-c", "id && touch /opt/conda/test-file"],
     )
     # usermod is slow so give it some time
     c.wait(timeout=120)
-    assert 'uid=1010(jovyan)' in c.logs(stdout=True).decode('utf-8')
+    assert "uid=1010(jovyan)" in c.logs(stdout=True).decode("utf-8")
 
 
 def test_gid_change(container):
     """Container should change the GID of the default user."""
     c = container.run(
         tty=True,
-        user='root',
-        environment=['NB_GID=110'],
-        command=['start.sh', 'id']
+        user="root",
+        environment=["NB_GID=110"],
+        command=["start.sh", "id"],
     )
     c.wait(timeout=10)
-    logs = c.logs(stdout=True).decode('utf-8')
-    assert 'gid=110(jovyan)' in logs
-    assert 'groups=110(jovyan),100(users)' in logs
+    logs = c.logs(stdout=True).decode("utf-8")
+    assert "gid=110(jovyan)" in logs
+    assert "groups=110(jovyan),100(users)" in logs
 
 
 def test_nb_user_change(container):
@@ -72,11 +68,8 @@ def test_nb_user_change(container):
     running_container = container.run(
         tty=True,
         user="root",
-        environment=[
-            f"NB_USER={nb_user}",
-            "CHOWN_HOME=yes"
-        ],
-        command=['start.sh', 'bash', '-c', 'sleep infinity']
+        environment=[f"NB_USER={nb_user}", "CHOWN_HOME=yes"],
+        command=["start.sh", "bash", "-c", "sleep infinity"],
     )
 
     # Give the chown time to complete. Use sleep, not wait, because the
@@ -98,90 +91,99 @@ def test_nb_user_change(container):
     expected_output = f"{nb_user} users"
     cmd = running_container.exec_run(command, workdir=f"/home/{nb_user}")
     output = cmd.output.decode("utf-8").strip("\n")
-    assert output == expected_output, f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}"
+    assert (
+        output == expected_output
+    ), f"Bad owner for the {nb_user} home folder {output}, expected {expected_output}"
 
 
 def test_chown_extra(container):
     """Container should change the UID/GID of CHOWN_EXTRA."""
     c = container.run(
         tty=True,
-        user='root',
+        user="root",
         environment=[
-            'NB_UID=1010',
-            'NB_GID=101',
-            'CHOWN_EXTRA=/opt/conda',
-            'CHOWN_EXTRA_OPTS=-R'
+            "NB_UID=1010",
+            "NB_GID=101",
+            "CHOWN_EXTRA=/opt/conda",
+            "CHOWN_EXTRA_OPTS=-R",
         ],
-        command=['start.sh', 'bash', '-c', 'stat -c \'%n:%u:%g\' /opt/conda/LICENSE.txt']
+        command=["start.sh", "bash", "-c", "stat -c '%n:%u:%g' /opt/conda/LICENSE.txt"],
     )
     # chown is slow so give it some time
     c.wait(timeout=120)
-    assert '/opt/conda/LICENSE.txt:1010:101' in c.logs(stdout=True).decode('utf-8')
+    assert "/opt/conda/LICENSE.txt:1010:101" in c.logs(stdout=True).decode("utf-8")
 
 
 def test_chown_home(container):
     """Container should change the NB_USER home directory owner and
     group to the current value of NB_UID and NB_GID."""
     c = container.run(
         tty=True,
-        user='root',
-        environment=[
-            'CHOWN_HOME=yes',
-            'CHOWN_HOME_OPTS=-R'
+        user="root",
+        environment=["CHOWN_HOME=yes", "CHOWN_HOME_OPTS=-R"],
+        command=[
+            "start.sh",
+            "bash",
+            "-c",
+            "chown root:root /home/jovyan && ls -alsh /home",
         ],
-        command=['start.sh', 'bash', '-c', 'chown root:root /home/jovyan && ls -alsh /home']
     )
     c.wait(timeout=120)
-    assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(stdout=True).decode('utf-8')
+    assert "Changing ownership of /home/jovyan to 1000:100 with options '-R'" in c.logs(
+        stdout=True
+    ).decode("utf-8")
 
 
 def test_sudo(container):
     """Container should grant passwordless sudo to the default user."""
     c = container.run(
         tty=True,
-        user='root',
-        environment=['GRANT_SUDO=yes'],
-        command=['start.sh', 'sudo', 'id']
+        user="root",
+        environment=["GRANT_SUDO=yes"],
+        command=["start.sh", "sudo", "id"],
     )
     rv = c.wait(timeout=10)
     assert rv == 0 or rv["StatusCode"] == 0
-    assert 'uid=0(root)' in c.logs(stdout=True).decode('utf-8')
+    assert "uid=0(root)" in c.logs(stdout=True).decode("utf-8")
 
 
 def test_sudo_path(container):
     """Container should include /opt/conda/bin in the sudo secure_path."""
     c = container.run(
         tty=True,
-        user='root',
-        environment=['GRANT_SUDO=yes'],
-        command=['start.sh', 'sudo', 'which', 'jupyter']
+        user="root",
+        environment=["GRANT_SUDO=yes"],
+        command=["start.sh", "sudo", "which", "jupyter"],
     )
     rv = c.wait(timeout=10)
     assert rv == 0 or rv["StatusCode"] == 0
-    assert c.logs(stdout=True).decode('utf-8').rstrip().endswith('/opt/conda/bin/jupyter')
+    logs = c.logs(stdout=True).decode("utf-8")
+    assert logs.rstrip().endswith("/opt/conda/bin/jupyter")
 
 
 def test_sudo_path_without_grant(container):
     """Container should include /opt/conda/bin in the sudo secure_path."""
     c = container.run(
         tty=True,
-        user='root',
-        command=['start.sh', 'which', 'jupyter']
+        user="root",
+        command=["start.sh", "which", "jupyter"],
     )
     rv = c.wait(timeout=10)
     assert rv == 0 or rv["StatusCode"] == 0
-    assert c.logs(stdout=True).decode('utf-8').rstrip().endswith('/opt/conda/bin/jupyter')
+    logs = c.logs(stdout=True).decode("utf-8")
+    assert logs.rstrip().endswith("/opt/conda/bin/jupyter")
 
 
 def test_group_add(container, tmpdir):
     """Container should run with the specified uid, gid, and secondary
     group.
     """
     c = container.run(
-        user='1010:1010',
-        group_add=['users'],
-        command=['start.sh', 'id']
+        user="1010:1010",
+        group_add=["users"],
+        command=["start.sh", "id"],
     )
     rv = c.wait(timeout=5)
     assert rv == 0 or rv["StatusCode"] == 0
-    assert 'uid=1010 gid=1010 groups=1010,100(users)' in c.logs(stdout=True).decode('utf-8')
+    logs = c.logs(stdout=True).decode("utf-8")
+    assert "uid=1010 gid=1010 groups=1010,100(users)" in logs

base-notebook/test/test_package_managers.py

@@ -24,7 +24,7 @@ def test_package_manager(container, package_manager, version_arg):
     )
     c = container.run(
         tty=True,
-        command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"]
+        command=["start.sh", "bash", "-c", f"{package_manager} {version_arg}"],
     )
     rv = c.wait(timeout=5)
     logs = c.logs(stdout=True).decode("utf-8")