diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 95a7d9c9ca..b80bd395f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -99,6 +99,25 @@ repos: - id: markdownlint args: ["--fix"] + # Run tools on Jupyter notebooks + + # strip output from Jupyter notebooks + - repo: https://github.com/kynan/nbstripout + rev: 0.5.0 + hooks: + - id: nbstripout + + # nbQA provides tools from the Python ecosystem like + # pyupgrade, black, and flake8, adjusted for notebooks. + - repo: https://github.com/nbQA-dev/nbQA + rev: 1.3.1 + hooks: + - id: nbqa-pyupgrade + args: [--py39-plus] + - id: nbqa-black + args: [--target-version=py39] + - id: nbqa-flake8 + # Docker hooks do not work in pre-commit.ci # See: ci: diff --git a/binder/README.ipynb b/binder/README.ipynb index 0ee8c1c44c..7f6059cbff 100644 --- a/binder/README.ipynb +++ b/binder/README.ipynb @@ -18,7 +18,10 @@ "outputs": [], "source": [ "import os\n", - "print(f'This container is using tag {os.environ[\"TAG\"]} of the jupyter/base-notebook image')" + "\n", + "print(\n", + " f'This container is using tag {os.environ[\"TAG\"]} of the jupyter/base-notebook image'\n", + ")" ] }, { @@ -111,7 +114,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -125,9 +128,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.1" + "version": "3.9.10" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/tests/all-spark-notebook/data/issue_1168.ipynb b/tests/all-spark-notebook/data/issue_1168.ipynb index 52b56f5930..4d3437185d 100644 --- a/tests/all-spark-notebook/data/issue_1168.ipynb +++ b/tests/all-spark-notebook/data/issue_1168.ipynb @@ -9,48 +9,37 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from pyspark.sql import SparkSession\n", - "from pyspark.sql.functions import pandas_udf\n", "\n", "# Spark session & context\n", - "spark = SparkSession.builder.master('local').getOrCreate()" + "spark = SparkSession.builder.master(\"local\").getOrCreate()" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "+---+---+\n", - "| id|age|\n", - "+---+---+\n", - "| 1| 21|\n", - "+---+---+\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "df = spark.createDataFrame([(1, 21), (2, 30)], (\"id\", \"age\"))\n", + "\n", + "\n", "def filter_func(iterator):\n", " for pdf in iterator:\n", " yield pdf[pdf.id == 1]\n", "\n", + "\n", "df.mapInPandas(filter_func, df.schema).show()" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -64,7 +53,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.6" + "version": "3.9.10" } }, "nbformat": 4, diff --git a/tests/all-spark-notebook/data/local_pyspark.ipynb b/tests/all-spark-notebook/data/local_pyspark.ipynb index 85be3b89c0..fb379ae94e 100644 --- a/tests/all-spark-notebook/data/local_pyspark.ipynb +++ b/tests/all-spark-notebook/data/local_pyspark.ipynb @@ -2,31 +2,14 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "output_type": "error", - "ename": "Error", - "evalue": "Jupyter cannot be started. Error attempting to locate jupyter: Data Science libraries jupyter and notebook are not installed in interpreter Python 3.7.7 64-bit ('jupyter': conda).", - "traceback": [ - "Error: Jupyter cannot be started. Error attempting to locate jupyter: Data Science libraries jupyter and notebook are not installed in interpreter Python 3.7.7 64-bit ('jupyter': conda).", - "at b.startServer (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:92:270430)", - "at async b.createServer (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:92:269873)", - "at async connect (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:92:397876)", - "at async w.ensureConnectionAndNotebookImpl (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:16:556625)", - "at async w.ensureConnectionAndNotebook (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:16:556303)", - "at async w.clearResult (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:16:552346)", - "at async w.reexecuteCell (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:16:540374)", - "at async w.reexecuteCells (/Users/romain/.vscode/extensions/ms-python.python-2020.5.80290/out/client/extension.js:16:537541)" - ] - } - ], + "outputs": [], "source": [ "from pyspark.sql import SparkSession\n", "\n", "# Spark session & context\n", - "spark = SparkSession.builder.master('local').getOrCreate()\n", + "spark = SparkSession.builder.master(\"local\").getOrCreate()\n", "sc = spark.sparkContext\n", "\n", "# Sum of the first 100 whole numbers\n", @@ -38,7 +21,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -52,7 +35,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.9.10" } }, "nbformat": 4, diff --git a/tests/all-spark-notebook/data/local_spylon.ipynb b/tests/all-spark-notebook/data/local_spylon.ipynb index 7545d99c43..94f3c5b55b 100644 --- a/tests/all-spark-notebook/data/local_spylon.ipynb +++ b/tests/all-spark-notebook/data/local_spylon.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -14,21 +14,9 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "rdd: org.apache.spark.rdd.RDD[Int] = ParallelCollectionRDD[8] at parallelize at :28\n", - "res4: Double = 5050.0\n" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "// Sum of the first 100 whole numbers\n", "val rdd = sc.parallelize(0 to 100)\n", diff --git a/tests/minimal-notebook/data/notebook_math.ipynb b/tests/minimal-notebook/data/notebook_math.ipynb index 3c400349d2..951f751ed0 100644 --- a/tests/minimal-notebook/data/notebook_math.ipynb +++ b/tests/minimal-notebook/data/notebook_math.ipynb @@ -1,149 +1,138 @@ { - "metadata": { - "name": "notebook1" - }, - "nbformat": 3, - "nbformat_minor": 0, - "worksheets": [ + "cells": [ + { + "cell_type": "markdown", + "id": "a69ceb22", + "metadata": {}, + "source": [ + "# A simple SymPy example" + ] + }, + { + "cell_type": "markdown", + "id": "3c43c88e", + "metadata": {}, + "source": [ + "First we import SymPy and initialize printing:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b561917", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "from sympy import init_printing, symbols, sin, diff, integrate" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6454356-d5a6-481f-aaac-9abcc101026a", + "metadata": {}, + "outputs": [], + "source": [ + "init_printing()" + ] + }, + { + "cell_type": "markdown", + "id": "fbe0a2f3", + "metadata": {}, + "source": [ + "Create a few symbols:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c99d7f17", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "x, y, z = symbols(\"x y z\")" + ] + }, + { + "cell_type": "markdown", + "id": "f61dddac", + "metadata": {}, + "source": [ + "Here is a basic expression:" + ] + }, { - "cells": [ - { - "cell_type": "heading", - "level": 1, - "metadata": {}, - "source": [ - "A simple SymPy example" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First we import SymPy and initialize printing:" - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "from sympy import init_printing\n", - "from sympy import *\n", - " init_printing()" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 2 - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a few symbols:" - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "x,y,z = symbols('x y z')" - ], - "language": "python", - "metadata": {}, - "outputs": [], - "prompt_number": 4 - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here is a basic expression:" - ] - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "e = x**2 + 2.0*y + sin(z); e" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "latex": [ - "$$x^{2} + 2.0 y + \\sin{\\left (z \\right )}$$" - ], - "metadata": {}, - "output_type": "pyout", - "png": "iVBORw0KGgoAAAANSUhEUgAAAKMAAAAZBAMAAACvE4OgAAAAMFBMVEX///8AAAAAAAAAAAAAAAAA\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEHarIkSJZt3NVLsy\nme8Q6PJIAAACz0lEQVRIDa1UTWjUQBT+ZpvdzW7TGlrxItjYSg/C6vbiDwjmoCgUpHioPYhdqig9\nFJYiPYmW4klB14NgFGnw4EHpj7UgUtTFXhSEBgVBxIOFggWVrrUqiMY3mZkkLNIK7oN575vvvfky\n8yYJIGzgkSlRrULKrivVSkvq6LbxtcaSjV3aSo0lgWyl5pK69V+SRlEsPxNTGYhhDrV3M2Ue2etc\nEDmuMmM+IjolrCuHXNoLoQDNSAXdzbjsfFVKTY1vCgFXFIxenG4cFSSzRewAPnN0FugXjPDr45MQ\nJwoKtitgXL9zT+CsJeIHYG+Z4H1gwhRU4G/FcAQbbYU3KdDo+0sCK8lRU0guA72uKqMYk9RehHxP\niDIu0NS2v90KGShJYi7T7tgvkrQ2vIT2XtRISWNra6lzGc8/PW3ji4PL7Vmge095YIX0iB71NCaZ\n5N3XyM0VCuNIyFNIyY3AMG/KDUvjn90DGmwq9wpIl5AyU5WsTYy0aJf6JFGB5An3Der5jExKHjNR\n4JKPge/EXqDBoOXpkxkmkJHFfAFRVhDIveWA0S57N2Me6yw+DSX1n1uCq3sIfCF2IcjNkjeWyKli\nginHubboOB4vSNAjyaiXE26ygrkyTfod55Lj3CTE+n2P73ImJpnk6wJJKjYJSwt3OQbNJu4icM5s\nKGGbzMuD70N6JSbJD44x7pLDyJrbkfiLpOEhYVMJSVEj83x5YFLyNrAzJsmvJ+uhLrieXvcJDshy\nHtQuD54c2IWWEnSXfUTDZJJfAjcpOW5imp9aHvw4ZZ4NDV4FGjw0tzadKgbFwinJUd//AT0P1tdW\nBtuRU39oKdk9ONQ163fM+nvu/s4D/FX30otdQIZGlSnJKpq6KUxKVqV1WxGHFIhishjhEO1Gi3r4\nkZCMg+hH1henV8EjmFoly1PTMs/Uadaox+FceY2STpmvt9co/Pe0Jvt1GvgDK/Osw/4jQ4wAAAAA\nSUVORK5CYII=\n", - "prompt_number": 6, - "text": [ - " 2 \n", - "x + 2.0\u22c5y + sin(z)" - ] - } - ], - "prompt_number": 6 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "diff(e, x)" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "latex": [ - "$$2 x$$" - ], - "metadata": {}, - "output_type": "pyout", - "png": "iVBORw0KGgoAAAANSUhEUgAAABQAAAAOBAMAAADd6iHDAAAAMFBMVEX///8AAAAAAAAAAAAAAAAA\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAIpm7MhCriUTv3c12\nVGZoascqAAAAgElEQVQIHWNgVDJ2YICAMAb2H1BmKgPDTChzFgNDvgOEvT8AzgQKrA9gPZPYUwNk\ncXxnCGd4dWA1kMllwFDKUB9wEchUZmAIYNgMZDDwJIDIPyDiEgOjAAPLFwZWBhYFBh6BqzwfGI4y\nSJUXZXH8Zf7A+IBh////v1hzjh5/xwAAW80hUDE8HYkAAAAASUVORK5CYII=\n", - "prompt_number": 7, - "text": [ - "2\u22c5x" - ] - } - ], - "prompt_number": 7 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [ - "integrate(e, z)" - ], - "language": "python", - "metadata": {}, - "outputs": [ - { - "latex": [ - "$$x^{2} z + 2.0 y z - \\cos{\\left (z \\right )}$$" - ], - "metadata": {}, - "output_type": "pyout", - "png": "iVBORw0KGgoAAAANSUhEUgAAALsAAAAZBAMAAACbakK8AAAAMFBMVEX///8AAAAAAAAAAAAAAAAA\nAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEHarIkSJZt3NVLsy\nme8Q6PJIAAADAklEQVRIDbVVS2gTURQ90/wmk0k6tCJCsR1SKShIsxE3CgNWBKUxq9qFmqFqShfF\nUKQrkaDiF0pcCKYgBBcuBLV+wIWKARe6kQ4UhNKKWdiF4KIptmA/xPvmzZuMxdYUzIPcd+655568\nvLlJAL6G32oOasQWNHz5Rvg6nrKh/mygfSzlX2ygPaBUGmov6//NXs1yq4sex2EPrsHemTd2snNg\ntkb+Cx1zBL6SqwxZLvQAKYHzKZaPY4fh4TeHd0S5Nox9OClItm/jiU9DrEwwVEawpiVis9VkimqX\nAOr4o2cCs/0BT2I5+FYJRhJbePQxgzcD7QLEqtV5gdnu2Icr3L45gcCyt74Z7neL4SLQ0nm4S+dM\nYCz1gSPHnhKZDWyHhcCCNKwjqaF/TkwGl0L6nClie/wc1D1xdoNsSLhT0IJkhi7Lzr22xb8keE/N\nPm0Sc9yEuhRUyuiG9HzvFNeImCyq39SriOhtQI7IV/TiTqE8glqwohjE0NJwiANxOZTdZoxtfzSa\nx2tI8DtHcKQoQFmV6f1XT2swibxFL+6k5EgenhBCqKLTPX3ULnaYdDlaTMcCSd8zuXTvBq2bJUJr\nlE4WgSV5ZRdBzLFgO6nzhJp1ltvrlB2HCoWxQuG+jTvt2GxBWUZaU2mMApZNuSHA3vJpCliRhqqs\nZtvbTrb9ZIk+i70Ut1OcnpgeKskTCFUwjaYy8Jhr3eiefq0HIfa7yC6HOwVyULRuNDn21JngbcL+\nE8A+MNnSxb+w59+Cj2tELJBbjEZr8SGwn0j2aLkTPdp08R2OcKV6fXB3ikPH3n8tM5WTfrETtZcw\ng3QWH0dH7nKNiMkszqo/EDafaHhJ5Bm6ee4UtdAabxnMcmUUl0SnYx+uVqs5XAGN9QGgdeCrASv0\n3TmCsJcOdhnozexD38goK9HXynEKr1OKDs9guhQD039kGySyIQpJAdbvJ9YTlPvyUl3/aLUf34G/\nuGxIyXpE37DoLbAHwJaU53t9MRCfrU8o/k4iRn36Lar8Wd5wAfgN4R6xelyy/ssAAAAASUVORK5C\nYII=\n", - "prompt_number": 8, - "text": [ - " 2 \n", - "x \u22c5z + 2.0\u22c5y\u22c5z - cos(z)" - ] - } - ], - "prompt_number": 8 - }, - { - "cell_type": "code", - "collapsed": false, - "input": [], - "language": "python", - "metadata": {}, - "outputs": [] + "cell_type": "code", + "execution_count": null, + "id": "0cfde73c", + "metadata": { + "jupyter": { + "outputs_hidden": false } - ], - "metadata": {} + }, + "outputs": [], + "source": [ + "e = x**2 + 2.0 * y + sin(z)\n", + "e" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cb7eb1ad", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "diff(e, x)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "07441ea9", + "metadata": { + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [ + "integrate(e, z)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.10" } - ] + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/tests/minimal-notebook/data/notebook_svg.ipynb b/tests/minimal-notebook/data/notebook_svg.ipynb index e26997e2c4..08cdec19c6 100644 --- a/tests/minimal-notebook/data/notebook_svg.ipynb +++ b/tests/minimal-notebook/data/notebook_svg.ipynb @@ -2,121 +2,26 @@ "cells": [ { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "image/svg+xml": [ - "\n", - "Group.svg\n", - "Created using Figma 0.90\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], + "source": [ + "from IPython.display import SVG, display" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ - "from IPython.display import SVG, display\n", - "\n", "display(SVG(filename=\"Jupyter_logo.svg\"))" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -130,7 +35,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.6" + "version": "3.9.10" } }, "nbformat": 4,