Disabled markdown in integration tests -- see jupyter/nbconvert#2017 …

…; Refactored jpyexec package, to clean up communication between user's program and GoNB.
janpfeifer · Sep 17, 2023 · 0d26a86 · 0d26a86
1 parent dc6c3ad
commit 0d26a86
Show file tree

Hide file tree

Showing 15 changed files with 630 additions and 561 deletions.
diff --git a/common/common.go b/common/common.go
@@ -74,9 +74,10 @@ func walkDirWithSymbolicLinksImpl(root, current string, dirFunc fs.WalkDirFunc,
 	}
 	visited.Insert(current)
 
-	return filepath.WalkDir(current, func(entryPath string, info fs.DirEntry, err error) error {
+	err := filepath.WalkDir(current, func(entryPath string, info fs.DirEntry, err error) error {
 		if info == nil {
-			return errors.Errorf("file %q does not exist!?", entryPath)
+			// This happens when a linked file/dir does not exist, ignoring.
+			return nil
 		}
 		if info.Type() == os.ModeSymlink {
 			// Recursively follow symbolic links.
@@ -85,12 +86,20 @@ func walkDirWithSymbolicLinksImpl(root, current string, dirFunc fs.WalkDirFunc,
 				err = errors.Wrapf(err, "WalkDirWithSymbolicLinks failed to resolve symlink %q", entryPath)
 				return err
 			}
-			return walkDirWithSymbolicLinksImpl(root, linkedPath, dirFunc, visited)
+			err = walkDirWithSymbolicLinksImpl(root, linkedPath, dirFunc, visited)
+			if err != nil {
+				err = errors.WithMessagef(err, "while traversing symlink %q -> %q", entryPath, linkedPath)
+				return err
+			}
 		}
 
 		// If not a symbolic link, call the user's function.
 		return dirFunc(entryPath, info, err)
 	})
+	if err != nil {
+		err = errors.WithMessagef(err, "while traversing dir %q", current)
+	}
+	return err
 }
 
 // ReplaceTildeInDir by the user's home directory. Returns dir if it doesn't start with "~".

diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
@@ -5,7 +5,7 @@
 * Added **widgets** support (experimental): 
   * a websocket opened from the front-end that communicates
     to the kernel, and through it to the users cells.
-  * (TODO) API to use it in `gonb/gonbui/widgets`. 
+  * API to use it in `gonb/gonbui/widgets`. 
 * Added "%wasm" support (experimental): 
   * Allows compiling cell to WASM and running that in the notebook. One
     can write widgets like this. **Experimental**: there are some use cases are not 100% clear. See

diff --git a/examples/tests/bash_script.ipynb b/examples/tests/bash_script.ipynb
@@ -26,15 +26,15 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "id": "382ac8a9-1bd0-4854-91df-5ef8fd14ea3a",
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/tmp/gonb_69c8222b\n"
+      "/tmp/gonb_e1896a14\n"
      ]
     }
    ],
@@ -59,15 +59,13 @@
      "output_type": "stream",
      "text": [
       "/home/janpf/Projects/gonb/examples/tests\n",
-      "/tmp/gonb_pipe_2636811841\n",
-      "/tmp/gonb_0c50904a\n",
+      "/tmp/gonb_e1896a14\n",
       "/home/janpf\n"
      ]
     }
    ],
    "source": [
     "!echo $GONB_DIR\n",
-    "!echo $GONB_PIPE\n",
     "!echo $GONB_TMP_DIR\n",
     "!echo $GONB_JUPYTER_ROOT"
    ]

diff --git a/examples/tests/goflags.ipynb b/examples/tests/goflags.ipynb
@@ -15,13 +15,11 @@
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/markdown": [
-       "* `%goflags=[\"-cover\"]`"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "%goflags=[\"-cover\"]\n"
+     ]
     }
    ],
    "source": [
@@ -35,13 +33,11 @@
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/markdown": [
-       "* `%goflags=[\"-cover\"]`"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "%goflags=[\"-cover\"]\n"
+     ]
     }
    ],
    "source": [
@@ -55,13 +51,11 @@
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/markdown": [
-       "* `%goflags=[]`"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "%goflags=[]\n"
+     ]
     }
    ],
    "source": [
@@ -76,13 +70,11 @@
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/markdown": [
-       "* `%goflags=[\"-cover\"]`"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "%goflags=[\"-cover\"]\n"
+     ]
     }
    ],
    "source": [
@@ -161,9 +153,9 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "gonb_9f1ee980/main.go:8:\tA\t\t100.0%\n",
-      "gonb_9f1ee980/main.go:12:\tB\t\t0.0%\n",
-      "gonb_9f1ee980/main.go:17:\tmain\t\t100.0%\n",
+      "gonb_6a7be5e7/main.go:8:\tA\t\t100.0%\n",
+      "gonb_6a7be5e7/main.go:12:\tB\t\t0.0%\n",
+      "gonb_6a7be5e7/main.go:17:\tmain\t\t100.0%\n",
       "total\t\t\t\t(statements)\t75.0%\n"
      ]
     }
@@ -246,7 +238,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "# gonb_9f1ee980\n",
+      "# gonb_6a7be5e7\n",
       "./main.go:10:6: can inline (*Point).ManhattanLen\n",
       "./main.go:16:12: inlining call to flag.Parse\n",
       "./main.go:18:28: inlining call to (*Point).ManhattanLen\n",

diff --git a/examples/tests/gonbui.ipynb b/examples/tests/gonbui.ipynb
@@ -13,7 +13,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 2,
    "id": "60efdb28-4184-4bf2-9216-284a531e9344",
    "metadata": {},
    "outputs": [
@@ -35,7 +35,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 3,
    "id": "d5a28f1b-18ff-4c38-b5f9-908f0863fec6",
    "metadata": {},
    "outputs": [
@@ -54,7 +54,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 4,
    "id": "b0f6f952-5503-490d-a371-7f4106fd43cf",
    "metadata": {},
    "outputs": [
@@ -64,8 +64,9 @@
        "html displayed"
       ]
      },
+     "execution_count": 4,
      "metadata": {},
-     "output_type": "display_data"
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -77,7 +78,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 5,
    "id": "3df60c07-56fd-486b-8202-cb7047cbb5d3",
    "metadata": {},
    "outputs": [
@@ -87,8 +88,9 @@
        "markdown displayed"
       ]
      },
+     "execution_count": 5,
      "metadata": {},
-     "output_type": "display_data"
+     "output_type": "execute_result"
     }
    ],
    "source": [

diff --git a/examples/tests/wasm.ipynb b/examples/tests/wasm.ipynb
@@ -21,8 +21,8 @@
      "output_type": "stream",
      "text": [
       "/home/janpf\n",
-      "/home/janpf/jupyter_files/6e2dedeb\n",
-      "/files/jupyter_files/6e2dedeb\n"
+      "/home/janpf/jupyter_files/eb626c95\n",
+      "/files/jupyter_files/eb626c95\n"
      ]
     }
    ],
@@ -43,7 +43,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/home/janpf/jupyter_files/6e2dedeb\n"
+      "/home/janpf/jupyter_files/eb626c95\n"
      ]
     }
    ],
@@ -62,24 +62,6 @@
     "the latest release from the web)."
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 3,
-   "id": "d17df1d5-3660-40ff-b66e-bcd0dde781ba",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Set: GONB_GIT_ROOT=\"/home/janpf/Projects/gonb\"\n"
-     ]
-    }
-   ],
-   "source": [
-    "%env GONB_GIT_ROOT=/home/janpf/Projects/gonb"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 4,
@@ -134,25 +116,36 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 6,
    "id": "8eba1e4b-50f3-4a58-840e-ec76efec2b07",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "text/html": [
-       "<script src=\"/files/jupyter_files/6e2dedeb/wasm_exec.js\"></script>\n",
-       "<script>\n",
-       "var go_6e2dedeb = new Go();\n",
-       " \n",
-       "WebAssembly.instantiateStreaming(fetch(\"/files/jupyter_files/6e2dedeb/gonb_cell.wasm\"), go_6e2dedeb.importObject).\n",
-       "\tthen((result) => { go_6e2dedeb.run(result.instance); });\n",
-       "</script>\n",
-       "<div id=\"gonb_id_991f92d3\"></div>\n"
+       "<div id=\"gonb_id_0631c703\"></div><script src=\"/files/jupyter_files/eb626c95/wasm_exec.js\"></script>"
       ]
      },
+     "execution_count": 6,
      "metadata": {},
-     "output_type": "display_data"
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/javascript": [
+       "\n",
+       "(() => {\n",
+       "\tvar go_eb626c95 = new globalThis.Go();\n",
+       "\tgo_eb626c95.argv = [\"js\"].concat([\"--x=42\", ]);\n",
+       "\tconsole.log(\"argv=\"+go_eb626c95.argv);\n",
+       "\tWebAssembly.instantiateStreaming(fetch(\"/files/jupyter_files/eb626c95/gonb_cell.wasm\"), go_eb626c95.importObject).\n",
+       "\t\tthen((result) => { go_eb626c95.run(result.instance); });\n",
+       "})();\n"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -165,7 +158,7 @@
     "var flagX = flag.String(\"x\", \"\", \"the meaning of life, the universe, and everything\")\n",
     "\n",
     "%% --x=42\n",
-    "wasm.ParseFlags(GonbWasmArgs)  // Parse flags form the args passed in GonbWasmArgs.\n",
+    "flag.Parse()\n",
     "div := wasm.ById(GonbWasmDivId)  // Div created for the wasm program.\n",
     "var parts []string\n",
     "parts = append(parts, fmt.Sprintf(\"Wasm File Directory: %s\\n\", GonbWasmDir))\n",

diff --git a/goexec/execcode.go b/goexec/execcode.go
@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	. "github.com/janpfeifer/gonb/common"
+	"github.com/janpfeifer/gonb/internal/jpyexec"
 	"github.com/janpfeifer/gonb/kernel"
 	"github.com/pkg/errors"
 	"golang.org/x/exp/slices"
@@ -134,7 +135,8 @@ func (s *State) Execute(msg kernel.Message, fileToCellIdAndLine []CellIdAndLine)
 	if len(args) == 0 && s.CellIsTest {
 		args = s.DefaultCellTestArgs()
 	}
-	err := kernel.PipeExecToJupyter(msg, s.BinaryPath(), args...).
+	err := jpyexec.New(msg, s.BinaryPath(), args...).
+		UseNamedPipes().
 		ExecutionCount(msg.Kernel().ExecCounter).
 		WithStderr(newJupyterStackTraceMapperWriter(msg, "stderr", s.CodePath(), fileToCellIdAndLine)).
 		Exec()

diff --git a/gonbui/protocol/protocol.go b/gonbui/protocol/protocol.go
@@ -6,12 +6,20 @@ import "encoding/gob"
 
 const (
 	// GONB_PIPE_ENV is the name of the environment variable holding
-	// the path to the unix named pipe to communicate rich content to the kernel.
+	// the path to the unix named pipe to communicate from the kernel to
+	// the Go program.
 	//
-	// One doesn't need to use this directly usually, just use gonbui package instead,
-	// they will use this.
+	// It is used to display rich content in Jupyter, and to update widgets.
+	//
+	// One doesn't need to use this directly usually, just use gonbui package instead.
 	GONB_PIPE_ENV = "GONB_PIPE"
 
+	// GONB_PIPE_BACK_ENV is the name of the environment variable holding
+	// the path to the unix named pipe to communicate from the Go program to the kernel.
+	//
+	// It is used to receive updates from widgets displayed in the front-end (Jupyter notebook).
+	GONB_PIPE_BACK_ENV = "GONB_PIPE_BACK"
+
 	// GONB_DIR_ENV is the name of the environment variable holding the
 	// current execution directory for the Go cells, and the scripts executed
 	// with `!` special command.