feat(project): added full support for chunked communication (local + …

…multi-node + multi-host)

re #9

README.md

@@ -109,6 +109,8 @@ For more information, read about [Configuration](#configuration) and [Deployment
 
 > Please make sure that Docker 23.0.0+ is installed before proceeding, or consider updating beforehand.
 
+> If you are using the Stacks feature to manage Docker Compose stacks, please use Docker 26.0.0+.
+
 ### Deploy with Docker
 
 You can run Isaiah with Docker on the command line very quickly.
@@ -422,6 +424,8 @@ To run Isaiah, you will need to set the following environment variables in a `.e
 | `SSL_ENABLED`           | `boolean` | Whether HTTPS should be used in place of HTTP. When configured, Isaiah will look for `certificate.pem` and `key.pem` next to the executable for configuring SSL. Note that if Isaiah is behind a proxy that already handles SSL, this should be set to `false`. | False        |
 | `SERVER_PORT`           | `integer` | The port Isaiah listens on. | 3000        |
 | `SERVER_MAX_READ_SIZE`  | `integer` | The maximum size (in bytes) per message that Isaiah will accept over Websocket. Note that, in a multi-node deployment, you may need to incrase the value of that setting. (Shouldn't be modified, unless your server randomly restarts the Websocket session for no obvious reason) | 100000        |
+| `SERVER_CHUNKED_COMMUNICATION_ENABLED`  | `boolean` | Whether resources should be sent in chunks, rather than all at once. (Recommended only in setups with 150+ Docker resources, and multi-node deployments) | False        |
+| `SERVER_CHUNKED_COMMUNICATION_SIZE`  | `integer` | The number of resources to send per chunk, when chunked communication is enabled | 50        |
 | `AUTHENTICATION_ENABLED`| `boolean` | Whether a password is required to access Isaiah. (Recommended) | True |
 | `AUTHENTICATION_SECRET` | `string`  | The master password used to secure your Isaiah instance against malicious actors. | one-very-long-and-mysterious-secret        |
 | `AUTHENTICATION_HASH`   | `string`  | The master password's hash (sha256 format) used to secure your Isaiah instance against malicious actors. Use this setting instead of `AUTHENTICATION_SECRET` if you feel uncomfortable providing a cleartext password. | Empty    |
@@ -552,6 +556,18 @@ You must update Docker on your system to fix that issue.
 
 Isaiah uses the native Docker Engine API, and it requires Docker to be at least on version 23.0.0+
 
+#### On a system with a lot of Docker resources, Isaiah is inconsistent or crashes
+
+This is due to a limitation on the message size during the WebSocket communication.
+When your server has a lot of Docker resources on it (~150+), Isaiah will send all these resources in a single
+WebSocket message by default. This can cause the server to crash.
+
+There are two solutions :
+- Increase the maximum size of a message : Increase the value of the setting `SERVER_MAX_READ_SIZE`
+- Enabled chunked communication : Set `SERVER_CHUNKED_COMMUNICATION_ENABLED` to `TRUE` and adjust `SERVER_CHUNKED_COMMUNICATION_SIZE` if it's not already satisfying.
+
+You may encounter the same issue in a multi-node deployment, and the cleanest solution here would be to enable chunked communication on the node that hosts a lot of Docker resources. 
+
 #### The emulated shell behaves unconsistently or displays unexpected characters
 
 Please note that the emulated shell works by performing the following steps :

app/client/assets/js/isaiah.js

@@ -563,25 +563,25 @@
       ? `<div class="cell">${prompt.input.name}${prompt.input.type === 'input' ? ':' : ''}</div>
          ${
            prompt.input.type === 'input'
-             ? `<input 
+             ? `<input
                   id="prompt-input"
                   placeholder="${prompt.input.placeholder}"
                   type="${prompt.isForAuthentication ? 'password' : 'text'}"
                   ${prompt.input.defaultValue ? `value="${prompt.input.defaultValue}"` : ''}
                />`
              : `<pre><code class="hljs language-yaml">${prompt.input.defaultValue ? prompt.input.defaultValue : ''}</code></pre>
-                <textarea 
+                <textarea
                   id="prompt-input"
-                  class="textarea" 
-                  placeholder="${prompt.input.placeholder}" 
+                  class="textarea"
+                  placeholder="${prompt.input.placeholder}"
                   spellcheck="false"
                   oninput="const block = this.parentNode.querySelector('pre code');
                            block.innerHTML = this.value;
                            if ( this.value[this.value.length - 1] == '\\n' )
                             block.innerHTML += ' ';
                            block.parentNode.scrollTop = this.scrollTop;
                            block.scrollLeft = this.scrollLeft;
-                           if (window.hljs) { 
+                           if (window.hljs) {
                             block.removeAttribute('data-highlighted');
                             hljs.highlightElement(block);
                           }"
@@ -1361,7 +1361,8 @@
     // 12.2. Log lines stripped background
     if (_state.settings.enableLogLinesStrippedBackground)
       if (_state.inspector.currentTab === 'Logs')
-        hgetTab('inspector').classList.add('stripped-background');
+        if (hgetTab('inspector'))
+          hgetTab('inspector').classList.add('stripped-background');
 
     // 12.3. Highlight code
     if (_state.settings.enableSyntaxHighlight)
@@ -4509,7 +4510,7 @@
 
     /**
      * @typedef Notification
-     * @property {"init"|"refresh"|"loading"|"report"|"prompt"|"tty"|"auth"} Category
+     * @property {"init"|"refresh"|"loading"|"report"|"prompt"|"tty"|"auth"|"init-chunk"|"refresh-chunk"} Category
      * @property {string} Type
      * @property {string} Title
      * @property {object} Content
@@ -4580,7 +4581,7 @@
         }
 
         // Jump to the picked resource if previously Jumped to a new host
-        if (state.jump.backlog) {
+        if (state.jump.backlog && !('ChunkIndex' in notification.Content)) {
           state.navigation.currentTab = state.jump.backlog.ParentKey;
           state.navigation.currentTabsRows[state.jump.backlog.ParentKey] =
             sgetCurrentTab().Rows.findIndex((r) =>
@@ -4595,6 +4596,71 @@
         if (!state.isFullyEmpty) cmdRun(cmds._inspectorTabs);
         break;
 
+      case 'init-chunk':
+        const { Tab } = notification.Content;
+        const isFirstChunk =
+          notification.Content.ChunkIndex === 1 ? true : false;
+
+        if (isFirstChunk) state.tabs = [];
+
+        if (!state.tabs.some((t) => t.Key === Tab.Key)) state.tabs.push(Tab);
+        else
+          state.tabs = state.tabs.map((t) =>
+            t.Key === Tab.Key ? { ...t, Rows: [...t.Rows, ...Tab.Rows] } : t
+          );
+
+        state.navigation.currentTab = state.tabs[0].Key;
+        state.navigation.currentTabsRows = state.tabs.reduce(
+          (a, b) => ({ ...a, [b.Key]: 1 }),
+          {}
+        );
+        state.isFullyEmpty = false;
+
+        // Perform sort if applicable
+        state.tabs = state.tabs.map((tab) => ({
+          ...tab,
+          Rows: !tab.SortBy
+            ? tab.Rows
+            : tab.Rows.toSorted((a, b) => {
+                const inReverse = tab.SortBy.startsWith('-');
+                const key = inReverse ? tab.SortBy.slice(1) : tab.SortBy;
+
+                let val1 = a[key];
+                let val2 = b[key];
+                const comparisonType = getGeneralType(!val1 ? val2 : val1);
+
+                if (comparisonType === 'string')
+                  return !inReverse
+                    ? val1.localeCompare(val2)
+                    : val2.localeCompare(val1);
+                else if (comparisonType === 'numeric')
+                  return !inReverse ? val1 - val2 : val2 - val1;
+              }),
+        }));
+
+        // Jump to the picked resource if previously Jumped to a new host
+        if (state.jump.backlog) {
+          if (state.tabs.some((t) => t.Key === state.jump.backlog.ParentKey)) {
+            state.navigation.currentTab = state.jump.backlog.ParentKey;
+
+            if (
+              sgetCurrentTab().Rows.some((r) => r.ID === state.jump.backlog.ID)
+            ) {
+              state.navigation.currentTabsRows[state.jump.backlog.ParentKey] =
+                sgetCurrentTab().Rows.findIndex((r) =>
+                  r.ID
+                    ? r.ID === state.jump.backlog.ID
+                    : r.Name === state.jump.backlog.Name
+                ) + 1;
+              // state.jump.backlog = null;
+            }
+          }
+        }
+
+        state.isLoading = false;
+        debouncedCmdRun(cmds._inspectorTabs);
+        break;
+
       case 'auth':
         // Load server-sent preferences if any
         if ('Preferences' in notification.Content) {
@@ -4787,6 +4853,52 @@
         state.isLoading = false;
         break;
 
+      case 'refresh-chunk':
+        if ('Tab' in notification.Content) {
+          if (notification.Content.ChunkIndex === 1) {
+            state.tabs = state.tabs.map((t) =>
+              t.Key !== notification.Content.Tab.Key
+                ? t
+                : notification.Content.Tab
+            );
+          } else {
+            state.tabs = state.tabs.map((t) =>
+              t.Key === notification.Content.Tab.Key
+                ? {
+                    ...t,
+                    Rows: [...t.Rows, ...notification.Content.Tab.Rows],
+                  }
+                : t
+            );
+          }
+
+          if (state.tabs.some((t) => t.Rows.length === 0))
+            state.tabs = state.tabs.filter((t) => t.Rows.length > 0);
+
+          state.navigation.currentTabsRows[notification.Content.Tab.Key] = 1;
+          if (searchIsEnabled) reapplySearch = true;
+        }
+
+        if ('Enumeration' in notification.Content) {
+          // Jump can be disabled if we chose a local resource before enumeration finished
+          if (!state.jump.isEnabled) return;
+
+          state.jump.remoteResources.push(
+            ...notification.Content.Enumeration.Rows.map((r) => ({
+              ...r,
+              Host: notification.Content.Host,
+              Parent: notification.Content.Enumeration.Title,
+              ParentKey: notification.Content.Enumeration.Key,
+            }))
+          );
+          state.isLoading = false;
+          cmdRun(cmds._performJumpSearch);
+          break;
+        }
+
+        state.isLoading = false;
+        break;
+
       case 'loading':
         state.isLoading = true;
         break;

app/default.env

@@ -3,6 +3,8 @@ SSL_ENABLED="FALSE"
 
 SERVER_PORT="3000"
 SERVER_MAX_READ_SIZE="100000"
+SERVER_CHUNKED_COMMUNICATION_ENABLED="FALSE"
+SERVER_CHUNKED_COMMUNICATION_SIZE="50"
 
 SERVER_ROLE="Master"
 

app/server/_internal/slices/slices.go

@@ -0,0 +1,8 @@
+package slices
+
+func Chunk[T any](items []T, chunkSize int) (chunks [][]T) {
+	for chunkSize < len(items) {
+		items, chunks = items[chunkSize:], append(chunks, items[0:chunkSize:chunkSize])
+	}
+	return append(chunks, items)
+}

app/server/server/containers.go

@@ -8,6 +8,8 @@ import (
 	_os "will-moss/isaiah/server/_internal/os"
 	"will-moss/isaiah/server/_internal/process"
 	_session "will-moss/isaiah/server/_internal/session"
+	_slices "will-moss/isaiah/server/_internal/slices"
+	_strconv "will-moss/isaiah/server/_internal/strconv"
 	"will-moss/isaiah/server/_internal/tty"
 	"will-moss/isaiah/server/resources"
 	"will-moss/isaiah/server/ui"
@@ -46,11 +48,32 @@ func (Containers) RunCommand(server *Server, session _session.GenericSession, co
 		containers := resources.ContainersList(server.Docker, filters.Args{})
 
 		rows := containers.ToRows(columns)
-		server.SendNotification(
-			session,
-			ui.NotificationData(ui.NP{
-				Content: ui.JSON{"Tab": ui.Tab{Key: "containers", Title: "Containers", Rows: rows, SortBy: _os.GetEnv("SORTBY_CONTAINERS")}}}),
-		)
+
+		// Default communication method - Send all at once
+		if _os.GetEnv("SERVER_CHUNKED_COMMUNICATION_ENABLED") != "TRUE" {
+			server.SendNotification(
+				session,
+				ui.NotificationData(ui.NP{
+					Content: ui.JSON{"Tab": ui.Tab{Key: "containers", Title: "Containers", Rows: rows, SortBy: _os.GetEnv("SORTBY_CONTAINERS")}}}),
+			)
+		} else {
+			// Chunked communication method, send resources chunk by chunk
+			chunkSize := int(_strconv.ParseInt(_os.GetEnv("SERVER_CHUNKED_COMMUNICATION_SIZE"), 10, 64))
+			chunkIndex := 1
+			chunks := _slices.Chunk(rows, chunkSize)
+			for _, c := range chunks {
+				server.SendNotification(
+					session,
+					ui.NotificationDataChunk(ui.NP{
+						Content: ui.JSON{
+							"Tab":        ui.Tab{Key: "containers", Title: "Containers", Rows: c, SortBy: _os.GetEnv("SORTBY_CONTAINERS")},
+							"ChunkIndex": chunkIndex,
+						},
+					}),
+				)
+				chunkIndex += 1
+			}
+		}
 
 	// Bulk - Prune
 	case "containers.prune":

app/server/server/images.go

@@ -7,6 +7,8 @@ import (
 	_os "will-moss/isaiah/server/_internal/os"
 	"will-moss/isaiah/server/_internal/process"
 	_session "will-moss/isaiah/server/_internal/session"
+	_slices "will-moss/isaiah/server/_internal/slices"
+	_strconv "will-moss/isaiah/server/_internal/strconv"
 	"will-moss/isaiah/server/resources"
 	"will-moss/isaiah/server/ui"
 
@@ -43,12 +45,31 @@ func (Images) RunCommand(server *Server, session _session.GenericSession, comman
 		images := resources.ImagesList(server.Docker)
 
 		rows := images.ToRows(columns)
-		server.SendNotification(
-			session,
-			ui.NotificationData(ui.NP{
-				Content: ui.JSON{"Tab": ui.Tab{Key: "images", Title: "Images", Rows: rows, SortBy: _os.GetEnv("SORTBY_IMAGES")}},
-			}),
-		)
+
+		// Default communication method - Send all at once
+		if _os.GetEnv("SERVER_CHUNKED_COMMUNICATION_ENABLED") != "TRUE" {
+			server.SendNotification(
+				session,
+				ui.NotificationData(ui.NP{
+					Content: ui.JSON{"Tab": ui.Tab{Key: "images", Title: "Images", Rows: rows, SortBy: _os.GetEnv("SORTBY_IMAGES")}}}),
+			)
+		} else {
+			// Chunked communication method, send resources chunk by chunk
+			chunkSize := int(_strconv.ParseInt(_os.GetEnv("SERVER_CHUNKED_COMMUNICATION_SIZE"), 10, 64))
+			chunkIndex := 1
+			chunks := _slices.Chunk(rows, chunkSize)
+			for _, c := range chunks {
+				server.SendNotification(
+					session,
+					ui.NotificationDataChunk(ui.NP{
+						Content: ui.JSON{
+							"Tab":        ui.Tab{Key: "images", Title: "Images", Rows: c, SortBy: _os.GetEnv("SORTBY_IMAGES")},
+							"ChunkIndex": chunkIndex,
+						}}),
+				)
+				chunkIndex += 1
+			}
+		}
 
 	// Bulk - Prune
 	case "images.prune":

app/server/server/networks.go

@@ -5,6 +5,8 @@ import (
 	"strings"
 	_os "will-moss/isaiah/server/_internal/os"
 	_session "will-moss/isaiah/server/_internal/session"
+	_slices "will-moss/isaiah/server/_internal/slices"
+	_strconv "will-moss/isaiah/server/_internal/strconv"
 	"will-moss/isaiah/server/resources"
 	"will-moss/isaiah/server/ui"
 
@@ -44,12 +46,31 @@ func (Networks) RunCommand(server *Server, session _session.GenericSession, comm
 		networks := resources.NetworksList(server.Docker)
 
 		rows := networks.ToRows(columns)
-		server.SendNotification(
-			session,
-			ui.NotificationData(ui.NP{
-				Content: ui.JSON{"Tab": ui.Tab{Key: "networks", Title: "Networks", Rows: rows, SortBy: _os.GetEnv("SORTBY_NETWORKS")}},
-			}),
-		)
+
+		// Default communication method - Send all at once
+		if _os.GetEnv("SERVER_CHUNKED_COMMUNICATION_ENABLED") != "TRUE" {
+			server.SendNotification(
+				session,
+				ui.NotificationData(ui.NP{
+					Content: ui.JSON{"Tab": ui.Tab{Key: "networks", Title: "Networks", Rows: rows, SortBy: _os.GetEnv("SORTBY_NETWORKS")}}}),
+			)
+		} else {
+			// Chunked communication method, send resources chunk by chunk
+			chunkSize := int(_strconv.ParseInt(_os.GetEnv("SERVER_CHUNKED_COMMUNICATION_SIZE"), 10, 64))
+			chunkIndex := 1
+			chunks := _slices.Chunk(rows, chunkSize)
+			for _, c := range chunks {
+				server.SendNotification(
+					session,
+					ui.NotificationDataChunk(ui.NP{
+						Content: ui.JSON{
+							"Tab":        ui.Tab{Key: "networks", Title: "Networks", Rows: c, SortBy: _os.GetEnv("SORTBY_NETWORKS")},
+							"ChunkIndex": chunkIndex,
+						}}),
+				)
+				chunkIndex += 1
+			}
+		}
 
 	// Bulk - Prune
 	case "networks.prune":