Lua executor fixes

Previous release had to change the behavior of how script.Script was
parsed. The change related on how the content of the script was saved.
It didn't keep headers which caused issues at execution. These changes
makes it we don't keep just the content but we pass around the
script.Script struct around. This change should help with the execution
speed a little as we don't have to parse the script on every execution.

All stores now keep the parsed script internally. For most cases it
doesn't affect the user but for the etcd store it will require to delete
everything currently in etcd before reapplying with an updated version
of msgscript-cli.

Author: numkem

executor/executor.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
 	"sync"
 	"time"
 
@@ -89,15 +88,6 @@ func (e *NoScriptFoundError) Error() string {
 	return "No script found for subject"
 }
 
-func createTempFile(pattern string) (*os.File, error) {
-	tmpFile, err := os.CreateTemp(os.TempDir(), pattern)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create temp file: %w", err)
-	}
-
-	return tmpFile, nil
-}
-
 type Executor interface {
 	HandleMessage(context.Context, *Message, ReplyFunc)
 	Stop()

executor/lua_executor.go

@@ -53,30 +53,34 @@ func replyWithError(fields log.Fields, rf ReplyFunc, msg string, a ...any) {
 // HandleMessage receives a message, matches it to a Lua script, and executes the script in a new goroutine
 func (le *LuaExecutor) HandleMessage(ctx context.Context, msg *Message, rf ReplyFunc) {
 	fields := log.Fields{
-		"subject": msg.Subject,
+		"subject":  msg.Subject,
 		"executor": "lua",
 	}
 
 	// Look up the Lua script for the given subject
 	scripts, err := le.store.GetScripts(ctx, msg.Subject)
-	if err != nil || scripts == nil {
-		replyWithError(fields, rf, "failed to get scripts for subject %s: %w", msg.Subject, err)
+	if err != nil {
+		replyWithError(fields, rf, "failed to get scripts for subject %s: %v", msg.Subject, err)
 		return
 	}
 
+	if scripts == nil {
+		replyWithError(fields, rf, "no scripts found for subject %s", msg.Subject)
+	}
+
 	errs := make(chan error, len(scripts))
 	var wg sync.WaitGroup
 	r := NewReply()
 	// Loop through each scripts attached to the subject as there might be more than one
-	for path, script := range scripts {
+	for path, scr := range scripts {
 		wg.Add(1)
 
 		ss := strings.Split(path, "/")
 		name := ss[len(ss)-1]
 		fields["path"] = name
 
 		// Run the Lua script in a separate goroutine to handle the message for each script
-		go func(content []byte) {
+		go func(scr *scriptLib.Script) {
 			defer wg.Done()
 
 			tmp, err := os.MkdirTemp(os.TempDir(), "msgscript-lua-*s")
@@ -92,14 +96,7 @@ func (le *LuaExecutor) HandleMessage(ctx context.Context, msg *Message, rf Reply
 				return
 			}
 
-			// Read the script to get the headers (for the libraries for example)
-			s, err := scriptLib.ReadString(string(content))
-			if err != nil {
-				errs <- fmt.Errorf("failed to read script: %w", err)
-				return
-			}
-
-			libs, err := le.store.LoadLibrairies(ctx, s.LibKeys)
+			libs, err := le.store.LoadLibrairies(ctx, scr.LibKeys)
 			if err != nil {
 				errs <- fmt.Errorf("failed to read librairies: %w", err)
 				return
@@ -117,7 +114,7 @@ func (le *LuaExecutor) HandleMessage(ctx context.Context, msg *Message, rf Reply
 			}
 			defer le.store.ReleaseLock(ctx, name)
 
-			log.WithFields(fields).WithField("isHTML", s.HTML).Debug("executing script")
+			log.WithFields(fields).WithField("isHTML", scr.HTML).Debug("executing script")
 
 			L := lua.NewState()
 			tctx, tcan := context.WithTimeout(le.ctx, MAX_LUA_RUNNING_TIME)
@@ -147,11 +144,11 @@ func (le *LuaExecutor) HandleMessage(ctx context.Context, msg *Message, rf Reply
 				sb.Write(l)
 				sb.WriteString("\n")
 			}
-			sb.Write(content)
+			sb.Write(scr.Content)
 			log.WithFields(fields).Debugf("script:\n%+s\n\n", sb.String())
 
 			res := &ScriptResult{
-				IsHTML:  s.HTML,
+				IsHTML:  scr.HTML,
 				Headers: make(map[string]string),
 			}
 			if err := L.DoString(sb.String()); err != nil {
@@ -163,7 +160,7 @@ func (le *LuaExecutor) HandleMessage(ctx context.Context, msg *Message, rf Reply
 			}
 
 			// Retrieve the result from the Lua state (assuming it's a string)
-			if s.HTML {
+			if scr.HTML {
 				// If the message is set to return HTML, we pass 2 things to the fonction named after the HTTP
 				// method received ex: POST(), GET()...
 				// The 2 things are:
@@ -177,7 +174,7 @@ func (le *LuaExecutor) HandleMessage(ctx context.Context, msg *Message, rf Reply
 				//   - The body of the message
 				le.executeRawMessage(fields, L, r, msg, res, name)
 			}
-		}(script)
+		}(scr)
 	}
 	wg.Wait()
 	log.WithField("subject", msg.Subject).Debugf("finished running %d scripts", len(scripts))

executor/nopodman_executor.go

@@ -0,0 +1,24 @@
+//go:build !podman
+// This is so that we can build without podman's dependancies
+
+package executor
+
+import (
+	"context"
+
+	msgstore "github.com/numkem/msgscript/store"
+)
+
+type noPodmanExecutor struct{}
+
+func NewPodmanExecutor(c context.Context, store msgstore.ScriptStore) (Executor, error) {
+	return &noPodmanExecutor{}, nil
+}
+
+func (e *noPodmanExecutor) HandleMessage(ctx context.Context, msg *Message, rf ReplyFunc) {
+	r := NewReply()
+	r.Error = "msgscript wasn't built with podman support"
+	rf(r)
+}
+
+func (e *noPodmanExecutor) Stop() {}

executor/nowasm_executor.go

@@ -0,0 +1,26 @@
+//go:build !wasm
+// This is so that we can build without wasm's dependancies
+
+package executor
+
+import (
+	"context"
+
+	"github.com/nats-io/nats.go"
+	msgplugins "github.com/numkem/msgscript/plugins"
+	msgstore "github.com/numkem/msgscript/store"
+)
+
+type noWasmExecutor struct{}
+
+func NewWasmExecutor(c context.Context, store msgstore.ScriptStore, plugins []msgplugins.PreloadFunc, nc *nats.Conn) Executor {
+	return &noWasmExecutor{}
+}
+
+func (e *noWasmExecutor) HandleMessage(ctx context.Context, msg *Message, rf ReplyFunc) {
+	r := NewReply()
+	r.Error = "msgscript wasn't built with wasm support"
+	rf(r)
+}
+
+func (e *noWasmExecutor) Stop() {}

executor/executor_podman.go renamed to executor/podman_executor.go

@@ -1,3 +1,5 @@
+//go:build podman
+
 package executor
 
 import (

executor/wasm_executor.go

@@ -1,6 +1,9 @@
+//go:build wasm
+
 package executor
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -14,7 +17,6 @@ import (
 	scriptLib "github.com/numkem/msgscript/script"
 	msgstore "github.com/numkem/msgscript/store"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/net/context"
 )
 
 type WasmExecutor struct {
@@ -56,8 +58,8 @@ func (we *WasmExecutor) HandleMessage(ctx context.Context, msg *Message, rf Repl
 		ss := strings.Split(path, "/")
 		name := ss[len(ss)-1]
 		fields := log.Fields{
-			"subject": msg.Subject,
-			"path":    name,
+			"subject":  msg.Subject,
+			"path":     name,
 			"executor": "wasm",
 		}
 
@@ -199,4 +201,3 @@ func (we *WasmExecutor) Stop() {
 	we.cancelFunc()
 	log.Debug("WasmExecutor stopped")
 }
-

flake.nix

@@ -8,62 +8,8 @@
   outputs =
     { self, nixpkgs }:
     let
-      version = "0.5.0";
-      vendorHash = "sha256-LcgdJIsn3/fHv3NGvGdfq/Y3N7CTuIH/b5Rv5tEMUg8=";
-
-      mkCli =
-        pkgs:
-        let
-          wasmtime = pkgs.callPackage ./nix/pkgs/wasmtime.nix { };
-        in
-        pkgs.buildGoModule {
-          pname = "msgscript-cli";
-          inherit version vendorHash;
-
-          src = self;
-
-          subPackages = [ "cmd/cli" ];
-
-          nativeBuildInputs = [ pkgs.pkg-config ];
-
-          buildInputs = [
-            wasmtime.dev
-            pkgs.btrfs-progs
-            pkgs.gpgme
-          ];
-
-          postInstall = ''
-            mv $out/bin/cli $out/bin/msgscriptcli
-          '';
-        };
-
-      mkServer =
-        pkgs:
-        let
-          wasmtime = pkgs.callPackage ./nix/pkgs/wasmtime.nix { };
-        in
-        pkgs.buildGoModule {
-          pname = "msgscript";
-          inherit version vendorHash;
-
-          src = self;
-
-          subPackages = [ "cmd/server" ];
-
-          nativeBuildInputs = [ pkgs.pkg-config ];
-
-          buildInputs = [
-            wasmtime.dev
-            pkgs.btrfs-progs
-            pkgs.gpgme
-          ];
-
-          doCheck = false; # Requires networking, will just timeout
-
-          postInstall = ''
-            mv $out/bin/server $out/bin/msgscript
-          '';
-        };
+      version = "0.6.0";
+      vendorHash = "sha256-+sI9iQsfaxPlVoQFCLDHGrl7VHgnq1iLOrRj57Fpudc=";
 
       mkPlugin =
         pkgs: name: path:
@@ -94,10 +40,16 @@
         let
           pkgs = import nixpkgs { system = "x86_64-linux"; };
           lib = pkgs.lib;
+
+          wasmtime = pkgs.callPackage ./nix/pkgs/wasmtime.nix { };
         in
         rec {
-          cli = mkCli pkgs;
-          server = mkServer pkgs;
+          cli = pkgs.callPackage ./nix/pkgs/cli.nix {
+            inherit wasmtime version vendorHash;
+          };
+          server = pkgs.callPackage ./nix/pkgs/server.nix {
+            inherit wasmtime version vendorHash;
+          };
           default = server;
 
           allPlugins = pkgs.symlinkJoin {
@@ -119,10 +71,16 @@
         let
           pkgs = import nixpkgs { system = "aarch64-linux"; };
           lib = pkgs.lib;
+
+          wasmtime = pkgs.callPackage ./nix/pkgs/wasmtime.nix { };
         in
         rec {
-          cli = mkCli "x86_64-linux";
-          server = mkServer "x86_64-linux";
+          cli = pkgs.callPackage ./nix/pkgs/cli.nix {
+            inherit wasmtime version vendorHash;
+          };
+          server = pkgs.callPackage ./nix/pkgs/server.nix {
+            inherit wasmtime version vendorHash;
+          };
           default = server;
 
           allPlugins = pkgs.symlinkJoin {

nix/pkgs/cli.nix

@@ -0,0 +1,35 @@
+{
+  lib,
+  buildGoModule,
+  btrfs-progs,
+  gpgme,
+  pkg-config,
+  wasmtime,
+  vendorHash,
+  version,
+  withPodman ? false,
+  withWasm ? false,
+}:
+
+buildGoModule {
+  pname = "msgscript-cli";
+  inherit version vendorHash;
+
+  src = ../..;
+
+  subPackages = [ "cmd/cli" ];
+
+  nativeBuildInputs = [ ] ++ (lib.optional withPodman [ pkg-config ]);
+
+  buildInputs =
+    [ ]
+    ++ (lib.optional withWasm [ wasmtime.dev ])
+    ++ (lib.optional withPodman [
+      btrfs-progs
+      gpgme
+    ]);
+
+  postInstall = ''
+    mv $out/bin/cli $out/bin/msgscriptcli
+  '';
+}

nix/pkgs/server.nix

@@ -0,0 +1,37 @@
+{
+  lib,
+  buildGoModule,
+  btrfs-progs,
+  gpgme,
+  pkg-config,
+  wasmtime,
+  vendorHash,
+  version,
+  withPodman ? false,
+  withWasm ? false,
+}:
+
+buildGoModule {
+  pname = "msgscript";
+  inherit version vendorHash;
+
+  src = ../..;
+
+  subPackages = [ "cmd/server" ];
+
+  nativeBuildInputs = [ pkg-config ];
+
+  buildInputs =
+    [ ]
+    ++ (lib.optional withWasm [ wasmtime.dev ])
+    ++ (lib.optional withPodman [
+      btrfs-progs
+      gpgme
+    ]);
+
+  doCheck = false; # Requires networking, will just timeout
+
+  postInstall = ''
+    mv $out/bin/server $out/bin/msgscript
+  '';
+}