Native driver bugfixes and org improvements (#57)

Fixed multiple issues with Native driver on macos - mostly:
* Proper passing of metadata as env variables to the workload - sudo & su are quite prohibitive there and to not make the node config harder I used just storing the metadata to env export variable and source it in the shell command. The file is stored in tmp and available via ACL to the newly created user only.
* Proper log output for the workload right into the node log - it prepends stdout/stderr lines with the resource identifier and looks nice.
* Using current fish node group as dynamic user group by default - otherwise the user is incomplete.
* Configuration for the binaries location - by default it tries to find it in PATH, but now it's easy to override.
* Modern MacOS on M1 doesn't allow to remove the created user, so have to give aquarium-fish binary "Full Disk Access" permission (added to wiki).

Also a couple of small improvements:
* Moved improved tests helpers from #54 
* Moved a couple of fixes for build system from #54 
* Found & fixed issue with single-process archiving during the build on macos

build.sh

@@ -16,6 +16,9 @@ root_dir=$(cd "$(dirname "$0")"; echo "$PWD")
 echo "ROOT DIR: ${root_dir}"
 cd "${root_dir}"
 
+# Disabling cgo in order to not link with libc and utilize static linkage binaries
+# which will help to not relay on glibc on linux and be truely independend from OS
+export CGO_ENABLED=0
 
 echo "--- GENERATE CODE FOR AQUARIUM-FISH ---"
 # Install oapi-codegen if it's not available or version is not the same with go.mod
@@ -66,7 +69,8 @@ fi
 # Run parallel builds but no more than limit (gox doesn't support all the os/archs we need)
 pwait() {
  # Note: Dash really don't like jobs to be executed in a pipe or in other shell, soooo...
- while jobs > /tmp/jobs_list.tmp; do
+ # Using "-p" to show only PIDs (because it could be multiline) and "-r" to show only running jobs
+ while jobs -pr > /tmp/jobs_list.tmp; do
  [ $(cat /tmp/jobs_list.tmp | wc -l) -ge $1 ] || break
  sleep 1
  done
@@ -117,29 +121,31 @@ done
 
 [ $errorcount -eq 0 ] || exit $errorcount
 
-echo
-echo "--- ARCHIVE ${BINARY_NAME} ($MAXJOBS in parallel) ---"
-
-# Pack the artifact archives
-for GOOS in $os_list; do
- for GOARCH in $arch_list; do
- name="$BINARY_NAME.${GOOS}_${GOARCH}"
- [ -f "$name" ] || continue
-
- echo "Archiving: $name ..."
- mkdir "$name.dir"
- bin_name='aquarium-fish'
- [ "$GOOS" != "windows" ] || bin_name="$bin_name.exe"
-
- cp -a "$name" "$name.dir/$bin_name"
- $(
- cd "$name.dir"
- tar -cJf "../$name.tar.xz" "$bin_name" >/dev/null 2>&1
- zip "../$name.zip" "$bin_name" >/dev/null 2>&1
- cd .. && rm -rf "$name.dir"
- ) &
- pwait $MAXJOBS
+if [ "x${RELEASE}" != "x" ]; then
+ echo
+ echo "--- ARCHIVE ${BINARY_NAME} ($MAXJOBS in parallel) ---"
+
+ # Pack the artifact archives
+ for GOOS in $os_list; do
+ for GOARCH in $arch_list; do
+ name="$BINARY_NAME.${GOOS}_${GOARCH}"
+ [ -f "$name" ] || continue
+
+ echo "Archiving: $(du -h "$name") ..."
+ mkdir "$name.dir"
+ bin_name='aquarium-fish'
+ [ "$GOOS" != "windows" ] || bin_name="$bin_name.exe"
+
+ cp -a "$name" "$name.dir/$bin_name"
+ $(
+ cd "$name.dir"
+ tar -cJf "../$name.tar.xz" "$bin_name" >/dev/null 2>&1
+ zip "../$name.zip" "$bin_name" >/dev/null 2>&1
+ cd .. && rm -rf "$name.dir"
+ ) &
+ pwait $MAXJOBS
+ done
  done
-done
 
-wait
+ wait
+fi

check.sh

@@ -32,7 +32,7 @@ done
 echo
 echo '---------------------- GoFmt verify ----------------------'
 echo
-reformat=$(gofmt -l .)
+reformat=$(gofmt -l . 2>&1)
 if [ "${reformat}" ]; then
  echo "Please run 'gofmt -w .': \n${reformat}"
  errors=$((${errors}+$(echo "${reformat}" | wc -l)))
@@ -57,7 +57,7 @@ echo
 vet=$(go vet ./... 2>&1)
 if [ "${vet}" ]; then
  echo "Please fix the issues: \n${vet}"
- errors=$(((${errors}+$(echo "${vet}" | wc -l))/2))
+ errors=$(( ${errors}+$(echo "${vet}" | wc -l) ))
 fi
 
 

cmd/fish/fish.go

@@ -33,6 +33,8 @@ import (
 )
 
 func main() {
+ log.Infof("Aquarium Fish %s (%s)", build.Version, build.Time)
+
  var api_address string
  var proxy_address string
  var node_address string
@@ -56,8 +58,6 @@ func main() {
  return log.InitLoggers()
  },
  RunE: func(cmd *cobra.Command, args []string) error {
- log.Infof("Aquarium Fish %s (%s)", build.Version, build.Time)
-
  cfg := &fish.Config{}
  if err := cfg.ReadConfigFile(cfg_path); err != nil {
  return log.Error("Fish: Unable to apply config file:", cfg_path, err)

go.mod

@@ -1,6 +1,6 @@
 module github.com/adobe/aquarium-fish
 
-go 1.20
+go 1.21
 
 require (
  github.com/alessio/shellescape v1.4.1

go.sum

@@ -59,6 +59,7 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=

lib/drivers/native/config.go

@@ -28,10 +28,24 @@ import (
 type Config struct {
  //TODO: Users []string `json:"users"` // List of precreated OS user names in format "user[:password]" to run the workload
 
- SudoPath string `json:"sudo_path"` // Path to the sudo (privilege escalation) binary
+ SuPath string `json:"su_path"` // Path to the su (login as user) binary
+ SudoPath string `json:"sudo_path"` // Path to the sudo (privilege escalation) binary
+ ShPath string `json:"sh_path"` // Path to the sh (simple user shell) binary
+ TarPath string `json:"tar_path"` // Path to the tar (unpacking images) binary
+ MountPath string `json:"mount_path"` // Path to the mount (list of mounted volumes) binary
+ ChownPath string `json:"chown_path"` // Path to the chown (change file/dir ownership) binary
+ ChmodPath string `json:"chmod_path"` // Path to the chmod (change file/dir access) binary
+ KillallPath string `json:"killall_path"` // Path to the killall (send signals to multiple processes) binary
+ RmPath string `json:"rm_path"` // Path to the rm (cleanup after execution) binary
+
  ImagesPath string `json:"images_path"` // Where to store/look the environment images
  WorkspacePath string `json:"workspace_path"` // Where to place the env disks
 
+ DsclPath string `json:"dscl_path"` // Path to the dscl (macos user control) binary
+ HdiutilPath string `json:"hdiutil_path"` // Path to the hdiutil (macos images create/mount/umount) binary
+ MdutilPath string `json:"mdutil_path"` // Path to the mdutil (macos disable indexing for disks) binary
+ CreatehomedirPath string `json:"createhomedir_path"` // Path to the createhomedir (macos create/prefill user directory) binary
+
  // Alter allows you to control how much resources will be used:
  // * Negative (<0) value will alter the total resource count before provisioning so you will be
  // able to save some resources for the host system (recommended -2 for CPU and -10 for RAM
@@ -89,8 +103,8 @@ func (c *Config) Apply(config []byte) (err error) {
 }
 
 func (c *Config) Validate() (err error) {
- // Sudo is used to become the separated unprevileged user which will execute the workload
- // and execute a number of administrative actions to create/delete the user and cleanup
+ // Sudo is used to run commands from superuser and execute a number of
+ // administrative actions to create/delete the user and cleanup
  if c.SudoPath == "" {
  if c.SudoPath, err = exec.LookPath("sudo"); err != nil {
  return fmt.Errorf("Native: Unable to locate `sudo` path: %s", err)
@@ -105,8 +119,181 @@ func (c *Config) Validate() (err error) {
  }
  }
 
+ // Su is used to become the separated unprevileged user and control whom to become in sudoers
+ if c.SuPath == "" {
+ if c.SuPath, err = exec.LookPath("su"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `su` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.SuPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `su` path: %s, %s", c.SuPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `su` binary is not executable: %s", c.SuPath)
+ }
+ }
+ }
+
+ // Sh is needed to set the unprevileged user default executable
+ if c.ShPath == "" {
+ if c.ShPath, err = exec.LookPath("sh"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `su` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.ShPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `sh` path: %s, %s", c.ShPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `sh` binary is not executable: %s", c.ShPath)
+ }
+ }
+ }
+ // Tar used to unpack the images
+ if c.TarPath == "" {
+ if c.TarPath, err = exec.LookPath("tar"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `tar` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.TarPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `tar` path: %s, %s", c.TarPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `tar` binary is not executable: %s", c.TarPath)
+ }
+ }
+ }
+ // Mount allows to look at the mounted volumes
+ if c.MountPath == "" {
+ if c.MountPath, err = exec.LookPath("mount"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `mount` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.MountPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `mount` path: %s, %s", c.MountPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `mount` binary is not executable: %s", c.MountPath)
+ }
+ }
+ }
+ // Chown needed to properly set ownership for the unprevileged user on available resources
+ if c.ChownPath == "" {
+ if c.ChownPath, err = exec.LookPath("chown"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `chown` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.ChownPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `chown` path: %s, %s", c.ChownPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `chown` binary is not executable: %s", c.ChownPath)
+ }
+ }
+ }
+ // Chmod needed to set additional read access for the unprevileged user on env metadata file
+ if c.ChmodPath == "" {
+ if c.ChmodPath, err = exec.LookPath("chmod"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `chmod` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.ChmodPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `chmod` path: %s, %s", c.ChmodPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `chmod` binary is not executable: %s", c.ChmodPath)
+ }
+ }
+ }
+ // Killall is running to stop all the unprevileged user processes during deallocation
+ if c.KillallPath == "" {
+ if c.KillallPath, err = exec.LookPath("killall"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `killall` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.KillallPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `killall` path: %s, %s", c.KillallPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `killall` binary is not executable: %s", c.KillallPath)
+ }
+ }
+ }
+ // Rm allows to clean up the leftowers after the execution
+ if c.RmPath == "" {
+ if c.RmPath, err = exec.LookPath("rm"); err != nil {
+ return fmt.Errorf("Native: Unable to locate `rm` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.RmPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate `rm` path: %s, %s", c.RmPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: `rm` binary is not executable: %s", c.RmPath)
+ }
+ }
+ }
+
+ // MacOS specific ones:
+ // Dscl creates/removes the unprevileged user
+ if c.DsclPath == "" {
+ if c.DsclPath, err = exec.LookPath("dscl"); err != nil {
+ return fmt.Errorf("Native: Unable to locate macos `dscl` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.DsclPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate macos `dscl` path: %s, %s", c.DsclPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: macos `dscl` binary is not executable: %s", c.DsclPath)
+ }
+ }
+ }
+ // Hdiutil allows to create disk images and mount them to restrict user by disk space
+ if c.HdiutilPath == "" {
+ if c.HdiutilPath, err = exec.LookPath("hdiutil"); err != nil {
+ return fmt.Errorf("Native: Unable to locate macos `hdiutil` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.HdiutilPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate macos `hdiutil` path: %s, %s", c.HdiutilPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: macos `hdiutil` binary is not executable: %s", c.HdiutilPath)
+ }
+ }
+ }
+ // Mdutil allows to disable the indexing for mounted volume
+ if c.MdutilPath == "" {
+ if c.MdutilPath, err = exec.LookPath("mdutil"); err != nil {
+ return fmt.Errorf("Native: Unable to locate macos `mdutil` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.MdutilPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate macos `mdutil` path: %s, %s", c.MdutilPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: macos `mdutil` binary is not executable: %s", c.MdutilPath)
+ }
+ }
+ }
+ // Createhomedir creates unprevileged user home directory and fulfills with default subdirs
+ if c.CreatehomedirPath == "" {
+ if c.CreatehomedirPath, err = exec.LookPath("createhomedir"); err != nil {
+ return fmt.Errorf("Native: Unable to locate macos `createhomedir` path: %s", err)
+ }
+ } else {
+ if info, err := os.Stat(c.CreatehomedirPath); os.IsNotExist(err) {
+ return fmt.Errorf("Native: Unable to locate macos `createhomedir` path: %s, %s", c.CreatehomedirPath, err)
+ } else {
+ if info.Mode()&0111 == 0 {
+ return fmt.Errorf("Native: macos `createhomedir` binary is not executable: %s", c.CreatehomedirPath)
+ }
+ }
+ }
+
  // Verify the configuration works for this machine
  var opts Options
+ opts.Validate()
  // If the users are not set - the user will be created dynamically
  // with "fish-" prefix and it's needed quite a good amount of access:
 
@@ -138,7 +325,7 @@ func (c *Config) Validate() (err error) {
 
  // Clean after the run
  if err = userDelete(c, user); err != nil {
- return fmt.Errorf("Native: Unable to delete user %q: %v", user, err)
+ return fmt.Errorf("Native: Unable to delete user in the end of driver verification %q: %v", user, err)
  }
 
  // TODO:

lib/drivers/native/driver.go

@@ -198,7 +198,7 @@ func (d *Driver) Allocate(def types.LabelDefinition, metadata map[string]any) (*
  return nil, log.Error("Native: Unable to run the entry workload:", err)
  }
 
- log.Info("Native: Started workload for user:", user, opts.Entry)
+ log.Infof("Native: Started environment for user %q", user)
 
  return &types.Resource{Identifier: user}, nil
 }