Auto mount in Docker containers (#1416)

src/dbg.c

@@ -39,6 +39,7 @@ bool g_ismusl = FALSE;
 bool g_isstatic = FALSE;
 bool g_isgo = FALSE;
 bool g_issighandler = FALSE;
+char g_libpath[PATH_MAX] = {};
 
 void
 dbgInit()

src/dbg.h

@@ -79,6 +79,7 @@ extern bool g_ismusl;
 extern bool g_isstatic;
 extern bool g_isgo;
 extern bool g_issighandler;
+extern char g_libpath[];
 
 void scopeLog(cfg_log_level_t, const char *, ...) PRINTF_FORMAT(2,3);
 void scopeLogHex(cfg_log_level_t, const void *, size_t, const char *, ...) PRINTF_FORMAT(4,5);

src/fn.c

@@ -333,6 +333,7 @@ initFn(void)
     GETADDR(g_fn.closedir, "closedir");
     GETADDR(g_fn.readdir, "readdir");
     GETADDR(g_fn.setrlimit, "setrlimit");
+    GETADDR(g_fn.mount, "mount");
 #ifdef __STATX__
     GETADDR(g_fn.statx, "statx");
 #endif

src/fn.h

@@ -216,6 +216,7 @@ typedef struct {
     void *(*__memcpy_chk)(void *, const void *, size_t, size_t);
     int (*__sprintf_chk)(char *, int, size_t, const char *, ...);
     long int (*__fdelt_chk)(long int);
+    int (*mount)(const char *, const char *, const char *, unsigned long, const void *);
 #ifdef __linux__
     // Couldn't easily get struct definitions for these on mac
     int (*statvfs64)(const char *, struct statvfs64 *);

src/gocontext_arm.S

@@ -121,6 +121,8 @@
     b.eq 2f
     cmp  x8, #SYS_close
     b.eq 2f
+    cmp  x8, #SYS_mount
+    b.eq 2f
 
     ldr  x17, =\retptr
     ldr  x17, [x17]

src/scopetypes.h

@@ -50,6 +50,8 @@ typedef enum {CFG_MTC_FS,
 #define SM_NAME "scope_anon"
 #define SCOPE_FILTER_USR_PATH ("/usr/lib/appscope/scope_filter")
 #define SCOPE_FILTER_TMP_PATH ("/tmp/appscope/scope_filter")
+#define SCOPE_SYS_PATH "/usr/lib/appscope/"
+#define SCOPE_TMP_PATH "/tmp/appscope/"
 
 typedef unsigned int bool;
 #define TRUE 1

src/transport.c

@@ -21,6 +21,7 @@
 #include "scopestdlib.h"
 #include "fn.h"
 #include "transport.h"
+#include "utils.h"
 
 // Yuck.  Avoids naming conflict between our src/wrap.c and libssl.a
 #define SSL_read SCOPE_SSL_read
@@ -1018,37 +1019,6 @@ transportConnectFile(transport_t *t)
     return 1;
 }
 
-#define EDGE_PATH_DOCKER "/var/run/appscope/appscope.sock"
-#define EDGE_PATH_DEFAULT "/opt/cribl/state/appscope.sock"
-#define READ_AND_WRITE (R_OK|W_OK)
-static char*
-edgePath(void){
-    // 1) If EDGE_PATH_DOCKER can be accessed, return that.
-    if (scope_access(EDGE_PATH_DOCKER, READ_AND_WRITE) == 0) {
-        return scope_strdup(EDGE_PATH_DOCKER);
-    }
-
-    // 2) If CRIBL_HOME is defined and can be accessed,
-    //    return $CRIBL_HOME/state/appscope.sock
-    const char *cribl_home = getenv("CRIBL_HOME");
-    if (cribl_home) {
-        char *new_path = NULL;
-        if (scope_asprintf(&new_path, "%s/%s", cribl_home, "state/appscope.sock") > 0) {
-            if (scope_access(new_path, READ_AND_WRITE) == 0) {
-                return new_path;
-            }
-            scope_free(new_path);
-        }
-    }
-
-    // 3) If EDGE_PATH_DEFAULT can be accessed, return it
-    if (scope_access(EDGE_PATH_DEFAULT, READ_AND_WRITE) == 0) {
-        return scope_strdup(EDGE_PATH_DEFAULT);
-    }
-
-    return NULL;
-}
-
 int
 transportConnect(transport_t *trans)
 {

src/utils.c

@@ -494,3 +494,105 @@ sigSafeWriteNumber(int fd, long val, int base) {
     return scope_write(fd, buf ,msgLen);
 }
 
+/*
+ * Returns normalized version string
+ * - "dev" for unofficial release
+ * - "%d.%d.%d" for official release (e.g. "1.3.0" for "v1.3.0")
+ * - "%d.%d.%d-%s%d" for candidate release (e.g. "1.3.0-rc0" for "v1.3.0-rc0")
+ */
+const char *
+libVersion(const char *version) {
+
+    if ((version == NULL) || (*version != 'v')) {
+        return "dev";
+    }
+
+    ++version;
+    size_t versionSize = scope_strlen(version);
+
+    for (int i = 0; i < versionSize; ++i) {
+        // Only digit and "." are accepted
+        if ((scope_isdigit(version[i]) == 0) && version[i] != '.' &&
+            version[i] != '-' && version[i] != 't' && version[i] != 'c' &&
+            version[i] != 'r') {
+            return "dev";
+        }
+        if (i == 0 || i == versionSize) {
+            // First and last character must be number
+            if (scope_isdigit(version[i]) == 0) {
+                return "dev";
+            }
+        }
+    }
+    return version;
+}
+
+/*
+ * The mkdir system call does not support the creation of intermediate dirs.
+ * This will walk the path and create all dirs one at a time.
+ */
+int
+makeIntermediateDirs(const char *path, mode_t mode) {
+    char *dup_path = scope_strdup(path);
+    if (!dup_path) {
+        return -1;
+    }
+
+    char *curr = dup_path;
+    char *slash;
+
+    while ((slash = scope_strrchr(curr, '/'))) {
+        *slash = '\0';
+        if (scope_mkdir(dup_path, mode) == -1) {
+            if (scope_errno != EEXIST) {
+                scope_free(dup_path);
+                return -1;
+            }
+        }
+
+        *slash = '/';
+        curr = slash + 1;
+    }
+
+    if (scope_mkdir(dup_path, mode) == -1) {
+        if (scope_errno != EEXIST) {
+            scope_free(dup_path);
+            return -1;
+        }
+    }
+
+    scope_free(dup_path);
+    return 0;
+}
+
+#define EDGE_PATH_DOCKER "/var/run/appscope/appscope.sock"
+#define EDGE_PATH_DEFAULT "/opt/cribl/state/appscope.sock"
+#define READ_AND_WRITE (R_OK|W_OK)
+
+char *
+edgePath(void) {
+    // 1) If EDGE_PATH_DOCKER can be accessed, return that.
+    if (scope_access(EDGE_PATH_DOCKER, READ_AND_WRITE) == 0) {
+        return scope_strdup(EDGE_PATH_DOCKER);
+    }
+
+    // 2) If CRIBL_HOME is defined and can be accessed,
+    //    return $CRIBL_HOME/state/appscope.sock
+    const char *cribl_home = getenv("CRIBL_HOME");
+    if (cribl_home) {
+        char *new_path = NULL;
+        if (scope_asprintf(&new_path, "%s/%s", cribl_home, "state/appscope.sock") > 0) {
+            if (scope_access(new_path, READ_AND_WRITE) == 0) {
+                return new_path;
+            }
+            scope_free(new_path);
+        }
+    }
+
+    // 3) If EDGE_PATH_DEFAULT can be accessed, return it
+    if (scope_access(EDGE_PATH_DEFAULT, READ_AND_WRITE) == 0) {
+        return scope_strdup(EDGE_PATH_DEFAULT);
+    }
+
+    return NULL;
+}

src/utils.h

@@ -23,6 +23,10 @@ int endsWith(const char *, const char *);
 void setUUID(char *);
 void setMachineID(char *);
 
+char *edgePath(void);
+int makeIntermediateDirs(const char *, mode_t);
+const char *libVersion(const char *);
+
 // Signal Safe API
 int sigSafeNanosleep(const struct timespec *);
 void sigSafeUtoa(unsigned long, char *, int, int *);

src/wrap.c

@@ -1731,6 +1731,24 @@ init(void)
     bool scopedFlag = FALSE;
     bool skipReadCfg = FALSE;
 
+    /*
+     * This might be able to merge with the filter file check below.
+     * However, we dont want to check a specific file. We may update
+     * how we define the filter file. So, for now, leaving them
+     * distinct.
+     *
+     * Would be nice to use an env var without the need to access
+     * a file. However, we don't rely on something like SCOPE_LIB_PATH
+     * when LD_PRELOAD alone is used.
+     */
+    if (!scope_access(SCOPE_SYS_PATH, R_OK)) {
+        scope_strncpy(g_libpath, SCOPE_SYS_PATH, sizeof(SCOPE_SYS_PATH));
+    } else if (!scope_access(SCOPE_TMP_PATH, R_OK)) {
+        scope_strncpy(g_libpath, SCOPE_TMP_PATH, sizeof(SCOPE_TMP_PATH));
+    }
+
+    scope_strncat(g_libpath, libVersion(SCOPE_VER), sizeof(SCOPE_VER));
+
     if (attachedFlag) {
         scopedFlag = TRUE;
     } else {

src/wrap_go.c

@@ -367,7 +367,7 @@ adjustGoStructOffsetsForVersion()
             scopeLogWarn("Architecture not supported. Not adjusting schema offsets.");
             return;
         }
-        tap_entry(tap_syscall)->func_name = "runtime/internal/syscall.Syscall6";
+        tap_entry(tap_syscall)->func_name = "runtime/internal/syscall.Syscall6.abi0";
         tap_entry(tap_rawsyscall)->func_name = "";
         tap_entry(tap_syscall6)->func_name = "";
     }
@@ -793,7 +793,6 @@ patch_addrs(funchook_t *funchook,
             } else if (tap->assembly_fn == go_hook_reg_syscall6) {
                 g_syscall6_return = (uint64_t)tap->return_addr;
             }
-
             break; // Done patching
         }
 
@@ -1203,6 +1202,111 @@ getFDFromConn(uint64_t tcpConn) {
     return -1;
 }
 
+/*
+ * Create bind mounts in the overlay files within a container.
+ * Provides access to the library, filter, config, CLI and
+ * the Edge Unix socket in a container.
+ *
+ * Update ld.so.preload such that all procs load the library.
+ * Initailly intended for dockerd.
+ */
+bool
+mountDirs(char *src, char *target, char *fstype)
+{
+    static int once = 0;
+
+    if (scope_strstr(fstype, "overlay")) once++;
+    funcprint("Scope: mount(%d) of src: %s target: %s fstype: %s\n", once, src, target, fstype);
+
+    /*
+     * Checks:
+     * only applies to the dockerd proc
+     * ensure the constructor populated a path
+     * proceed only after container mounts are done
+     */
+    if ((src != NULL) && (target != NULL) &&
+        scope_strstr(g_proc.procname, "dockerd") &&
+        (scope_strstr(g_libpath, "/")) &&
+        (once == 3)) {
+        int ldfd;
+        char *filterdir;
+        size_t targetlen = scope_strlen(target);
+
+        funcprint("Interposed Mount overlay: %s %d\n", g_proc.procname, once);
+        once = 0;
+
+        if ((filterdir = scope_malloc(targetlen + 128)) == NULL) return FALSE;
+        scope_strcpy(filterdir, target);
+        scope_strcat(filterdir, g_libpath);
+
+        // make a dir in the merged dir
+        if (makeIntermediateDirs((const char *)filterdir, 0666) != 0) {
+            scopeLogWarn("Warn: mkdir of %s from %s:%d", filterdir, __FUNCTION__, __LINE__);
+            scope_free(filterdir);
+            return FALSE;
+        }
+
+        // mount the merged dir addition into the host FS
+        if (g_fn.mount(g_libpath, filterdir, "overlay", MS_BIND, NULL) != 0) {
+            scope_free(filterdir);
+            scopeLogWarn("WARN: mount %s on %s from %s:%d", g_libpath, filterdir, __FUNCTION__, __LINE__);
+            return FALSE;
+        }
+
+        // mount the Edge socket path if it exists
+        char *sockpath = edgePath();
+        if (sockpath) {
+            char *sockdir = scope_dirname(sockpath);
+            filterdir[targetlen + 1] = '\0';
+            scope_strcat(filterdir, sockdir);
+
+            // make the socket dir in the merged dir
+            if (makeIntermediateDirs((const char *)filterdir, 0666) == 0) {
+                // mount the Edge socket
+                if (g_fn.mount(sockdir, filterdir, "overlay", MS_BIND, NULL) != 0) {
+                    scopeLogWarn("WARN: mount %s on %s from %s:%d", sockpath, filterdir, __FUNCTION__, __LINE__);
+                }
+            } else {
+                scopeLogWarn("WARN: mkdir of %s from %s:%d", filterdir, __FUNCTION__, __LINE__);
+            }
+
+            scope_free(sockpath);
+        } else {
+            scopeLogInfo("Note: can't resolve the Edge socket path %s:%d", __FUNCTION__, __LINE__);
+        }
+
+        // write to /etc/ld.so.preload, not a mount
+        filterdir[targetlen + 1] = '\0';
+        scope_strcat(filterdir, "etc/ld.so.preload");
+
+        if ((ldfd = scope_open(filterdir, O_RDWR | O_CREAT | O_APPEND, 0666)) == -1) {
+            scopeLogWarn("WARN: open %s:%d", __FUNCTION__, __LINE__);
+            scope_free(filterdir);
+            return FALSE;
+        }
+
+        size_t liblen = strlen(g_libpath);
+        char libpath[liblen + sizeof("/libscope.so")];
+
+        scope_memset(libpath, 0, liblen + sizeof("/libscope.so"));
+        scope_strncpy(libpath, g_libpath, liblen);
+        scope_strcat(libpath, "/libscope.so");
+
+        if (scope_write(ldfd, libpath, sizeof(libpath)) <= 0) {
+            scopeLogWarn("WARN: write %s:%d", __FUNCTION__, __LINE__);
+            scope_close(ldfd);
+            scope_free(filterdir);
+            return FALSE;
+        }
+
+        scope_close(ldfd);
+        scope_free(filterdir);
+    }
+
+    return TRUE;
+}
+
+
 // Extract data from syscalls. Values are available in registers saved on sys_stack.
 static void
 c_syscall(char *sys_stack, char *g_stack)
@@ -1298,6 +1402,16 @@ c_syscall(char *sys_stack, char *g_stack)
             doCloseAndReportFailures(fd, (rc != -1), "go_close"); // If net, deletes a net object
         }
         break;
+    case SYS_mount:
+        {
+            char *src = (char *)*(uint64_t *)(sys_stack + g_go_schema->arg_offsets.c_syscall_p1);
+            char *target = (char *)*(uint64_t *)(sys_stack + g_go_schema->arg_offsets.c_syscall_p2);
+            char *fstype = (char *)*(uint64_t *)(sys_stack + g_go_schema->arg_offsets.c_syscall_p3);
+
+            mountDirs(src, target, fstype);
+        break;
+        }
+
     default:
         break;
     }