diff --git a/cmd/cortex/main.go b/cmd/cortex/main.go index d8a4867752..5df30aa5ea 100644 --- a/cmd/cortex/main.go +++ b/cmd/cortex/main.go @@ -23,6 +23,8 @@ import ( _ "go.uber.org/automaxprocs" "gopkg.in/yaml.v2" + "github.com/KimMachineGun/automemlimit/memlimit" + "github.com/cortexproject/cortex/pkg/cortex" "github.com/cortexproject/cortex/pkg/tracing" "github.com/cortexproject/cortex/pkg/util" @@ -172,6 +174,17 @@ func main() { util_log.InitLogger(&cfg.Server) util.InitEvents(eventSampleRate) + if _, err := memlimit.SetGoMemLimitWithOpts( + memlimit.WithProvider( + memlimit.ApplyFallback( + memlimit.FromCgroup, + memlimit.FromSystem, + ), + ), + ); err != nil { + level.Warn(util_log.Logger).Log("msg", "Failed to set GOMEMLIMIT automatically", "err", err.Error()) + } + ctx, cancelFn := context.WithCancel(context.Background()) // In testing mode skip tracing setup to avoid panic due to // "duplicate metrics collector registration attempted" diff --git a/go.mod b/go.mod index e983393326..4b41824db6 100644 --- a/go.mod +++ b/go.mod @@ -75,6 +75,7 @@ require ( ) require ( + github.com/KimMachineGun/automemlimit v0.7.3 github.com/VictoriaMetrics/fastcache v1.12.2 github.com/aws/aws-sdk-go-v2 v1.38.3 github.com/aws/aws-sdk-go-v2/config v1.29.15 @@ -223,6 +224,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/exp/metrics v0.129.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.129.0 // indirect github.com/open-telemetry/opentelemetry-collector-contrib/processor/deltatocumulativeprocessor v0.129.0 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect diff --git a/go.sum b/go.sum index 98d6b6cbf8..d5f4a3a733 100644 --- a/go.sum +++ b/go.sum @@ -820,6 +820,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapp github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/KimMachineGun/automemlimit v0.7.3 h1:oPgMp0bsWez+4fvgSa11Rd9nUDrd8RLtDjBoT3ro+/A= +github.com/KimMachineGun/automemlimit v0.7.3/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= @@ -1584,6 +1586,8 @@ github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= diff --git a/vendor/github.com/KimMachineGun/automemlimit/LICENSE b/vendor/github.com/KimMachineGun/automemlimit/LICENSE new file mode 100644 index 0000000000..1f5b8f6b30 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Geon Kim + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go new file mode 100644 index 0000000000..69d1771f06 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups.go @@ -0,0 +1,408 @@ +package memlimit + +import ( + "bufio" + "errors" + "fmt" + "io" + "math" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +var ( + // ErrNoCgroup is returned when the process is not in cgroup. + ErrNoCgroup = errors.New("process is not in cgroup") + // ErrCgroupsNotSupported is returned when the system does not support cgroups. + ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system") +) + +// fromCgroup retrieves the memory limit from the cgroup. +// The versionDetector function is used to detect the cgroup version from the mountinfo. +func fromCgroup(versionDetector func(mis []mountInfo) (bool, bool)) (uint64, error) { + mf, err := os.Open("/proc/self/mountinfo") + if err != nil { + return 0, fmt.Errorf("failed to open /proc/self/mountinfo: %w", err) + } + defer mf.Close() + + mis, err := parseMountInfo(mf) + if err != nil { + return 0, fmt.Errorf("failed to parse mountinfo: %w", err) + } + + v1, v2 := versionDetector(mis) + if !(v1 || v2) { + return 0, ErrNoCgroup + } + + cf, err := os.Open("/proc/self/cgroup") + if err != nil { + return 0, fmt.Errorf("failed to open /proc/self/cgroup: %w", err) + } + defer cf.Close() + + chs, err := parseCgroupFile(cf) + if err != nil { + return 0, fmt.Errorf("failed to parse cgroup file: %w", err) + } + + if v2 { + limit, err := getMemoryLimitV2(chs, mis) + if err == nil { + return limit, nil + } else if !v1 { + return 0, err + } + } + + return getMemoryLimitV1(chs, mis) +} + +// detectCgroupVersion detects the cgroup version from the mountinfo. +func detectCgroupVersion(mis []mountInfo) (bool, bool) { + var v1, v2 bool + for _, mi := range mis { + switch mi.FilesystemType { + case "cgroup": + v1 = true + case "cgroup2": + v2 = true + } + } + return v1, v2 +} + +// getMemoryLimitV2 retrieves the memory limit from the cgroup v2 controller. +func getMemoryLimitV2(chs []cgroupHierarchy, mis []mountInfo) (uint64, error) { + // find the cgroup v2 path for the memory controller. + // in cgroup v2, the paths are unified and the controller list is empty. + idx := slices.IndexFunc(chs, func(ch cgroupHierarchy) bool { + return ch.HierarchyID == "0" && ch.ControllerList == "" + }) + if idx == -1 { + return 0, errors.New("cgroup v2 path not found") + } + relPath := chs[idx].CgroupPath + + // find the mountpoint for the cgroup v2 controller. + idx = slices.IndexFunc(mis, func(mi mountInfo) bool { + return mi.FilesystemType == "cgroup2" + }) + if idx == -1 { + return 0, errors.New("cgroup v2 mountpoint not found") + } + root, mountPoint := mis[idx].Root, mis[idx].MountPoint + + // resolve the actual cgroup path + cgroupPath, err := resolveCgroupPath(mountPoint, root, relPath) + if err != nil { + return 0, err + } + + // retrieve the memory limit from the memory.max file + return readMemoryLimitV2FromPath(filepath.Join(cgroupPath, "memory.max")) +} + +// readMemoryLimitV2FromPath reads the memory limit for cgroup v2 from the given path. +// this function expects the path to be memory.max file. +func readMemoryLimitV2FromPath(path string) (uint64, error) { + b, err := os.ReadFile(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return 0, ErrNoLimit + } + return 0, fmt.Errorf("failed to read memory.max: %w", err) + } + + slimit := strings.TrimSpace(string(b)) + if slimit == "max" { + return 0, ErrNoLimit + } + + limit, err := strconv.ParseUint(slimit, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse memory.max value: %w", err) + } + + return limit, nil +} + +// getMemoryLimitV1 retrieves the memory limit from the cgroup v1 controller. +func getMemoryLimitV1(chs []cgroupHierarchy, mis []mountInfo) (uint64, error) { + // find the cgroup v1 path for the memory controller. + idx := slices.IndexFunc(chs, func(ch cgroupHierarchy) bool { + return slices.Contains(strings.Split(ch.ControllerList, ","), "memory") + }) + if idx == -1 { + return 0, errors.New("cgroup v1 path for memory controller not found") + } + relPath := chs[idx].CgroupPath + + // find the mountpoint for the cgroup v1 controller. + idx = slices.IndexFunc(mis, func(mi mountInfo) bool { + return mi.FilesystemType == "cgroup" && slices.Contains(strings.Split(mi.SuperOptions, ","), "memory") + }) + if idx == -1 { + return 0, errors.New("cgroup v1 mountpoint for memory controller not found") + } + root, mountPoint := mis[idx].Root, mis[idx].MountPoint + + // resolve the actual cgroup path + cgroupPath, err := resolveCgroupPath(mountPoint, root, relPath) + if err != nil { + return 0, err + } + + // retrieve the memory limit from the memory.stats and memory.limit_in_bytes files. + return readMemoryLimitV1FromPath(cgroupPath) +} + +// getCgroupV1NoLimit returns the maximum value that is used to represent no limit in cgroup v1. +// the max memory limit is max int64, but it should be multiple of the page size. +func getCgroupV1NoLimit() uint64 { + ps := uint64(os.Getpagesize()) + return math.MaxInt64 / ps * ps +} + +// readMemoryLimitV1FromPath reads the memory limit for cgroup v1 from the given path. +// this function expects the path to be the cgroup directory. +func readMemoryLimitV1FromPath(cgroupPath string) (uint64, error) { + // read hierarchical_memory_limit and memory.limit_in_bytes files. + // but if hierarchical_memory_limit is not available, then use the max value as a fallback. + hml, err := readHierarchicalMemoryLimit(filepath.Join(cgroupPath, "memory.stats")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, fmt.Errorf("failed to read hierarchical_memory_limit: %w", err) + } else if hml == 0 { + hml = math.MaxUint64 + } + + // read memory.limit_in_bytes file. + b, err := os.ReadFile(filepath.Join(cgroupPath, "memory.limit_in_bytes")) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return 0, fmt.Errorf("failed to read memory.limit_in_bytes: %w", err) + } + lib, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse memory.limit_in_bytes value: %w", err) + } else if lib == 0 { + hml = math.MaxUint64 + } + + // use the minimum value between hierarchical_memory_limit and memory.limit_in_bytes. + // if the limit is the maximum value, then it is considered as no limit. + limit := min(hml, lib) + if limit >= getCgroupV1NoLimit() { + return 0, ErrNoLimit + } + + return limit, nil +} + +// readHierarchicalMemoryLimit extracts hierarchical_memory_limit from memory.stats. +// this function expects the path to be memory.stats file. +func readHierarchicalMemoryLimit(path string) (uint64, error) { + file, err := os.Open(path) + if err != nil { + return 0, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + + fields := strings.Split(line, " ") + if len(fields) < 2 { + return 0, fmt.Errorf("failed to parse memory.stats %q: not enough fields", line) + } + + if fields[0] == "hierarchical_memory_limit" { + if len(fields) > 2 { + return 0, fmt.Errorf("failed to parse memory.stats %q: too many fields for hierarchical_memory_limit", line) + } + return strconv.ParseUint(fields[1], 10, 64) + } + } + if err := scanner.Err(); err != nil { + return 0, err + } + + return 0, nil +} + +// https://www.man7.org/linux/man-pages/man5/proc_pid_mountinfo.5.html +// 731 771 0:59 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +// +// 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue +// (1)(2)(3) (4) (5) (6) (7) (8) (9) (10) (11) +// +// (1) mount ID: a unique ID for the mount (may be reused after umount(2)). +// (2) parent ID: the ID of the parent mount (or of self for the root of this mount namespace's mount tree). +// (3) major:minor: the value of st_dev for files on this filesystem (see stat(2)). +// (4) root: the pathname of the directory in the filesystem which forms the root of this mount. +// (5) mount point: the pathname of the mount point relative to the process's root directory. +// (6) mount options: per-mount options (see mount(2)). +// (7) optional fields: zero or more fields of the form "tag[:value]"; see below. +// (8) separator: the end of the optional fields is marked by a single hyphen. +// (9) filesystem type: the filesystem type in the form "type[.subtype]". +// (10) mount source: filesystem-specific information or "none". +// (11) super options: per-superblock options (see mount(2)). +type mountInfo struct { + Root string + MountPoint string + FilesystemType string + SuperOptions string +} + +// parseMountInfoLine parses a line from the mountinfo file. +func parseMountInfoLine(line string) (mountInfo, error) { + if line == "" { + return mountInfo{}, errors.New("empty line") + } + + fieldss := strings.SplitN(line, " - ", 2) + if len(fieldss) != 2 { + return mountInfo{}, fmt.Errorf("invalid separator") + } + + fields1 := strings.SplitN(fieldss[0], " ", 7) + if len(fields1) < 6 { + return mountInfo{}, fmt.Errorf("not enough fields before separator: %v", fields1) + } else if len(fields1) == 6 { + fields1 = append(fields1, "") + } + + fields2 := strings.SplitN(fieldss[1], " ", 3) + if len(fields2) < 3 { + return mountInfo{}, fmt.Errorf("not enough fields after separator: %v", fields2) + } + + return mountInfo{ + Root: fields1[3], + MountPoint: fields1[4], + FilesystemType: fields2[0], + SuperOptions: fields2[2], + }, nil +} + +// parseMountInfo parses the mountinfo file. +func parseMountInfo(r io.Reader) ([]mountInfo, error) { + var ( + s = bufio.NewScanner(r) + mis []mountInfo + ) + for s.Scan() { + line := s.Text() + + mi, err := parseMountInfoLine(line) + if err != nil { + return nil, fmt.Errorf("failed to parse mountinfo file %q: %w", line, err) + } + + mis = append(mis, mi) + } + if err := s.Err(); err != nil { + return nil, err + } + + return mis, nil +} + +// https://www.man7.org/linux/man-pages/man7/cgroups.7.html +// +// 5:cpuacct,cpu,cpuset:/daemons +// (1) (2) (3) +// +// (1) hierarchy ID: +// +// cgroups version 1 hierarchies, this field +// contains a unique hierarchy ID number that can be +// matched to a hierarchy ID in /proc/cgroups. For the +// cgroups version 2 hierarchy, this field contains the +// value 0. +// +// (2) controller list: +// +// For cgroups version 1 hierarchies, this field +// contains a comma-separated list of the controllers +// bound to the hierarchy. For the cgroups version 2 +// hierarchy, this field is empty. +// +// (3) cgroup path: +// +// This field contains the pathname of the control group +// in the hierarchy to which the process belongs. This +// pathname is relative to the mount point of the +// hierarchy. +type cgroupHierarchy struct { + HierarchyID string + ControllerList string + CgroupPath string +} + +// parseCgroupHierarchyLine parses a line from the cgroup file. +func parseCgroupHierarchyLine(line string) (cgroupHierarchy, error) { + if line == "" { + return cgroupHierarchy{}, errors.New("empty line") + } + + fields := strings.Split(line, ":") + if len(fields) < 3 { + return cgroupHierarchy{}, fmt.Errorf("not enough fields: %v", fields) + } else if len(fields) > 3 { + return cgroupHierarchy{}, fmt.Errorf("too many fields: %v", fields) + } + + return cgroupHierarchy{ + HierarchyID: fields[0], + ControllerList: fields[1], + CgroupPath: fields[2], + }, nil +} + +// parseCgroupFile parses the cgroup file. +func parseCgroupFile(r io.Reader) ([]cgroupHierarchy, error) { + var ( + s = bufio.NewScanner(r) + chs []cgroupHierarchy + ) + for s.Scan() { + line := s.Text() + + ch, err := parseCgroupHierarchyLine(line) + if err != nil { + return nil, fmt.Errorf("failed to parse cgroup file %q: %w", line, err) + } + + chs = append(chs, ch) + } + if err := s.Err(); err != nil { + return nil, err + } + + return chs, nil +} + +// resolveCgroupPath resolves the actual cgroup path from the mountpoint, root, and cgroupRelPath. +func resolveCgroupPath(mountpoint, root, cgroupRelPath string) (string, error) { + rel, err := filepath.Rel(root, cgroupRelPath) + if err != nil { + return "", err + } + + // if the relative path is ".", then the cgroupRelPath is the root itself. + if rel == "." { + return mountpoint, nil + } + + // if the relative path starts with "..", then it is outside the root. + if strings.HasPrefix(rel, "..") { + return "", fmt.Errorf("invalid cgroup path: %s is not under root %s", cgroupRelPath, root) + } + + return filepath.Join(mountpoint, rel), nil +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go new file mode 100644 index 0000000000..fd2c7e4973 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go @@ -0,0 +1,32 @@ +//go:build linux +// +build linux + +package memlimit + +// FromCgroup retrieves the memory limit from the cgroup. +func FromCgroup() (uint64, error) { + return fromCgroup(detectCgroupVersion) +} + +// FromCgroupV1 retrieves the memory limit from the cgroup v1 controller. +// After v1.0.0, this function could be removed and FromCgroup should be used instead. +func FromCgroupV1() (uint64, error) { + return fromCgroup(func(_ []mountInfo) (bool, bool) { + return true, false + }) +} + +// FromCgroupHybrid retrieves the memory limit from the cgroup v2 and v1 controller sequentially, +// basically, it is equivalent to FromCgroup. +// After v1.0.0, this function could be removed and FromCgroup should be used instead. +func FromCgroupHybrid() (uint64, error) { + return FromCgroup() +} + +// FromCgroupV2 retrieves the memory limit from the cgroup v2 controller. +// After v1.0.0, this function could be removed and FromCgroup should be used instead. +func FromCgroupV2() (uint64, error) { + return fromCgroup(func(_ []mountInfo) (bool, bool) { + return false, true + }) +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_unsupported.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_unsupported.go new file mode 100644 index 0000000000..9feca81a53 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_unsupported.go @@ -0,0 +1,20 @@ +//go:build !linux +// +build !linux + +package memlimit + +func FromCgroup() (uint64, error) { + return 0, ErrCgroupsNotSupported +} + +func FromCgroupV1() (uint64, error) { + return 0, ErrCgroupsNotSupported +} + +func FromCgroupHybrid() (uint64, error) { + return 0, ErrCgroupsNotSupported +} + +func FromCgroupV2() (uint64, error) { + return 0, ErrCgroupsNotSupported +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go new file mode 100644 index 0000000000..dee95f5207 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go @@ -0,0 +1,14 @@ +package memlimit + +import ( + "github.com/pbnjay/memory" +) + +// FromSystem returns the total memory of the system. +func FromSystem() (uint64, error) { + limit := memory.TotalMemory() + if limit == 0 { + return 0, ErrNoLimit + } + return limit, nil +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go new file mode 100644 index 0000000000..2a7c320edc --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go @@ -0,0 +1,59 @@ +package memlimit + +import ( + "fmt" + "os" + "reflect" + "strings" +) + +const ( + envAUTOMEMLIMIT_EXPERIMENT = "AUTOMEMLIMIT_EXPERIMENT" +) + +// Experiments is a set of experiment flags. +// It is used to enable experimental features. +// +// You can set the flags by setting the environment variable AUTOMEMLIMIT_EXPERIMENT. +// The value of the environment variable is a comma-separated list of experiment names. +// +// The following experiment names are known: +// +// - none: disable all experiments +// - system: enable fallback to system memory limit +type Experiments struct { + // System enables fallback to system memory limit. + System bool +} + +func parseExperiments() (Experiments, error) { + var exp Experiments + + // Create a map of known experiment names. + names := make(map[string]func(bool)) + rv := reflect.ValueOf(&exp).Elem() + rt := rv.Type() + for i := 0; i < rt.NumField(); i++ { + field := rv.Field(i) + names[strings.ToLower(rt.Field(i).Name)] = field.SetBool + } + + // Parse names. + for _, f := range strings.Split(os.Getenv(envAUTOMEMLIMIT_EXPERIMENT), ",") { + if f == "" { + continue + } + if f == "none" { + exp = Experiments{} + continue + } + val := true + set, ok := names[f] + if !ok { + return Experiments{}, fmt.Errorf("unknown AUTOMEMLIMIT_EXPERIMENT %s", f) + } + set(val) + } + + return exp, nil +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/logger.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/logger.go new file mode 100644 index 0000000000..4cf0b589d2 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/logger.go @@ -0,0 +1,13 @@ +package memlimit + +import ( + "context" + "log/slog" +) + +type noopLogger struct{} + +func (noopLogger) Enabled(context.Context, slog.Level) bool { return false } +func (noopLogger) Handle(context.Context, slog.Record) error { return nil } +func (d noopLogger) WithAttrs([]slog.Attr) slog.Handler { return d } +func (d noopLogger) WithGroup(string) slog.Handler { return d } diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go new file mode 100644 index 0000000000..b23980a515 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/memlimit.go @@ -0,0 +1,285 @@ +package memlimit + +import ( + "errors" + "fmt" + "log/slog" + "math" + "os" + "runtime/debug" + "strconv" + "time" +) + +const ( + envGOMEMLIMIT = "GOMEMLIMIT" + envAUTOMEMLIMIT = "AUTOMEMLIMIT" + // Deprecated: use memlimit.WithLogger instead + envAUTOMEMLIMIT_DEBUG = "AUTOMEMLIMIT_DEBUG" + + defaultAUTOMEMLIMIT = 0.9 +) + +// ErrNoLimit is returned when the memory limit is not set. +var ErrNoLimit = errors.New("memory is not limited") + +type config struct { + logger *slog.Logger + ratio float64 + provider Provider + refresh time.Duration +} + +// Option is a function that configures the behavior of SetGoMemLimitWithOptions. +type Option func(cfg *config) + +// WithRatio configures the ratio of the memory limit to set as GOMEMLIMIT. +// +// Default: 0.9 +func WithRatio(ratio float64) Option { + return func(cfg *config) { + cfg.ratio = ratio + } +} + +// WithProvider configures the provider. +// +// Default: FromCgroup +func WithProvider(provider Provider) Option { + return func(cfg *config) { + cfg.provider = provider + } +} + +// WithLogger configures the logger. +// It automatically attaches the "package" attribute to the logs. +// +// Default: slog.New(noopLogger{}) +func WithLogger(logger *slog.Logger) Option { + return func(cfg *config) { + cfg.logger = memlimitLogger(logger) + } +} + +// WithRefreshInterval configures the refresh interval for automemlimit. +// If a refresh interval is greater than 0, automemlimit periodically fetches +// the memory limit from the provider and reapplies it if it has changed. +// If the provider returns an error, it logs the error and continues. +// ErrNoLimit is treated as math.MaxInt64. +// +// Default: 0 (no refresh) +func WithRefreshInterval(refresh time.Duration) Option { + return func(cfg *config) { + cfg.refresh = refresh + } +} + +// WithEnv configures whether to use environment variables. +// +// Default: false +// +// Deprecated: currently this does nothing. +func WithEnv() Option { + return func(cfg *config) {} +} + +func memlimitLogger(logger *slog.Logger) *slog.Logger { + if logger == nil { + return slog.New(noopLogger{}) + } + return logger.With(slog.String("package", "github.com/KimMachineGun/automemlimit/memlimit")) +} + +// SetGoMemLimitWithOpts sets GOMEMLIMIT with options and environment variables. +// +// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT +// through AUTOMEMLIMIT environment variable in the half-open range (0.0,1.0]. +// +// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.) +// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing. +// +// If AUTOMEMLIMIT_EXPERIMENT is set, it enables experimental features. +// Please see the documentation of Experiments for more details. +// +// Options: +// - WithRatio +// - WithProvider +// - WithLogger +func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) { + // init config + cfg := &config{ + logger: slog.New(noopLogger{}), + ratio: defaultAUTOMEMLIMIT, + provider: FromCgroup, + } + // TODO: remove this + if debug, ok := os.LookupEnv(envAUTOMEMLIMIT_DEBUG); ok { + defaultLogger := memlimitLogger(slog.Default()) + defaultLogger.Warn("AUTOMEMLIMIT_DEBUG is deprecated, use memlimit.WithLogger instead") + if debug == "true" { + cfg.logger = defaultLogger + } + } + for _, opt := range opts { + opt(cfg) + } + + // log error if any on return + defer func() { + if _err != nil { + cfg.logger.Error("failed to set GOMEMLIMIT", slog.Any("error", _err)) + } + }() + + // parse experiments + exps, err := parseExperiments() + if err != nil { + return 0, fmt.Errorf("failed to parse experiments: %w", err) + } + if exps.System { + cfg.logger.Info("system experiment is enabled: using system memory limit as a fallback") + cfg.provider = ApplyFallback(cfg.provider, FromSystem) + } + + // rollback to previous memory limit on panic + snapshot := debug.SetMemoryLimit(-1) + defer rollbackOnPanic(cfg.logger, snapshot, &_err) + + // check if GOMEMLIMIT is already set + if val, ok := os.LookupEnv(envGOMEMLIMIT); ok { + cfg.logger.Info("GOMEMLIMIT is already set, skipping", slog.String(envGOMEMLIMIT, val)) + return 0, nil + } + + // parse AUTOMEMLIMIT + ratio := cfg.ratio + if val, ok := os.LookupEnv(envAUTOMEMLIMIT); ok { + if val == "off" { + cfg.logger.Info("AUTOMEMLIMIT is set to off, skipping") + return 0, nil + } + ratio, err = strconv.ParseFloat(val, 64) + if err != nil { + return 0, fmt.Errorf("cannot parse AUTOMEMLIMIT: %s", val) + } + } + + // apply ratio to the provider + provider := capProvider(ApplyRatio(cfg.provider, ratio)) + + // set the memory limit and start refresh + limit, err := updateGoMemLimit(uint64(snapshot), provider, cfg.logger) + refresh(provider, cfg.logger, cfg.refresh) + if err != nil { + if errors.Is(err, ErrNoLimit) { + cfg.logger.Info("memory is not limited, skipping") + // TODO: consider returning the snapshot + return 0, nil + } + return 0, fmt.Errorf("failed to set GOMEMLIMIT: %w", err) + } + + return int64(limit), nil +} + +// updateGoMemLimit updates the Go's memory limit, if it has changed. +func updateGoMemLimit(currLimit uint64, provider Provider, logger *slog.Logger) (uint64, error) { + newLimit, err := provider() + if err != nil { + return 0, err + } + + if newLimit == currLimit { + logger.Debug("GOMEMLIMIT is not changed, skipping", slog.Uint64(envGOMEMLIMIT, newLimit)) + return newLimit, nil + } + + debug.SetMemoryLimit(int64(newLimit)) + logger.Info("GOMEMLIMIT is updated", slog.Uint64(envGOMEMLIMIT, newLimit), slog.Uint64("previous", currLimit)) + + return newLimit, nil +} + +// refresh spawns a goroutine that runs every refresh duration and updates the GOMEMLIMIT if it has changed. +// See more details in the documentation of WithRefreshInterval. +func refresh(provider Provider, logger *slog.Logger, refresh time.Duration) { + if refresh == 0 { + return + } + + provider = noErrNoLimitProvider(provider) + + t := time.NewTicker(refresh) + go func() { + for range t.C { + err := func() (_err error) { + snapshot := debug.SetMemoryLimit(-1) + defer rollbackOnPanic(logger, snapshot, &_err) + + _, err := updateGoMemLimit(uint64(snapshot), provider, logger) + if err != nil { + return err + } + + return nil + }() + if err != nil { + logger.Error("failed to refresh GOMEMLIMIT", slog.Any("error", err)) + } + } + }() +} + +// rollbackOnPanic rollbacks to the snapshot on panic. +// Since it uses recover, it should be called in a deferred function. +func rollbackOnPanic(logger *slog.Logger, snapshot int64, err *error) { + panicErr := recover() + if panicErr != nil { + if *err != nil { + logger.Error("failed to set GOMEMLIMIT", slog.Any("error", *err)) + } + *err = fmt.Errorf("panic during setting the Go's memory limit, rolling back to previous limit %d: %v", + snapshot, panicErr, + ) + debug.SetMemoryLimit(snapshot) + } +} + +// SetGoMemLimitWithEnv sets GOMEMLIMIT with the value from the environment variables. +// Since WithEnv is deprecated, this function is equivalent to SetGoMemLimitWithOpts(). +// Deprecated: use SetGoMemLimitWithOpts instead. +func SetGoMemLimitWithEnv() { + _, _ = SetGoMemLimitWithOpts() +} + +// SetGoMemLimit sets GOMEMLIMIT with the value from the cgroup's memory limit and given ratio. +func SetGoMemLimit(ratio float64) (int64, error) { + return SetGoMemLimitWithOpts(WithRatio(ratio)) +} + +// SetGoMemLimitWithProvider sets GOMEMLIMIT with the value from the given provider and ratio. +func SetGoMemLimitWithProvider(provider Provider, ratio float64) (int64, error) { + return SetGoMemLimitWithOpts(WithProvider(provider), WithRatio(ratio)) +} + +func noErrNoLimitProvider(provider Provider) Provider { + return func() (uint64, error) { + limit, err := provider() + if errors.Is(err, ErrNoLimit) { + return math.MaxInt64, nil + } + return limit, err + } +} + +func capProvider(provider Provider) Provider { + return func() (uint64, error) { + limit, err := provider() + if err != nil { + return 0, err + } else if limit > math.MaxInt64 { + return math.MaxInt64, nil + } + return limit, nil + } +} diff --git a/vendor/github.com/KimMachineGun/automemlimit/memlimit/provider.go b/vendor/github.com/KimMachineGun/automemlimit/memlimit/provider.go new file mode 100644 index 0000000000..4f83770d13 --- /dev/null +++ b/vendor/github.com/KimMachineGun/automemlimit/memlimit/provider.go @@ -0,0 +1,43 @@ +package memlimit + +import ( + "fmt" +) + +// Provider is a function that returns the memory limit. +type Provider func() (uint64, error) + +// Limit is a helper Provider function that returns the given limit. +func Limit(limit uint64) func() (uint64, error) { + return func() (uint64, error) { + return limit, nil + } +} + +// ApplyRationA is a helper Provider function that applies the given ratio to the given provider. +func ApplyRatio(provider Provider, ratio float64) Provider { + if ratio == 1 { + return provider + } + return func() (uint64, error) { + if ratio <= 0 || ratio > 1 { + return 0, fmt.Errorf("invalid ratio: %f, ratio should be in the range (0.0,1.0]", ratio) + } + limit, err := provider() + if err != nil { + return 0, err + } + return uint64(float64(limit) * ratio), nil + } +} + +// ApplyFallback is a helper Provider function that sets the fallback provider. +func ApplyFallback(provider Provider, fallback Provider) Provider { + return func() (uint64, error) { + limit, err := provider() + if err != nil { + return fallback() + } + return limit, nil + } +} diff --git a/vendor/github.com/pbnjay/memory/LICENSE b/vendor/github.com/pbnjay/memory/LICENSE new file mode 100644 index 0000000000..63ca4a6d24 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2017, Jeremy Jay +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/pbnjay/memory/README.md b/vendor/github.com/pbnjay/memory/README.md new file mode 100644 index 0000000000..e98f261a0f --- /dev/null +++ b/vendor/github.com/pbnjay/memory/README.md @@ -0,0 +1,41 @@ +# memory + +Package `memory` provides two methods reporting total physical system memory +accessible to the kernel, and free memory available to the running application. + +This package has no external dependency besides the standard library and default operating system tools. + +Documentation: +[![GoDoc](https://godoc.org/github.com/pbnjay/memory?status.svg)](https://godoc.org/github.com/pbnjay/memory) + +This is useful for dynamic code to minimize thrashing and other contention, similar to the stdlib `runtime.NumCPU` +See some history of the proposal at https://github.com/golang/go/issues/21816 + + +## Example + +```go +fmt.Printf("Total system memory: %d\n", memory.TotalMemory()) +fmt.Printf("Free memory: %d\n", memory.FreeMemory()) +``` + + +## Testing + +Tested/working on: + - macOS 10.12.6 (16G29), 10.15.7 (19H2) + - Windows 10 1511 (10586.1045) + - Linux RHEL (3.10.0-327.3.1.el7.x86_64) + - Raspberry Pi 3 (ARMv8) on Raspbian, ODROID-C1+ (ARMv7) on Ubuntu, C.H.I.P + (ARMv7). + - Amazon Linux 2 aarch64 (m6a.large, 4.14.203-156.332.amzn2.aarch64) + +Tested on virtual machines: + - Windows 7 SP1 386 + - Debian stretch 386 + - NetBSD 7.1 amd64 + 386 + - OpenBSD 6.1 amd64 + 386 + - FreeBSD 11.1 amd64 + 386 + - DragonFly BSD 4.8.1 amd64 + +If you have access to untested systems please test and report success/bugs. diff --git a/vendor/github.com/pbnjay/memory/doc.go b/vendor/github.com/pbnjay/memory/doc.go new file mode 100644 index 0000000000..4e4f984c0f --- /dev/null +++ b/vendor/github.com/pbnjay/memory/doc.go @@ -0,0 +1,24 @@ +// Package memory provides a single method reporting total system memory +// accessible to the kernel. +package memory + +// TotalMemory returns the total accessible system memory in bytes. +// +// The total accessible memory is installed physical memory size minus reserved +// areas for the kernel and hardware, if such reservations are reported by +// the operating system. +// +// If accessible memory size could not be determined, then 0 is returned. +func TotalMemory() uint64 { + return sysTotalMemory() +} + +// FreeMemory returns the total free system memory in bytes. +// +// The total free memory is installed physical memory size minus reserved +// areas for other applications running on the same system. +// +// If free memory size could not be determined, then 0 is returned. +func FreeMemory() uint64 { + return sysFreeMemory() +} diff --git a/vendor/github.com/pbnjay/memory/memory_bsd.go b/vendor/github.com/pbnjay/memory/memory_bsd.go new file mode 100644 index 0000000000..49d808a9e7 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_bsd.go @@ -0,0 +1,19 @@ +// +build freebsd openbsd dragonfly netbsd + +package memory + +func sysTotalMemory() uint64 { + s, err := sysctlUint64("hw.physmem") + if err != nil { + return 0 + } + return s +} + +func sysFreeMemory() uint64 { + s, err := sysctlUint64("hw.usermem") + if err != nil { + return 0 + } + return s +} diff --git a/vendor/github.com/pbnjay/memory/memory_darwin.go b/vendor/github.com/pbnjay/memory/memory_darwin.go new file mode 100644 index 0000000000..a3f4576990 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_darwin.go @@ -0,0 +1,49 @@ +// +build darwin + +package memory + +import ( + "os/exec" + "regexp" + "strconv" +) + +func sysTotalMemory() uint64 { + s, err := sysctlUint64("hw.memsize") + if err != nil { + return 0 + } + return s +} + +func sysFreeMemory() uint64 { + cmd := exec.Command("vm_stat") + outBytes, err := cmd.Output() + if err != nil { + return 0 + } + + rePageSize := regexp.MustCompile("page size of ([0-9]*) bytes") + reFreePages := regexp.MustCompile("Pages free: *([0-9]*)\\.") + + // default: page size of 4096 bytes + matches := rePageSize.FindSubmatchIndex(outBytes) + pageSize := uint64(4096) + if len(matches) == 4 { + pageSize, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64) + if err != nil { + return 0 + } + } + + // ex: Pages free: 1126961. + matches = reFreePages.FindSubmatchIndex(outBytes) + freePages := uint64(0) + if len(matches) == 4 { + freePages, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64) + if err != nil { + return 0 + } + } + return freePages * pageSize +} diff --git a/vendor/github.com/pbnjay/memory/memory_linux.go b/vendor/github.com/pbnjay/memory/memory_linux.go new file mode 100644 index 0000000000..3d07711ce5 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_linux.go @@ -0,0 +1,29 @@ +// +build linux + +package memory + +import "syscall" + +func sysTotalMemory() uint64 { + in := &syscall.Sysinfo_t{} + err := syscall.Sysinfo(in) + if err != nil { + return 0 + } + // If this is a 32-bit system, then these fields are + // uint32 instead of uint64. + // So we always convert to uint64 to match signature. + return uint64(in.Totalram) * uint64(in.Unit) +} + +func sysFreeMemory() uint64 { + in := &syscall.Sysinfo_t{} + err := syscall.Sysinfo(in) + if err != nil { + return 0 + } + // If this is a 32-bit system, then these fields are + // uint32 instead of uint64. + // So we always convert to uint64 to match signature. + return uint64(in.Freeram) * uint64(in.Unit) +} diff --git a/vendor/github.com/pbnjay/memory/memory_windows.go b/vendor/github.com/pbnjay/memory/memory_windows.go new file mode 100644 index 0000000000..c8500cc6f3 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memory_windows.go @@ -0,0 +1,60 @@ +// +build windows + +package memory + +import ( + "syscall" + "unsafe" +) + +// omitting a few fields for brevity... +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366589(v=vs.85).aspx +type memStatusEx struct { + dwLength uint32 + dwMemoryLoad uint32 + ullTotalPhys uint64 + ullAvailPhys uint64 + unused [5]uint64 +} + +func sysTotalMemory() uint64 { + kernel32, err := syscall.LoadDLL("kernel32.dll") + if err != nil { + return 0 + } + // GetPhysicallyInstalledSystemMemory is simpler, but broken on + // older versions of windows (and uses this under the hood anyway). + globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx") + if err != nil { + return 0 + } + msx := &memStatusEx{ + dwLength: 64, + } + r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) + if r == 0 { + return 0 + } + return msx.ullTotalPhys +} + +func sysFreeMemory() uint64 { + kernel32, err := syscall.LoadDLL("kernel32.dll") + if err != nil { + return 0 + } + // GetPhysicallyInstalledSystemMemory is simpler, but broken on + // older versions of windows (and uses this under the hood anyway). + globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx") + if err != nil { + return 0 + } + msx := &memStatusEx{ + dwLength: 64, + } + r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx))) + if r == 0 { + return 0 + } + return msx.ullAvailPhys +} diff --git a/vendor/github.com/pbnjay/memory/memsysctl.go b/vendor/github.com/pbnjay/memory/memsysctl.go new file mode 100644 index 0000000000..438d9eff8e --- /dev/null +++ b/vendor/github.com/pbnjay/memory/memsysctl.go @@ -0,0 +1,21 @@ +// +build darwin freebsd openbsd dragonfly netbsd + +package memory + +import ( + "syscall" + "unsafe" +) + +func sysctlUint64(name string) (uint64, error) { + s, err := syscall.Sysctl(name) + if err != nil { + return 0, err + } + // hack because the string conversion above drops a \0 + b := []byte(s) + if len(b) < 8 { + b = append(b, 0) + } + return *(*uint64)(unsafe.Pointer(&b[0])), nil +} diff --git a/vendor/github.com/pbnjay/memory/stub.go b/vendor/github.com/pbnjay/memory/stub.go new file mode 100644 index 0000000000..f29473ba08 --- /dev/null +++ b/vendor/github.com/pbnjay/memory/stub.go @@ -0,0 +1,10 @@ +// +build !linux,!darwin,!windows,!freebsd,!dragonfly,!netbsd,!openbsd + +package memory + +func sysTotalMemory() uint64 { + return 0 +} +func sysFreeMemory() uint64 { + return 0 +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 41074bf8fc..4bb98ec2cf 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -129,6 +129,9 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric # github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.52.0 ## explicit; go 1.23.8 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping +# github.com/KimMachineGun/automemlimit v0.7.3 +## explicit; go 1.22.0 +github.com/KimMachineGun/automemlimit/memlimit # github.com/Masterminds/squirrel v1.5.4 ## explicit; go 1.14 github.com/Masterminds/squirrel @@ -939,6 +942,9 @@ github.com/parquet-go/parquet-go/internal/bytealg github.com/parquet-go/parquet-go/internal/debug github.com/parquet-go/parquet-go/internal/unsafecast github.com/parquet-go/parquet-go/sparse +# github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 +## explicit; go 1.16 +github.com/pbnjay/memory # github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c ## explicit; go 1.20 github.com/philhofer/fwd