From 4a3770263ff5cd008499df1f4585410237098a99 Mon Sep 17 00:00:00 2001 From: Flavio Crisciani Date: Fri, 18 May 2018 14:10:14 -0700 Subject: [PATCH] Allows to set generic knobs on the Sandbox Refactor the ostweaks file to allows a more easy reuse Add a method on the osl.Sandbox interface to allow setting knobs on the sandbox Signed-off-by: Flavio Crisciani --- .gitignore | 4 +++ controller.go | 5 +++ drivers/overlay/ostweaks_linux.go | 55 ++----------------------------- network.go | 3 +- osl/kernel/knobs.go | 16 +++++++++ osl/kernel/knobs_linux.go | 47 ++++++++++++++++++++++++++ osl/kernel/knobs_unsupported.go | 8 +++++ osl/namespace_linux.go | 12 +++++++ osl/sandbox.go | 3 ++ sandbox.go | 8 +++++ 10 files changed, 108 insertions(+), 53 deletions(-) create mode 100644 osl/kernel/knobs.go create mode 100644 osl/kernel/knobs_linux.go create mode 100644 osl/kernel/knobs_unsupported.go diff --git a/.gitignore b/.gitignore index f5e6e52d7f..d99448027c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,11 @@ *.o *.a *.so +*~ +*.swp +tags bin/ +.gtm/ # Folders integration-tmp/ diff --git a/controller.go b/controller.go index f7b1d010d8..a4e2682669 100644 --- a/controller.go +++ b/controller.go @@ -1146,6 +1146,11 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (S } } + // Apply operating specific knobs on the load balancer sandbox + if sb.loadBalancer { + sb.osSbox.ApplyOSTweaks() + } + c.Lock() c.sandboxes[sb.id] = sb c.Unlock() diff --git a/drivers/overlay/ostweaks_linux.go b/drivers/overlay/ostweaks_linux.go index 68f1ee9cee..8677591ff3 100644 --- a/drivers/overlay/ostweaks_linux.go +++ b/drivers/overlay/ostweaks_linux.go @@ -1,72 +1,23 @@ package overlay import ( - "io/ioutil" - "path" "strconv" - "strings" - "github.com/sirupsen/logrus" + "github.com/docker/libnetwork/osl/kernel" ) -type conditionalCheck func(val1, val2 string) bool - -type osValue struct { - value string - checkFn conditionalCheck -} - -var osConfig = map[string]osValue{ +var ovConfig = map[string]kernel.OSValue{ "net.ipv4.neigh.default.gc_thresh1": {"8192", checkHigher}, "net.ipv4.neigh.default.gc_thresh2": {"49152", checkHigher}, "net.ipv4.neigh.default.gc_thresh3": {"65536", checkHigher}, } -func propertyIsValid(val1, val2 string, check conditionalCheck) bool { - if check == nil || check(val1, val2) { - return true - } - return false -} - func checkHigher(val1, val2 string) bool { val1Int, _ := strconv.ParseInt(val1, 10, 32) val2Int, _ := strconv.ParseInt(val2, 10, 32) return val1Int < val2Int } -// writeSystemProperty writes the value to a path under /proc/sys as determined from the key. -// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward. -func writeSystemProperty(key, value string) error { - keyPath := strings.Replace(key, ".", "/", -1) - return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644) -} - -func readSystemProperty(key string) (string, error) { - keyPath := strings.Replace(key, ".", "/", -1) - value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath)) - if err != nil { - return "", err - } - return string(value), nil -} - func applyOStweaks() { - for k, v := range osConfig { - // read the existing property from disk - oldv, err := readSystemProperty(k) - if err != nil { - logrus.Errorf("error reading the kernel parameter %s, error: %s", k, err) - continue - } - - if propertyIsValid(oldv, v.value, v.checkFn) { - // write new prop value to disk - if err := writeSystemProperty(k, v.value); err != nil { - logrus.Errorf("error setting the kernel parameter %s = %s, (leaving as %s) error: %s", k, v.value, oldv, err) - continue - } - logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.value, oldv) - } - } + kernel.ApplyOSTweaks(ovConfig) } diff --git a/network.go b/network.go index 3ed8f95357..095a499e32 100644 --- a/network.go +++ b/network.go @@ -2113,7 +2113,8 @@ func (n *network) lbEndpointName() string { func (n *network) createLoadBalancerSandbox() error { var err error sandboxName := n.lbSandboxName() - sbOptions := []SandboxOption{} + // Mark the sandbox to be a load balancer + sbOptions := []SandboxOption{OptionLoadBalancer()} if n.ingress { sbOptions = append(sbOptions, OptionIngress()) } diff --git a/osl/kernel/knobs.go b/osl/kernel/knobs.go new file mode 100644 index 0000000000..5088f0e7f9 --- /dev/null +++ b/osl/kernel/knobs.go @@ -0,0 +1,16 @@ +package kernel + +type conditionalCheck func(val1, val2 string) bool + +// OSValue represents a tuple, value defired, check function when to apply the value +type OSValue struct { + Value string + CheckFn conditionalCheck +} + +func propertyIsValid(val1, val2 string, check conditionalCheck) bool { + if check == nil || check(val1, val2) { + return true + } + return false +} diff --git a/osl/kernel/knobs_linux.go b/osl/kernel/knobs_linux.go new file mode 100644 index 0000000000..9937ca08c5 --- /dev/null +++ b/osl/kernel/knobs_linux.go @@ -0,0 +1,47 @@ +package kernel + +import ( + "io/ioutil" + "path" + "strings" + + "github.com/sirupsen/logrus" +) + +// writeSystemProperty writes the value to a path under /proc/sys as determined from the key. +// For e.g. net.ipv4.ip_forward translated to /proc/sys/net/ipv4/ip_forward. +func writeSystemProperty(key, value string) error { + keyPath := strings.Replace(key, ".", "/", -1) + return ioutil.WriteFile(path.Join("/proc/sys", keyPath), []byte(value), 0644) +} + +// readSystemProperty reads the value from the path under /proc/sys and returns it +func readSystemProperty(key string) (string, error) { + keyPath := strings.Replace(key, ".", "/", -1) + value, err := ioutil.ReadFile(path.Join("/proc/sys", keyPath)) + if err != nil { + return "", err + } + return string(value), nil +} + +// ApplyOSTweaks applies the configuration values passed as arguments +func ApplyOSTweaks(osConfig map[string]OSValue) { + for k, v := range osConfig { + // read the existing property from disk + oldv, err := readSystemProperty(k) + if err != nil { + logrus.WithError(err).Errorf("error reading the kernel parameter %s", k) + continue + } + + if propertyIsValid(oldv, v.Value, v.CheckFn) { + // write new prop value to disk + if err := writeSystemProperty(k, v.Value); err != nil { + logrus.WithError(err).Errorf("error setting the kernel parameter %s = %s, (leaving as %s)", k, v.Value, oldv) + continue + } + logrus.Debugf("updated kernel parameter %s = %s (was %s)", k, v.Value, oldv) + } + } +} diff --git a/osl/kernel/knobs_unsupported.go b/osl/kernel/knobs_unsupported.go new file mode 100644 index 0000000000..33325c160a --- /dev/null +++ b/osl/kernel/knobs_unsupported.go @@ -0,0 +1,8 @@ +// +build !linux + +package kernel + +// ApplyOSTweaks applies the configuration values passed as arguments +func ApplyOSTweaks(osConfig map[string]OSValue) { + return nil +} diff --git a/osl/namespace_linux.go b/osl/namespace_linux.go index a55932babe..e07a66d3d2 100644 --- a/osl/namespace_linux.go +++ b/osl/namespace_linux.go @@ -16,6 +16,7 @@ import ( "github.com/docker/docker/pkg/reexec" "github.com/docker/libnetwork/ns" + "github.com/docker/libnetwork/osl/kernel" "github.com/docker/libnetwork/types" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" @@ -630,3 +631,14 @@ func setIPv6(path, iface string, enable bool) error { } return nil } + +var sbConfig = map[string]kernel.OSValue{ + // expires connection from the IPVS connection table when the backend is not available + // more info: https://github.com/torvalds/linux/blob/master/Documentation/networking/ipvs-sysctl.txt#L126:1 + "net.ipv4.vs.expire_nodest_conn": {"1", nil}, +} + +// ApplyOSTweaks applies linux configs on the sandbox +func (n *networkNamespace) ApplyOSTweaks() { + kernel.ApplyOSTweaks(sbConfig) +} diff --git a/osl/sandbox.go b/osl/sandbox.go index 06149062fb..105df3ffa0 100644 --- a/osl/sandbox.go +++ b/osl/sandbox.go @@ -70,6 +70,9 @@ type Sandbox interface { // restore sandbox Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error + + // ApplyOSTweaks applies operating system specific knobs on the sandbox + ApplyOSTweaks() } // NeighborOptionSetter interface defines the option setter methods for interface options diff --git a/sandbox.go b/sandbox.go index 1837c1103f..e294f7c663 100644 --- a/sandbox.go +++ b/sandbox.go @@ -85,6 +85,7 @@ type sandbox struct { inDelete bool ingress bool ndotsSet bool + loadBalancer bool sync.Mutex // This mutex is used to serialize service related operation for an endpoint // The lock is here because the endpoint is saved into the store so is not unique @@ -1151,6 +1152,13 @@ func OptionIngress() SandboxOption { } } +// OptionLoadBalancer function returns an option setter for marking a +// sandbox as a load balancer sandbox. +func OptionLoadBalancer() SandboxOption { + return func(sb *sandbox) { + sb.loadBalancer = true + } +} func (eh epHeap) Len() int { return len(eh) } func (eh epHeap) Less(i, j int) bool {