diff --git a/go.mod b/go.mod index 41705ec5461..f86665d3daf 100644 --- a/go.mod +++ b/go.mod @@ -22,5 +22,6 @@ require ( // NOTE: urfave/cli must be <= v1.22.1 due to a regression: https://github.com/urfave/cli/issues/1092 github.com/urfave/cli v1.22.1 github.com/vishvananda/netlink v1.1.0 + github.com/willf/bitset v1.1.11 golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f ) diff --git a/go.sum b/go.sum index 8dd188fc776..cc533e29222 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,8 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7Zo github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243 h1:R43TdZy32XXSXjJn7M/HhALJ9imq6ztLnChfYJpVDnM= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/libcontainer/cgroups/systemd/common.go b/libcontainer/cgroups/systemd/common.go index d49754ba9a8..2207285626a 100644 --- a/libcontainer/cgroups/systemd/common.go +++ b/libcontainer/cgroups/systemd/common.go @@ -19,6 +19,13 @@ import ( "github.com/sirupsen/logrus" ) +const ( + // Default kernel value for cpu quota period is 100000 us (100 ms), same for v1 and v2. + // v1: https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html and + // v2: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html + defCPUQuotaPeriod = uint64(100000) +) + var ( connOnce sync.Once connDbus *systemdDbus.Conn @@ -410,10 +417,8 @@ func addCpuQuota(conn *systemdDbus.Conn, properties *[]systemdDbus.Property, quo cpuQuotaPerSecUSec := uint64(math.MaxUint64) if quota > 0 { if period == 0 { - // assume the default kernel value of 100000 us (100 ms), same for v1 and v2. - // v1: https://www.kernel.org/doc/html/latest/scheduler/sched-bwc.html and - // v2: https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html - period = 100000 + // assume the default + period = defCPUQuotaPeriod } // systemd converts CPUQuotaPerSecUSec (microseconds per CPU second) to CPUQuota // (integer percentage of CPU) internally. This means that if a fractional percent of diff --git a/libcontainer/cgroups/systemd/cpuset.go b/libcontainer/cgroups/systemd/cpuset.go new file mode 100644 index 00000000000..07098218883 --- /dev/null +++ b/libcontainer/cgroups/systemd/cpuset.go @@ -0,0 +1,67 @@ +package systemd + +import ( + "encoding/binary" + "strconv" + "strings" + + "github.com/pkg/errors" + "github.com/willf/bitset" +) + +// rangeToBits converts a text representation of a CPU mask (as written to +// or read from cgroups' cpuset.* files, e.g. "1,3-5") to a slice of bytes +// with the corresponding bits set (as consumed by systemd over dbus as +// AllowedCPUs/AllowedMemoryNodes unit property value). +func rangeToBits(str string) ([]byte, error) { + bits := &bitset.BitSet{} + + for _, r := range strings.Split(str, ",") { + // allow extra spaces around + r = strings.TrimSpace(r) + // allow empty elements (extra commas) + if r == "" { + continue + } + ranges := strings.SplitN(r, "-", 2) + if len(ranges) > 1 { + start, err := strconv.ParseUint(ranges[0], 10, 32) + if err != nil { + return nil, err + } + end, err := strconv.ParseUint(ranges[1], 10, 32) + if err != nil { + return nil, err + } + if start > end { + return nil, errors.New("invalid range: " + r) + } + for i := uint(start); i <= uint(end); i++ { + bits.Set(i) + } + } else { + val, err := strconv.ParseUint(ranges[0], 10, 32) + if err != nil { + return nil, err + } + bits.Set(uint(val)) + } + } + + val := bits.Bytes() + if len(val) == 0 { + // do not allow empty values + return nil, errors.New("empty value") + } + ret := make([]byte, len(val)*8) + for i := range val { + // bitset uses BigEndian internally + binary.BigEndian.PutUint64(ret[i*8:], val[len(val)-1-i]) + } + // remove upper all-zero bytes + for ret[0] == 0 { + ret = ret[1:] + } + + return ret, nil +} diff --git a/libcontainer/cgroups/systemd/cpuset_test.go b/libcontainer/cgroups/systemd/cpuset_test.go new file mode 100644 index 00000000000..6f8fde07324 --- /dev/null +++ b/libcontainer/cgroups/systemd/cpuset_test.go @@ -0,0 +1,58 @@ +package systemd + +import ( + "bytes" + "testing" +) + +func TestRangeToBits(t *testing.T) { + testCases := []struct { + in string + out []byte + isErr bool + }{ + {in: "", isErr: true}, + {in: "0", out: []byte{1}}, + {in: "1", out: []byte{2}}, + {in: "0-1", out: []byte{3}}, + {in: "0,1", out: []byte{3}}, + {in: ",0,1,", out: []byte{3}}, + {in: "0-3", out: []byte{0x0f}}, + {in: "0,1,2-3", out: []byte{0x0f}}, + {in: "4-7", out: []byte{0xf0}}, + {in: "0-7", out: []byte{0xff}}, + {in: "0-15", out: []byte{0xff, 0xff}}, + {in: "16", out: []byte{1, 0, 0}}, + {in: "0-3,32-33", out: []byte{3, 0, 0, 0, 0x0f}}, + // extra spaces and tabs are ok + {in: "1, 2, 1-2", out: []byte{6}}, + {in: " , 1 , 3 , 5-7, ", out: []byte{0xea}}, + // somewhat large values + {in: "128-130,1", out: []byte{7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}}, + + {in: "-", isErr: true}, + {in: "1-", isErr: true}, + {in: "-3", isErr: true}, + // bad range (start > end) + {in: "54-53", isErr: true}, + // kernel does not allow extra spaces inside a range + {in: "1 - 2", isErr: true}, + } + + for _, tc := range testCases { + t.Logf("case: %q", tc.in) + out, err := rangeToBits(tc.in) + if err != nil { + t.Logf(" got error: %s", err) + if !tc.isErr { + t.Error(" ^^^ unexpected error") + } + + continue + } + t.Logf(" expected %v, got %v", tc.out, out) + if !bytes.Equal(out, tc.out) { + t.Error(" ^^^ unexpected result") + } + } +} diff --git a/libcontainer/cgroups/systemd/v2.go b/libcontainer/cgroups/systemd/v2.go index e1a6622a0d3..5691b43dfb2 100644 --- a/libcontainer/cgroups/systemd/v2.go +++ b/libcontainer/cgroups/systemd/v2.go @@ -3,6 +3,8 @@ package systemd import ( + "fmt" + "math" "os" "path/filepath" "strconv" @@ -34,6 +36,125 @@ func NewUnifiedManager(config *configs.Cgroup, path string, rootless bool) cgrou } } +// unifiedResToSystemdProps tries to convert from Cgroup.Resources.Unified +// key/value map (where key is cgroupfs file name) to systemd unit properties. +// This is on a best-effort basis, so the properties that are not known +// (to this function and/or systemd) are ignored (but logged with "debug" +// log level). +// +// For the list of keys, see https://www.kernel.org/doc/Documentation/cgroup-v2.txt +// +// For the list of systemd unit properties, see systemd.resource-control(5). +func unifiedResToSystemdProps(conn *systemdDbus.Conn, res map[string]string) (props []systemdDbus.Property, _ error) { + var err error + + for k, v := range res { + if strings.Contains(k, "/") { + return nil, fmt.Errorf("unified resource %q must be a file name (no slashes)", k) + } + sk := strings.SplitN(k, ".", 2) + if len(sk) != 2 { + return nil, fmt.Errorf("unified resource %q must be in the form CONTROLLER.PARAMETER", k) + } + // Kernel is quite forgiving to extra whitespace + // around the value, and so should we. + v = strings.TrimSpace(v) + // Please keep cases in alphabetical order. + switch k { + case "cpu.max": + // value: quota [period] + quota := int64(0) // 0 means "unlimited" for addCpuQuota, if period is set + period := defCPUQuotaPeriod + sv := strings.Fields(v) + if len(sv) < 1 || len(sv) > 2 { + return nil, fmt.Errorf("unified resource %q value invalid: %q", k, v) + } + // quota + if sv[0] != "max" { + quota, err = strconv.ParseInt(sv[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("unified resource %q period value conversion error: %w", k, err) + } + } + // period + if len(sv) == 2 { + period, err = strconv.ParseUint(sv[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("unified resource %q quota value conversion error: %w", k, err) + } + } + addCpuQuota(conn, &props, quota, period) + + case "cpu.weight": + num, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("unified resource %q value conversion error: %w", k, err) + } + props = append(props, + newProp("CPUWeight", num)) + + case "cpuset.cpus", "cpuset.mems": + bits, err := rangeToBits(v) + if err != nil { + return nil, fmt.Errorf("unified resource %q=%q conversion error: %w", k, v, err) + } + m := map[string]string{ + "cpuset.cpus": "AllowedCPUs", + "cpuset.mems": "AllowedMemoryNodes", + } + props = append(props, + newProp(m[k], bits)) + + case "memory.high", "memory.low", "memory.min", "memory.max", "memory.swap.max": + num := uint64(math.MaxUint64) + if v != "max" { + num, err = strconv.ParseUint(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("unified resource %q value conversion error: %w", k, err) + } + } + m := map[string]string{ + "memory.high": "MemoryHigh", + "memory.low": "MemoryLow", + "memory.min": "MemoryMin", + "memory.max": "MemoryMax", + "memory.swap.max": "MemorySwapMax", + } + props = append(props, + newProp(m[k], num)) + + case "pids.max": + num := uint64(math.MaxUint64) + if v != "max" { + var err error + num, err = strconv.ParseUint(v, 10, 64) + if err != nil { + return nil, fmt.Errorf("unified resource %q value conversion error: %w", k, err) + } + } + props = append(props, + newProp("TasksAccounting", true), + newProp("TasksMax", num)) + + case "memory.oom.group": + // Setting this to 1 is roughly equivalent to OOMPolicy=kill + // (as per systemd.service(5) and + // https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html), + // but it's not clear what to do if it is unset or set + // to 0 in runc update, as there are two other possible + // values for OOMPolicy (continue/stop). + fallthrough + + default: + // Ignore the unknown resource here -- will still be + // applied in Set which calls fs2.Set. + logrus.Debugf("don't know how to convert unified resource %q=%q to systemd unit property; skipping (will still be applied to cgroupfs)", k, v) + } + } + + return props, nil +} + func genV2ResourcesProperties(c *configs.Cgroup, conn *systemdDbus.Conn) ([]systemdDbus.Property, error) { var properties []systemdDbus.Property r := c.Resources @@ -82,6 +203,15 @@ func genV2ResourcesProperties(c *configs.Cgroup, conn *systemdDbus.Conn) ([]syst // ignore r.KernelMemory + // convert Resources.Unified map to systemd properties + if r.Unified != nil { + unifiedProps, err := unifiedResToSystemdProps(conn, r.Unified) + if err != nil { + return nil, err + } + properties = append(properties, unifiedProps...) + } + return properties, nil } diff --git a/tests/integration/cgroups.bats b/tests/integration/cgroups.bats index eb0d97a53e3..a319ed04218 100644 --- a/tests/integration/cgroups.bats +++ b/tests/integration/cgroups.bats @@ -195,13 +195,15 @@ function setup() { set_cgroups_path "$BUSYBOX_BUNDLE" update_config ' .linux.resources.unified |= { - "memory.min": "131072", - "memory.low": "524288", - "memory.high": "5242880", - "memory.max": "10485760", - "pids.max": "99", - "cpu.max": "10000 100000" - }' "$BUSYBOX_BUNDLE" + "memory.min": "131072", + "memory.low": "524288", + "memory.high": "5242880", + "memory.max": "10485760", + "memory.swap.max": "20971520", + "pids.max": "99", + "cpu.max": "10000 100000", + "cpu.weight": "42" + }' "$BUSYBOX_BUNDLE" runc run -d --console-socket "$CONSOLE_SOCKET" test_cgroups_unified [ "$status" -eq 0 ] @@ -214,18 +216,39 @@ function setup() { echo "$output" | grep -q '^memory.low:524288$' echo "$output" | grep -q '^memory.high:5242880$' echo "$output" | grep -q '^memory.max:10485760$' + echo "$output" | grep -q '^memory.swap.max:20971520$' echo "$output" | grep -q '^pids.max:99$' echo "$output" | grep -q '^cpu.max:10000 100000$' + + check_systemd_value "MemoryMin" 131072 + check_systemd_value "MemoryLow" 524288 + check_systemd_value "MemoryHigh" 5242880 + check_systemd_value "MemoryMax" 10485760 + check_systemd_value "MemorySwapMax" 20971520 + check_systemd_value "TasksMax" 99 + check_cpu_quota 10000 100000 "100ms" + check_cpu_weight 42 } + @test "runc run (cgroup v2 resources.unified override)" { requires root cgroups_v2 set_cgroups_path "$BUSYBOX_BUNDLE" - update_config ' .linux.resources.memory |= {"limit": 33554432} - | .linux.resources.memorySwap |= {"limit": 33554432} - | .linux.resources.unified |= - {"memory.min": "131072", "memory.max": "10485760" }' \ - "$BUSYBOX_BUNDLE" + # CPU shares of 3333 corresponds to CPU weight of 128. + update_config ' .linux.resources.memory |= {"limit": 33554432} + | .linux.resources.memorySwap |= {"limit": 33554432} + | .linux.resources.cpu |= { + "shares": 3333, + "quota": 40000, + "period": 100000 + } + | .linux.resources.unified |= { + "memory.min": "131072", + "memory.max": "10485760", + "pids.max": "42", + "cpu.max": "5000 50000", + "cpu.weight": "42" + }' "$BUSYBOX_BUNDLE" runc run -d --console-socket "$CONSOLE_SOCKET" test_cgroups_unified [ "$status" -eq 0 ] @@ -237,4 +260,13 @@ function setup() { runc exec test_cgroups_unified cat /sys/fs/cgroup/memory.max [ "$status" -eq 0 ] [ "$output" = '10485760' ] + + runc exec test_cgroups_unified cat /sys/fs/cgroup/pids.max + [ "$status" -eq 0 ] + [ "$output" = '42' ] + check_systemd_value "TasksMax" 42 + + check_cpu_quota 5000 50000 "100ms" + + check_cpu_weight 42 } diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash index b7f5a48a5c5..cea6deda6a3 100644 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -207,6 +207,55 @@ function check_systemd_value() { [ "$current" = "$expected" ] || [ -n "$expected2" -a "$current" = "$expected2" ] } +function check_cpu_quota() { + local quota=$1 + local period=$2 + local sd_quota=$3 + + if [ "$CGROUP_UNIFIED" = "yes" ]; then + if [ "$quota" = "-1" ]; then + quota="max" + fi + check_cgroup_value "cpu.max" "$quota $period" + else + check_cgroup_value "cpu.cfs_quota_us" $quota + check_cgroup_value "cpu.cfs_period_us" "$period" + fi + # systemd values are the same for v1 and v2 + check_systemd_value "CPUQuotaPerSecUSec" "$sd_quota" + + # CPUQuotaPeriodUSec requires systemd >= v242 + [ "$(systemd_version)" -lt 242 ] && return + + local sd_period=$((period / 1000))ms + [ "$sd_period" = "1000ms" ] && sd_period="1s" + local sd_infinity="" + # 100ms is the default value, and if not set, shown as infinity + [ "$sd_period" = "100ms" ] && sd_infinity="infinity" + check_systemd_value "CPUQuotaPeriodUSec" $sd_period $sd_infinity +} + +# Works for cgroup v1 and v2, accepts v1 shares as an argument. +function check_cpu_shares() { + local shares=$1 + + if [ "$CGROUP_UNIFIED" = "yes" ]; then + local weight=$((1 + ((shares - 2) * 9999) / 262142)) + check_cpu_weight "$weight" + else + check_cgroup_value "cpu.shares" "$shares" + check_systemd_value "CPUShares" "$shares" + fi +} + +# Works only for cgroup v2, accept v2 weight. +function check_cpu_weight() { + local weight=$1 + + check_cgroup_value "cpu.weight" $weight + check_systemd_value "CPUWeight" $weight +} + # Helper function to set a resources limit function set_resources_limit() { bundle="${1:-.}" @@ -308,6 +357,12 @@ function requires() { skip_me=1 fi ;; + smp) + local cpu_count=$(grep -c '^processor' /proc/cpuinfo) + if [ "$cpu_count" -lt 2 ]; then + skip_me=1 + fi + ;; systemd) if [ -z "${RUNC_USE_SYSTEMD}" ]; then skip_me=1 diff --git a/tests/integration/update.bats b/tests/integration/update.bats index 72a4c2bf564..908ce0ccbf0 100644 --- a/tests/integration/update.bats +++ b/tests/integration/update.bats @@ -240,47 +240,6 @@ EOF check_systemd_value "TasksMax" 20 } -function check_cpu_quota() { - local quota=$1 - local period=$2 - local sd_quota=$3 - - if [ "$CGROUP_UNIFIED" = "yes" ]; then - if [ "$quota" = "-1" ]; then - quota="max" - fi - check_cgroup_value "cpu.max" "$quota $period" - else - check_cgroup_value "cpu.cfs_quota_us" $quota - check_cgroup_value "cpu.cfs_period_us" "$period" - fi - # systemd values are the same for v1 and v2 - check_systemd_value "CPUQuotaPerSecUSec" "$sd_quota" - - # CPUQuotaPeriodUSec requires systemd >= v242 - [ "$(systemd_version)" -lt 242 ] && return - - local sd_period=$((period / 1000))ms - [ "$sd_period" = "1000ms" ] && sd_period="1s" - local sd_infinity="" - # 100ms is the default value, and if not set, shown as infinity - [ "$sd_period" = "100ms" ] && sd_infinity="infinity" - check_systemd_value "CPUQuotaPeriodUSec" $sd_period $sd_infinity -} - -function check_cpu_shares() { - local shares=$1 - - if [ "$CGROUP_UNIFIED" = "yes" ]; then - local weight=$((1 + ((shares - 2) * 9999) / 262142)) - check_cgroup_value "cpu.weight" $weight - check_systemd_value "CPUWeight" $weight - else - check_cgroup_value "cpu.shares" "$shares" - check_systemd_value "CPUShares" "$shares" - fi -} - @test "update cgroup cpu limits" { [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup @@ -347,6 +306,7 @@ EOF } } EOF + [ "$status" -eq 0 ] runc update -r "$BATS_TMPDIR"/runc-cgroups-integration-test.json test_update [ "$status" -eq 0 ] @@ -403,6 +363,82 @@ EOF check_cpu_quota 30000 100000 "300ms" } +@test "update cgroup v2 resources via unified map" { + [[ "$ROOTLESS" -ne 0 ]] && requires rootless_cgroup + requires cgroups_v2 + + runc run -d --console-socket "$CONSOLE_SOCKET" test_update + [ "$status" -eq 0 ] + + # check that initial values were properly set + check_cpu_quota 500000 1000000 "500ms" + # initial cpu shares of 100 corresponds to weight of 4 + check_cpu_weight 4 + check_systemd_value "TasksMax" 20 + + runc update -r - test_update < -# @link https://github.com/willf/bitset -# ------------------------------------------------------------------------------ - -# List special make targets that are not associated with files -.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke - -# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS). -SHELL=/bin/bash - -# CVS path (path to the parent dir containing the project) -CVSPATH=github.com/willf - -# Project owner -OWNER=willf - -# Project vendor -VENDOR=willf - -# Project name -PROJECT=bitset - -# Project version -VERSION=$(shell cat VERSION) - -# Name of RPM or DEB package -PKGNAME=${VENDOR}-${PROJECT} - -# Current directory -CURRENTDIR=$(shell pwd) - -# GO lang path -ifneq ($(GOPATH),) - ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),) - # the defined GOPATH is not valid - GOPATH= - endif -endif -ifeq ($(GOPATH),) - # extract the GOPATH - GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR))) -endif - -# --- MAKE TARGETS --- - -# Display general help about this command -help: - @echo "" - @echo "$(PROJECT) Makefile." - @echo "GOPATH=$(GOPATH)" - @echo "The following commands are available:" - @echo "" - @echo " make qa : Run all the tests" - @echo " make test : Run the unit tests" - @echo "" - @echo " make format : Format the source code" - @echo " make fmtcheck : Check if the source code has been formatted" - @echo " make vet : Check for suspicious constructs" - @echo " make lint : Check for style errors" - @echo " make coverage : Generate the coverage report" - @echo " make cyclo : Generate the cyclomatic complexity report" - @echo " make ineffassign : Detect ineffectual assignments" - @echo " make misspell : Detect commonly misspelled words in source files" - @echo " make structcheck : Find unused struct fields" - @echo " make varcheck : Find unused global variables and constants" - @echo " make errcheck : Check that error return values are used" - @echo " make gosimple : Suggest code simplifications" - @echo " make astscan : GO AST scanner" - @echo "" - @echo " make docs : Generate source code documentation" - @echo "" - @echo " make deps : Get the dependencies" - @echo " make clean : Remove any build artifact" - @echo " make nuke : Deletes any intermediate file" - @echo "" - -# Alias for help target -all: help - -# Run the unit tests -test: - @mkdir -p target/test - @mkdir -p target/report - GOPATH=$(GOPATH) \ - go test \ - -covermode=atomic \ - -bench=. \ - -race \ - -cpuprofile=target/report/cpu.out \ - -memprofile=target/report/mem.out \ - -mutexprofile=target/report/mutex.out \ - -coverprofile=target/report/coverage.out \ - -v ./... | \ - tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \ - test $${PIPESTATUS[0]} -eq 0 - -# Format the source code -format: - @find . -type f -name "*.go" -exec gofmt -s -w {} \; - -# Check if the source code has been formatted -fmtcheck: - @mkdir -p target - @find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff - @test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; } - -# Check for syntax errors -vet: - GOPATH=$(GOPATH) go vet . - -# Check for style errors -lint: - GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint . - -# Generate the coverage report -coverage: - @mkdir -p target/report - GOPATH=$(GOPATH) \ - go tool cover -html=target/report/coverage.out -o target/report/coverage.html - -# Report cyclomatic complexity -cyclo: - @mkdir -p target/report - GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0 - -# Detect ineffectual assignments -ineffassign: - @mkdir -p target/report - GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0 - -# Detect commonly misspelled words in source files -misspell: - @mkdir -p target/report - GOPATH=$(GOPATH) misspell -error ./ | tee target/report/misspell.txt ; test $${PIPESTATUS[0]} -eq 0 - -# Find unused struct fields -structcheck: - @mkdir -p target/report - GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt - -# Find unused global variables and constants -varcheck: - @mkdir -p target/report - GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt - -# Check that error return values are used -errcheck: - @mkdir -p target/report - GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt - -# AST scanner -astscan: - @mkdir -p target/report - GOPATH=$(GOPATH) gosec . | tee target/report/astscan.txt ; test $${PIPESTATUS[0]} -eq 0 || true - -# Generate source docs -docs: - @mkdir -p target/docs - nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 & - wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060` - @echo ''${PKGNAME}' Documentation ...' > target/docs/index.html - -# Alias to run all quality-assurance checks -qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan - -# --- INSTALL --- - -# Get the dependencies -deps: - GOPATH=$(GOPATH) go get ./... - GOPATH=$(GOPATH) go get golang.org/x/lint/golint - GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report - GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov - GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo - GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign - GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell - GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck - GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck - GOPATH=$(GOPATH) go get github.com/kisielk/errcheck - GOPATH=$(GOPATH) go get github.com/securego/gosec/cmd/gosec/... - -# Remove any build artifact -clean: - GOPATH=$(GOPATH) go clean ./... - -# Deletes any intermediate file -nuke: - rm -rf ./target - GOPATH=$(GOPATH) go clean -i ./... diff --git a/vendor/github.com/willf/bitset/README.md b/vendor/github.com/willf/bitset/README.md index 6c62b20c6c8..50338e71dfd 100644 --- a/vendor/github.com/willf/bitset/README.md +++ b/vendor/github.com/willf/bitset/README.md @@ -2,10 +2,10 @@ *Go language library to map between non-negative integers and boolean values* -[![Master Build Status](https://secure.travis-ci.org/willf/bitset.png?branch=master)](https://travis-ci.org/willf/bitset?branch=master) +[![Test](https://github.com/willf/bitset/workflows/Test/badge.svg)](https://github.com/willf/bitset/actions?query=workflow%3ATest) [![Master Coverage Status](https://coveralls.io/repos/willf/bitset/badge.svg?branch=master&service=github)](https://coveralls.io/github/willf/bitset?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/willf/bitset)](https://goreportcard.com/report/github.com/willf/bitset) -[![GoDoc](https://godoc.org/github.com/willf/bitset?status.svg)](http://godoc.org/github.com/willf/bitset) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/willf/bitset?tab=doc)](https://pkg.go.dev/github.com/willf/bitset?tab=doc) ## Description @@ -63,8 +63,11 @@ func main() { As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. -Godoc documentation is at: https://godoc.org/github.com/willf/bitset +Package documentation is at: https://pkg.go.dev/github.com/willf/bitset?tab=doc +## Memory Usage + +The memory usage of a bitset using N bits is at least N/8 bytes. The number of bits in a bitset is at least as large as one plus the greatest bit index you have accessed. Thus it is possible to run out of memory while using a bitset. If you have lots of bits, you might prefer compressed bitsets, like the [Roaring bitmaps](http://roaringbitmap.org) and its [Go implementation](https://github.com/RoaringBitmap/roaring). ## Implementation Note @@ -82,15 +85,10 @@ go get github.com/willf/bitset If you wish to contribute to this project, please branch and issue a pull request against master ("[GitHub Flow](https://guides.github.com/introduction/flow/)") -This project include a Makefile that allows you to test and build the project with simple commands. -To see all available options: -```bash -make help -``` - ## Running all tests -Before committing the code, please check if it passes all tests using (note: this will install some dependencies): +Before committing the code, please check if it passes tests, has adequate coverage, etc. ```bash -make qa +go test +go test -cover ``` diff --git a/vendor/github.com/willf/bitset/bitset.go b/vendor/github.com/willf/bitset/bitset.go index 22e5d42e5d6..21e889da2e0 100644 --- a/vendor/github.com/willf/bitset/bitset.go +++ b/vendor/github.com/willf/bitset/bitset.go @@ -138,6 +138,9 @@ func (b *BitSet) Len() uint { // extendSetMaybe adds additional words to incorporate new bits if needed func (b *BitSet) extendSetMaybe(i uint) { if i >= b.length { // if we need more bits, make 'em + if i >= Cap() { + panic("You are exceeding the capacity") + } nsize := wordsNeeded(i + 1) if b.set == nil { b.set = make([]uint64, nsize) @@ -160,7 +163,12 @@ func (b *BitSet) Test(i uint) bool { return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 } -// Set bit i to 1 +// Set bit i to 1, the capacity of the bitset is automatically +// increased accordingly. +// If i>= Cap(), this function will panic. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. func (b *BitSet) Set(i uint) *BitSet { b.extendSetMaybe(i) b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) @@ -176,7 +184,11 @@ func (b *BitSet) Clear(i uint) *BitSet { return b } -// SetTo sets bit i to value +// SetTo sets bit i to value. +// If i>= Cap(), this function will panic. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. func (b *BitSet) SetTo(i uint, value bool) *BitSet { if value { return b.Set(i) @@ -184,7 +196,11 @@ func (b *BitSet) SetTo(i uint, value bool) *BitSet { return b.Clear(i) } -// Flip bit at i +// Flip bit at i. +// If i>= Cap(), this function will panic. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. func (b *BitSet) Flip(i uint) *BitSet { if i >= b.length { return b.Set(i) @@ -193,26 +209,51 @@ func (b *BitSet) Flip(i uint) *BitSet { return b } -// Shrink shrinks BitSet to desired length in bits. It clears all bits > length -// and reduces the size and length of the set. +// Shrink shrinks BitSet so that the provided value is the last possible +// set value. It clears all bits > the provided index and reduces the size +// and length of the set. +// +// Note that the parameter value is not the new length in bits: it is the +// maximal value that can be stored in the bitset after the function call. +// The new length in bits is the parameter value + 1. Thus it is not possible +// to use this function to set the length to 0, the minimal value of the length +// after this function call is 1. // // A new slice is allocated to store the new bits, so you may see an increase in // memory usage until the GC runs. Normally this should not be a problem, but if you // have an extremely large BitSet its important to understand that the old BitSet will // remain in memory until the GC frees it. -func (b *BitSet) Shrink(length uint) *BitSet { - idx := wordsNeeded(length + 1) +func (b *BitSet) Shrink(lastbitindex uint) *BitSet { + length := lastbitindex + 1 + idx := wordsNeeded(length) if idx > len(b.set) { return b } shrunk := make([]uint64, idx) copy(shrunk, b.set[:idx]) b.set = shrunk - b.length = length + 1 - b.set[idx-1] &= (allBits >> (uint64(64) - uint64(length&(wordSize-1)) - 1)) + b.length = length + b.set[idx-1] &= (allBits >> (uint64(64) - uint64(length&(wordSize-1)))) return b } +// Compact shrinks BitSet to so that we preserve all set bits, while minimizing +// memory usage. Compact calls Shrink. +func (b *BitSet) Compact() *BitSet { + idx := len(b.set) - 1 + for ; idx >= 0 && b.set[idx] == 0; idx-- { + } + newlength := uint((idx + 1) << log2WordSize) + if newlength >= b.length { + return b // nothing to do + } + if newlength > 0 { + return b.Shrink(newlength - 1) + } + // We preserve one word + return b.Shrink(63) +} + // InsertAt takes an index which indicates where a bit should be // inserted. Then it shifts all the bits in the set to the left by 1, starting // from the given index position, and sets the index position to 0. @@ -323,6 +364,9 @@ func (b *BitSet) DeleteAt(i uint) *BitSet { // including possibly the current index // along with an error code (true = valid, false = no set bit found) // for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} +// +// Users concerned with performance may want to use NextSetMany to +// retrieve several values at once. func (b *BitSet) NextSet(i uint) (uint, bool) { x := int(i >> log2WordSize) if x >= len(b.set) { @@ -358,6 +402,14 @@ func (b *BitSet) NextSet(i uint) (uint, bool) { // j += 1 // } // +// +// It is possible to retrieve all set bits as follow: +// +// indices := make([]uint, bitmap.Count()) +// bitmap.NextSetMany(0, indices) +// +// However if bitmap.Count() is large, it might be preferable to +// use several calls to NextSetMany, for performance reasons. func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) { myanswer := buffer capacity := cap(buffer) @@ -809,7 +861,7 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { newset := New(uint(length)) if uint64(newset.length) != length { - return 0, errors.New("Unmarshalling error: type mismatch") + return 0, errors.New("unmarshalling error: type mismatch") } // Read remaining bytes as set diff --git a/vendor/github.com/willf/bitset/go.mod b/vendor/github.com/willf/bitset/go.mod new file mode 100644 index 00000000000..583ecab78f7 --- /dev/null +++ b/vendor/github.com/willf/bitset/go.mod @@ -0,0 +1,3 @@ +module github.com/willf/bitset + +go 1.14 diff --git a/vendor/github.com/willf/bitset/go.sum b/vendor/github.com/willf/bitset/go.sum new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vendor/modules.txt b/vendor/modules.txt index c63cecfde5d..dee3f070bd2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -69,7 +69,8 @@ github.com/vishvananda/netlink github.com/vishvananda/netlink/nl # github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df github.com/vishvananda/netns -# github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243 +# github.com/willf/bitset v1.1.11 +## explicit github.com/willf/bitset # golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f ## explicit