Skip to content

Commit

Permalink
fork gomemlimit to fit go1.18
Browse files Browse the repository at this point in the history
  • Loading branch information
amyangfei committed Mar 27, 2023
1 parent 435e811 commit d011bf5
Show file tree
Hide file tree
Showing 8 changed files with 279 additions and 12 deletions.
5 changes: 2 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ go 1.18
require (
github.com/BurntSushi/toml v1.1.0
github.com/DATA-DOG/go-sqlmock v1.5.0
github.com/KimMachineGun/automemlimit v0.2.4
github.com/Shopify/sarama v1.36.0
github.com/apache/pulsar-client-go v0.6.0
github.com/aws/aws-sdk-go v1.35.3
Expand All @@ -15,6 +14,7 @@ require (
github.com/chaos-mesh/go-sqlsmith v0.0.0-20220512075501-53f2916ae240
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
github.com/cockroachdb/pebble v0.0.0-20211124172904-3ca75111760c
github.com/containerd/cgroups/v3 v3.0.1
github.com/coreos/go-semver v0.3.0
github.com/davecgh/go-spew v1.1.1
github.com/deepmap/oapi-codegen v1.9.0
Expand Down Expand Up @@ -122,12 +122,11 @@ require (
github.com/carlmjohnson/flagext v0.21.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cheggaaa/pb/v3 v3.0.8 // indirect
github.com/cilium/ebpf v0.4.0 // indirect
github.com/cilium/ebpf v0.9.1 // indirect
github.com/cockroachdb/errors v1.8.1 // indirect
github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
github.com/cockroachdb/redact v1.0.8 // indirect
github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 // indirect
github.com/coocood/freecache v1.2.1 // indirect
github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 // indirect
Expand Down
12 changes: 5 additions & 7 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,6 @@ github.com/Jeffail/gabs/v2 v2.5.1 h1:ANfZYjpMlfTTKebycu4X1AgkVWumFVDYQl7JwOr4mDk
github.com/Jeffail/gabs/v2 v2.5.1/go.mod h1:xCn81vdHKxFUuWWAaD5jCTQDNPBMh5pPs9IJ+NcziBI=
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM=
github.com/KimMachineGun/automemlimit v0.2.4 h1:GBty8TK8k0aJer1Pq5/3Vdt2ef+YpLhcqNo+PSD5CoI=
github.com/KimMachineGun/automemlimit v0.2.4/go.mod h1:38QAnnnNhnFuAIW3+aPlaVUHqzE9buJYZK3m/jsra8E=
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
Expand Down Expand Up @@ -185,8 +183,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5O
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cilium/ebpf v0.4.0 h1:QlHdikaxALkqWasW8hAC1mfR0jdmvbfaBdBPFmRSglA=
github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4=
github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
Expand Down Expand Up @@ -218,8 +216,8 @@ github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
github.com/colinmarc/hdfs/v2 v2.1.1/go.mod h1:M3x+k8UKKmxtFu++uAZ0OtDU8jR3jnaZIAc6yK4Ue0c=
github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA=
github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA=
github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE=
github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw=
github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ=
github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0=
github.com/coocood/freecache v1.2.1 h1:/v1CqMq45NFH9mp/Pt142reundeBM0dVUD3osQBeu/U=
Expand Down Expand Up @@ -336,8 +334,8 @@ github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9
github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU=
Expand Down
80 changes: 80 additions & 0 deletions pkg/util/cgroup/cgroups.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
//
// ============================================================
// Forked from https://github.com/KimMachineGun/automemlimit
//
// Written by Geon Kim <geon0250@gmail.com>
// limitations under the MIT License.

//go:build linux
// +build linux

package cgroup

import (
"github.com/containerd/cgroups/v3"
"github.com/containerd/cgroups/v3/cgroup1"
"github.com/containerd/cgroups/v3/cgroup2"
)

// FromCgroup returns the memory limit based on the cgroups version on this system.
func FromCgroup() (uint64, error) {
switch cgroups.Mode() {
case cgroups.Legacy:
return FromCgroupV1()
case cgroups.Hybrid, cgroups.Unified:
return FromCgroupV2()
}
return 0, ErrNoCgroup
}

// FromCgroupV1 returns the memory limit from the cgroup v1.
func FromCgroupV1() (uint64, error) {
cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy(
cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory),
))
if err != nil {
return 0, err
}

metrics, err := cg.Stat(cgroup1.IgnoreNotExist)
if err != nil {
return 0, err
} else if metrics.Memory == nil {
return 0, ErrNoLimit
}

return metrics.Memory.HierarchicalMemoryLimit, nil
}

// FromCgroupV2 returns the memory limit from the cgroup v2.
func FromCgroupV2() (uint64, error) {
path, err := cgroup2.NestedGroupPath("")
if err != nil {
return 0, err
}

m, err := cgroup2.Load(path, cgroup2.WithMountpoint(cgroupMountPoint))
if err != nil {
return 0, err
}

stats, err := m.Stat()
if err != nil {
return 0, err
} else if stats.Memory == nil {
return 0, ErrNoLimit
}

return stats.Memory.UsageLimit, nil
}
68 changes: 68 additions & 0 deletions pkg/util/cgroup/cgroups_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
//
// ============================================================
// Forked from https://github.com/KimMachineGun/automemlimit
//
// Written by Geon Kim <geon0250@gmail.com>
// limitations under the MIT License.

//go:build linux
// +build linux

package cgroup

import (
"testing"

"github.com/containerd/cgroups/v3"
)

func TestFromCgroup(t *testing.T) {
limit, err := FromCgroup()
if cgVersion == cgroups.Unavailable && err != ErrNoCgroup {
t.Fatalf("FromCgroup() error = %v, wantErr %v", err, ErrNoCgroup)
}

if err != nil {
t.Fatalf("FromCgroup() error = %v, wantErr %v", err, nil)
}
if limit != expected {
t.Fatalf("FromCgroup() got = %v, want %v", limit, expected)
}
}

func TestFromCgroupV1(t *testing.T) {
if cgVersion != cgroups.Legacy {
t.Skip("cgroups v1 is not supported")
}
limit, err := FromCgroupV1()
if err != nil {
t.Fatalf("FromCgroupV1() error = %v, wantErr %v", err, nil)
}
if limit != expected {
t.Fatalf("FromCgroupV1() got = %v, want %v", limit, expected)
}
}

func TestFromCgroupV2(t *testing.T) {
if cgVersion != cgroups.Hybrid && cgVersion != cgroups.Unified {
t.Skip("cgroups v2 is not supported")
}
limit, err := FromCgroupV2()
if err != nil {
t.Fatalf("FromCgroupV2() error = %v, wantErr %v", err, nil)
}
if limit != expected {
t.Fatalf("FromCgroupV2() got = %v, want %v", limit, expected)
}
}
34 changes: 34 additions & 0 deletions pkg/util/cgroup/cgroups_unsupported.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
//
// ============================================================
// Forked from https://github.com/KimMachineGun/automemlimit
//
// Written by Geon Kim <geon0250@gmail.com>
// limitations under the MIT License.

//go:build !linux
// +build !linux

package cgroup

func FromCgroup() (uint64, error) {
return 0, ErrCgroupsNotSupported
}

func FromCgroupV1() (uint64, error) {
return 0, ErrCgroupsNotSupported
}

func FromCgroupV2() (uint64, error) {
return 0, ErrCgroupsNotSupported
}
56 changes: 56 additions & 0 deletions pkg/util/cgroup/cgroups_unsupported_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
//
// ============================================================
// Forked from https://github.com/KimMachineGun/automemlimit
//
// Written by Geon Kim <geon0250@gmail.com>
// limitations under the MIT License.

//go:build !linux
// +build !linux

package cgroup

import (
"testing"
)

func TestFromCgroup(t *testing.T) {
limit, err := FromCgroup()
if err != ErrCgroupsNotSupported {
t.Fatalf("FromCgroup() error = %v, wantErr %v", err, ErrCgroupsNotSupported)
}
if limit != 0 {
t.Fatalf("FromCgroup() got = %v, want %v", limit, 0)
}
}

func TestFromCgroupV1(t *testing.T) {
limit, err := FromCgroupV1()
if err != ErrCgroupsNotSupported {
t.Fatalf("FromCgroupV1() error = %v, wantErr %v", err, ErrCgroupsNotSupported)
}
if limit != 0 {
t.Fatalf("FromCgroupV1() got = %v, want %v", limit, 0)
}
}

func TestFromCgroupV2(t *testing.T) {
limit, err := FromCgroupV2()
if err != ErrCgroupsNotSupported {
t.Fatalf("FromCgroupV2() error = %v, wantErr %v", err, ErrCgroupsNotSupported)
}
if limit != 0 {
t.Fatalf("FromCgroupV2() got = %v, want %v", limit, 0)
}
}
32 changes: 32 additions & 0 deletions pkg/util/cgroup/constant.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// Copyright 2023 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
//
// ============================================================
// Forked from https://github.com/KimMachineGun/automemlimit
//
// Written by Geon Kim <geon0250@gmail.com>
// limitations under the MIT License.

package cgroup

import "errors"

const (
cgroupMountPoint = "/sys/fs/cgroup"
)

var (
// ErrNoCgroup is returned when the process is not in cgroup.
ErrNoCgroup = errors.New("process is not in cgroup")
// ErrCgroupsNotSupported is returned when the system does not support cgroups.
ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system")
)
4 changes: 2 additions & 2 deletions pkg/util/memory.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ package util
import (
"math"

"github.com/KimMachineGun/automemlimit/memlimit"
"github.com/pingcap/log"
"github.com/pingcap/tidb/util/memory"
"github.com/pingcap/tiflow/pkg/errors"
"github.com/pingcap/tiflow/pkg/util/cgroup"
"go.uber.org/zap"
)

Expand All @@ -31,7 +31,7 @@ var MemTotal func() (uint64, error) = GetMemoryLimit
// If the cgourp is not set or memory.max is set to max, returns the available
// memory of host.
func GetMemoryLimit() (uint64, error) {
totalMemory, err := memlimit.FromCgroup()
totalMemory, err := cgroup.FromCgroup()
if err != nil || totalMemory == memoryMax {
log.Info("no cgroup memory limit", zap.Error(err))
totalMemory, err = memory.MemTotal()
Expand Down

0 comments on commit d011bf5

Please sign in to comment.