From 73545ec820059a6a7e227f9740ecc6d5811889b0 Mon Sep 17 00:00:00 2001 From: Minje Park Date: Sun, 23 Dec 2018 02:09:43 +0900 Subject: [PATCH 1/6] AvailableMemory is't calculated automatically under kenel 3.14 so it is needed to calcuate manually manual available memory calculation should be done if only memavail is false Manual Available Memory calculation under kernel 3.14 accurate manaul available memory calculation --- mem/mem_linux.go | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/mem/mem_linux.go b/mem/mem_linux.go index a3425d705..8f7660f1c 100644 --- a/mem/mem_linux.go +++ b/mem/mem_linux.go @@ -4,6 +4,8 @@ package mem import ( "context" + "math" + "os" "strconv" "strings" @@ -103,8 +105,9 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { ret.Cached += ret.SReclaimable if !memavail { - ret.Available = ret.Free + ret.Buffers + ret.Cached + ret.Available = calcuateAvailVmem(ret) } + ret.Used = ret.Total - ret.Free - ret.Buffers - ret.Cached ret.UsedPercent = float64(ret.Used) / float64(ret.Total) * 100.0 @@ -156,3 +159,39 @@ func SwapMemoryWithContext(ctx context.Context) (*SwapMemoryStat, error) { } return ret, nil } + +func calcuateAvailVmem(ret *VirtualMemoryStat) uint64 { + var watermarkLow uint64 + fn := common.HostProc("zoneinfo") + lines, _ := common.ReadLines(fn) + pagesize := uint64(os.Getpagesize()) + watermarkLow = 0 + + for _, line := range lines { + fields := strings.Fields(line) + + if strings.HasPrefix(fields[0], "low") { + lowValue, err := strconv.ParseUint(fields[1], 10, 64) + + if err != nil { + lowValue = 0 + } + + watermarkLow += lowValue + } + } + + watermarkLow *= pagesize + + availMemory := ret.Free - watermarkLow + pageCache := ret.Active + ret.Inactive + pageCache -= uint64(math.Min(float64(pageCache/2), float64(watermarkLow))) + availMemory += pageCache + availMemory += ret.SReclaimable - uint64(math.Min(float64(ret.SReclaimable/2.0), float64(watermarkLow))) + + if availMemory < 0 { + availMemory = 0 + } + + return availMemory +} From 9fb3b9625d4be8aa5eb6b87010e522e6eca0ad43 Mon Sep 17 00:00:00 2001 From: Minje Park Date: Wed, 30 Jan 2019 21:58:44 +0900 Subject: [PATCH 2/6] mem available fallback under kernel 2.6.13 --- mem/mem_linux.go | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/mem/mem_linux.go b/mem/mem_linux.go index 8f7660f1c..7e7ddcba9 100644 --- a/mem/mem_linux.go +++ b/mem/mem_linux.go @@ -22,6 +22,7 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { lines, _ := common.ReadLines(filename) // flag if MemAvailable is in /proc/meminfo (kernel 3.14+) memavail := false + memavailFallback := false ret := &VirtualMemoryStat{} for _, line := range lines { @@ -50,8 +51,10 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { case "Cached": ret.Cached = t * 1024 case "Active": + memavailFallback = true ret.Active = t * 1024 case "Inactive": + memavailFallback = true ret.Inactive = t * 1024 case "Writeback": ret.Writeback = t * 1024 @@ -64,6 +67,7 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { case "Slab": ret.Slab = t * 1024 case "SReclaimable": + memavailFallback = true ret.SReclaimable = t * 1024 case "PageTables": ret.PageTables = t * 1024 @@ -105,7 +109,7 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { ret.Cached += ret.SReclaimable if !memavail { - ret.Available = calcuateAvailVmem(ret) + ret.Available = calcuateAvailVmem(ret, memavailFallback) } ret.Used = ret.Total - ret.Free - ret.Buffers - ret.Cached @@ -160,10 +164,29 @@ func SwapMemoryWithContext(ctx context.Context) (*SwapMemoryStat, error) { return ret, nil } -func calcuateAvailVmem(ret *VirtualMemoryStat) uint64 { +// calcuateAvailVmem is a fallback under kernel 3.14 where /proc/meminfo does not provide +// "MemAvailable:" column. It reimplements an algorithm from the link below +// https://github.com/giampaolo/psutil/pull/890 +func calcuateAvailVmem(ret *VirtualMemoryStat, memavailFallback bool) uint64 { var watermarkLow uint64 + + fallback := ret.Free + ret.Cached + + if (memavailFallback) { + return fallback + } + + active := ret.Active + inactive := ret.Inactive + sreclaimable := ret.SReclaimable + fn := common.HostProc("zoneinfo") - lines, _ := common.ReadLines(fn) + lines, err := common.ReadLines(fn) + + if err != nil { + return fallback // fallback under kernel 2.6.13 + } + pagesize := uint64(os.Getpagesize()) watermarkLow = 0 @@ -184,10 +207,10 @@ func calcuateAvailVmem(ret *VirtualMemoryStat) uint64 { watermarkLow *= pagesize availMemory := ret.Free - watermarkLow - pageCache := ret.Active + ret.Inactive + pageCache := active + inactive pageCache -= uint64(math.Min(float64(pageCache/2), float64(watermarkLow))) availMemory += pageCache - availMemory += ret.SReclaimable - uint64(math.Min(float64(ret.SReclaimable/2.0), float64(watermarkLow))) + availMemory += ret.SReclaimable - uint64(math.Min(float64(sreclaimable/2.0), float64(watermarkLow))) if availMemory < 0 { availMemory = 0 From d7ce61cac83261c1ca1677f84dbecc21539abced Mon Sep 17 00:00:00 2001 From: Minje Park Date: Thu, 31 Jan 2019 00:13:52 +0900 Subject: [PATCH 3/6] Error checks and fallback for old linux --- mem/mem.go | 2 ++ mem/mem_linux.go | 42 ++++++++++++++++++++++-------------------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/mem/mem.go b/mem/mem.go index 995364fb9..e5a6b59a5 100644 --- a/mem/mem.go +++ b/mem/mem.go @@ -50,6 +50,8 @@ type VirtualMemoryStat struct { // https://www.centos.org/docs/5/html/5.1/Deployment_Guide/s2-proc-meminfo.html // https://www.kernel.org/doc/Documentation/filesystems/proc.txt // https://www.kernel.org/doc/Documentation/vm/overcommit-accounting + ActiveFile uint64 `json:activefile` + InactiveFile uint64 `json:inactivefile` Buffers uint64 `json:"buffers"` Cached uint64 `json:"cached"` Writeback uint64 `json:"writeback"` diff --git a/mem/mem_linux.go b/mem/mem_linux.go index 7e7ddcba9..7e46abd71 100644 --- a/mem/mem_linux.go +++ b/mem/mem_linux.go @@ -20,9 +20,12 @@ func VirtualMemory() (*VirtualMemoryStat, error) { func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { filename := common.HostProc("meminfo") lines, _ := common.ReadLines(filename) + // flag if MemAvailable is in /proc/meminfo (kernel 3.14+) memavail := false - memavailFallback := false + activeFile := false // "Active(file)" not available: 2.6.28 / Dec 2008 + inactiveFile := false // "Inactive(file)" not available: 2.6.28 / Dec 2008 + sReclaimable := false // "SReclaimable:" not available: 2.6.19 / Nov 2006 ret := &VirtualMemoryStat{} for _, line := range lines { @@ -51,11 +54,17 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { case "Cached": ret.Cached = t * 1024 case "Active": - memavailFallback = true + activeFile = true ret.Active = t * 1024 case "Inactive": - memavailFallback = true + inactiveFile = true ret.Inactive = t * 1024 + case "Active(file)": + activeFile = true + ret.ActiveFile = t * 1024 + case "InActive(file)": + inactiveFile = true + ret.InactiveFile = t * 1024 case "Writeback": ret.Writeback = t * 1024 case "WritebackTmp": @@ -67,7 +76,7 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { case "Slab": ret.Slab = t * 1024 case "SReclaimable": - memavailFallback = true + sReclaimable = true ret.SReclaimable = t * 1024 case "PageTables": ret.PageTables = t * 1024 @@ -109,7 +118,11 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { ret.Cached += ret.SReclaimable if !memavail { - ret.Available = calcuateAvailVmem(ret, memavailFallback) + if !(activeFile && inactiveFile && sReclaimable) { + ret.Available = ret.Cached + ret.Free + } else { + ret.Available = calcuateAvailVmem(ret) + } } ret.Used = ret.Total - ret.Free - ret.Buffers - ret.Cached @@ -167,24 +180,14 @@ func SwapMemoryWithContext(ctx context.Context) (*SwapMemoryStat, error) { // calcuateAvailVmem is a fallback under kernel 3.14 where /proc/meminfo does not provide // "MemAvailable:" column. It reimplements an algorithm from the link below // https://github.com/giampaolo/psutil/pull/890 -func calcuateAvailVmem(ret *VirtualMemoryStat, memavailFallback bool) uint64 { +func calcuateAvailVmem(ret *VirtualMemoryStat) uint64 { var watermarkLow uint64 - fallback := ret.Free + ret.Cached - - if (memavailFallback) { - return fallback - } - - active := ret.Active - inactive := ret.Inactive - sreclaimable := ret.SReclaimable - fn := common.HostProc("zoneinfo") lines, err := common.ReadLines(fn) if err != nil { - return fallback // fallback under kernel 2.6.13 + return ret.Free + ret.Cached // fallback under kernel 2.6.13 } pagesize := uint64(os.Getpagesize()) @@ -199,7 +202,6 @@ func calcuateAvailVmem(ret *VirtualMemoryStat, memavailFallback bool) uint64 { if err != nil { lowValue = 0 } - watermarkLow += lowValue } } @@ -207,10 +209,10 @@ func calcuateAvailVmem(ret *VirtualMemoryStat, memavailFallback bool) uint64 { watermarkLow *= pagesize availMemory := ret.Free - watermarkLow - pageCache := active + inactive + pageCache := ret.ActiveFile + ret.InactiveFile pageCache -= uint64(math.Min(float64(pageCache/2), float64(watermarkLow))) availMemory += pageCache - availMemory += ret.SReclaimable - uint64(math.Min(float64(sreclaimable/2.0), float64(watermarkLow))) + availMemory += ret.SReclaimable - uint64(math.Min(float64(ret.SReclaimable/2.0), float64(watermarkLow))) if availMemory < 0 { availMemory = 0 From d23abede783a5783c86d1202785ff0517321ace7 Mon Sep 17 00:00:00 2001 From: Minje Park Date: Thu, 31 Jan 2019 00:31:43 +0900 Subject: [PATCH 4/6] struct typo and broken test case were fixed --- mem/mem.go | 4 ++-- mem/mem_test.go | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/mem/mem.go b/mem/mem.go index e5a6b59a5..47dbb13ed 100644 --- a/mem/mem.go +++ b/mem/mem.go @@ -50,8 +50,8 @@ type VirtualMemoryStat struct { // https://www.centos.org/docs/5/html/5.1/Deployment_Guide/s2-proc-meminfo.html // https://www.kernel.org/doc/Documentation/filesystems/proc.txt // https://www.kernel.org/doc/Documentation/vm/overcommit-accounting - ActiveFile uint64 `json:activefile` - InactiveFile uint64 `json:inactivefile` + ActiveFile uint64 `json:"activefile"` + InactiveFile uint64 `json:"inactivefile"` Buffers uint64 `json:"buffers"` Cached uint64 `json:"cached"` Writeback uint64 `json:"writeback"` diff --git a/mem/mem_test.go b/mem/mem_test.go index 0651a61cb..fe3acfb6b 100644 --- a/mem/mem_test.go +++ b/mem/mem_test.go @@ -71,7 +71,10 @@ func TestVirtualMemoryStat_String(t *testing.T) { UsedPercent: 30.1, Free: 40, } - e := `{"total":10,"available":20,"used":30,"usedPercent":30.1,"free":40,"active":0,"inactive":0,"wired":0,"laundry":0,"buffers":0,"cached":0,"writeback":0,"dirty":0,"writebacktmp":0,"shared":0,"slab":0,"sreclaimable":0,"pagetables":0,"swapcached":0,"commitlimit":0,"committedas":0,"hightotal":0,"highfree":0,"lowtotal":0,"lowfree":0,"swaptotal":0,"swapfree":0,"mapped":0,"vmalloctotal":0,"vmallocused":0,"vmallocchunk":0,"hugepagestotal":0,"hugepagesfree":0,"hugepagesize":0}` + e := `{"total":10,"available":20,"used":30,"usedPercent":30.1,"free":40,"active":0,"inactive":0,"wired":0,"laundry":0,"activefile":0,"inactivefile":0,"buffers":0,"cached":0,"writeback":0,"dirty":0,"writebacktmp":0,"shared":0,"slab":0,"sreclaimable":0,"pagetables":0,"swapcached":0,"commitlimit":0,"committedas":0,"hightotal":0,"highfree":0,"lowtotal":0,"lowfree":0,"swaptotal":0,"swapfree":0,"mapped":0,"vmalloctotal":0,"vmallocused":0,"vmallocchunk":0,"hugepagestotal":0,"hugepagesfree":0,"hugepagesize":0}` + + fmt.Println(v) + fmt.Println(e) if e != fmt.Sprintf("%v", v) { t.Errorf("VirtualMemoryStat string is invalid: %v", v) } From 47ed6aaebd83708af12247e8cd09c3903e13c5fc Mon Sep 17 00:00:00 2001 From: Minje Park Date: Wed, 13 Feb 2019 00:55:05 +0900 Subject: [PATCH 5/6] activefile and inactivefile fields were removed from VirtualMemoryStat --- mem/mem.go | 2 -- mem/mem_linux.go | 23 ++++++++++++++--------- mem/mem_test.go | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/mem/mem.go b/mem/mem.go index 47dbb13ed..995364fb9 100644 --- a/mem/mem.go +++ b/mem/mem.go @@ -50,8 +50,6 @@ type VirtualMemoryStat struct { // https://www.centos.org/docs/5/html/5.1/Deployment_Guide/s2-proc-meminfo.html // https://www.kernel.org/doc/Documentation/filesystems/proc.txt // https://www.kernel.org/doc/Documentation/vm/overcommit-accounting - ActiveFile uint64 `json:"activefile"` - InactiveFile uint64 `json:"inactivefile"` Buffers uint64 `json:"buffers"` Cached uint64 `json:"cached"` Writeback uint64 `json:"writeback"` diff --git a/mem/mem_linux.go b/mem/mem_linux.go index 7e46abd71..0fe6209fc 100644 --- a/mem/mem_linux.go +++ b/mem/mem_linux.go @@ -13,6 +13,11 @@ import ( "golang.org/x/sys/unix" ) +type VirtualMemoryExStat struct { + ActiveFile uint64 `json:"activefile"` + InactiveFile uint64 `json:"inactivefile"` +} + func VirtualMemory() (*VirtualMemoryStat, error) { return VirtualMemoryWithContext(context.Background()) } @@ -28,6 +33,8 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { sReclaimable := false // "SReclaimable:" not available: 2.6.19 / Nov 2006 ret := &VirtualMemoryStat{} + retEx := &VirtualMemoryExStat{} + for _, line := range lines { fields := strings.Split(line, ":") if len(fields) != 2 { @@ -54,17 +61,15 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { case "Cached": ret.Cached = t * 1024 case "Active": - activeFile = true ret.Active = t * 1024 case "Inactive": - inactiveFile = true ret.Inactive = t * 1024 case "Active(file)": activeFile = true - ret.ActiveFile = t * 1024 + retEx.ActiveFile = t * 1024 case "InActive(file)": inactiveFile = true - ret.InactiveFile = t * 1024 + retEx.InactiveFile = t * 1024 case "Writeback": ret.Writeback = t * 1024 case "WritebackTmp": @@ -118,10 +123,10 @@ func VirtualMemoryWithContext(ctx context.Context) (*VirtualMemoryStat, error) { ret.Cached += ret.SReclaimable if !memavail { - if !(activeFile && inactiveFile && sReclaimable) { - ret.Available = ret.Cached + ret.Free + if (activeFile && inactiveFile && sReclaimable) { + ret.Available = calcuateAvailVmem(ret, retEx) } else { - ret.Available = calcuateAvailVmem(ret) + ret.Available = ret.Cached + ret.Free } } @@ -180,7 +185,7 @@ func SwapMemoryWithContext(ctx context.Context) (*SwapMemoryStat, error) { // calcuateAvailVmem is a fallback under kernel 3.14 where /proc/meminfo does not provide // "MemAvailable:" column. It reimplements an algorithm from the link below // https://github.com/giampaolo/psutil/pull/890 -func calcuateAvailVmem(ret *VirtualMemoryStat) uint64 { +func calcuateAvailVmem(ret *VirtualMemoryStat, retEx *VirtualMemoryExStat) uint64 { var watermarkLow uint64 fn := common.HostProc("zoneinfo") @@ -209,7 +214,7 @@ func calcuateAvailVmem(ret *VirtualMemoryStat) uint64 { watermarkLow *= pagesize availMemory := ret.Free - watermarkLow - pageCache := ret.ActiveFile + ret.InactiveFile + pageCache := retEx.ActiveFile + retEx.InactiveFile pageCache -= uint64(math.Min(float64(pageCache/2), float64(watermarkLow))) availMemory += pageCache availMemory += ret.SReclaimable - uint64(math.Min(float64(ret.SReclaimable/2.0), float64(watermarkLow))) diff --git a/mem/mem_test.go b/mem/mem_test.go index fe3acfb6b..c0272aaa8 100644 --- a/mem/mem_test.go +++ b/mem/mem_test.go @@ -71,7 +71,7 @@ func TestVirtualMemoryStat_String(t *testing.T) { UsedPercent: 30.1, Free: 40, } - e := `{"total":10,"available":20,"used":30,"usedPercent":30.1,"free":40,"active":0,"inactive":0,"wired":0,"laundry":0,"activefile":0,"inactivefile":0,"buffers":0,"cached":0,"writeback":0,"dirty":0,"writebacktmp":0,"shared":0,"slab":0,"sreclaimable":0,"pagetables":0,"swapcached":0,"commitlimit":0,"committedas":0,"hightotal":0,"highfree":0,"lowtotal":0,"lowfree":0,"swaptotal":0,"swapfree":0,"mapped":0,"vmalloctotal":0,"vmallocused":0,"vmallocchunk":0,"hugepagestotal":0,"hugepagesfree":0,"hugepagesize":0}` + e := `{"total":10,"available":20,"used":30,"usedPercent":30.1,"free":40,"active":0,"inactive":0,"wired":0,"laundry":0,"buffers":0,"cached":0,"writeback":0,"dirty":0,"writebacktmp":0,"shared":0,"slab":0,"sreclaimable":0,"pagetables":0,"swapcached":0,"commitlimit":0,"committedas":0,"hightotal":0,"highfree":0,"lowtotal":0,"lowfree":0,"swaptotal":0,"swapfree":0,"mapped":0,"vmalloctotal":0,"vmallocused":0,"vmallocchunk":0,"hugepagestotal":0,"hugepagesfree":0,"hugepagesize":0}` fmt.Println(v) fmt.Println(e) From c75deb9dc09cca7d4f9f2e51a0e2908fd53feb23 Mon Sep 17 00:00:00 2001 From: Minje Park Date: Wed, 13 Feb 2019 01:02:18 +0900 Subject: [PATCH 6/6] debug codes were removed --- mem/mem_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/mem/mem_test.go b/mem/mem_test.go index c0272aaa8..0651a61cb 100644 --- a/mem/mem_test.go +++ b/mem/mem_test.go @@ -72,9 +72,6 @@ func TestVirtualMemoryStat_String(t *testing.T) { Free: 40, } e := `{"total":10,"available":20,"used":30,"usedPercent":30.1,"free":40,"active":0,"inactive":0,"wired":0,"laundry":0,"buffers":0,"cached":0,"writeback":0,"dirty":0,"writebacktmp":0,"shared":0,"slab":0,"sreclaimable":0,"pagetables":0,"swapcached":0,"commitlimit":0,"committedas":0,"hightotal":0,"highfree":0,"lowtotal":0,"lowfree":0,"swaptotal":0,"swapfree":0,"mapped":0,"vmalloctotal":0,"vmallocused":0,"vmallocchunk":0,"hugepagestotal":0,"hugepagesfree":0,"hugepagesize":0}` - - fmt.Println(v) - fmt.Println(e) if e != fmt.Sprintf("%v", v) { t.Errorf("VirtualMemoryStat string is invalid: %v", v) }