stats: modernize stats collection and implement Pod level stats reporting #5427
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| package cgmgr | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "os" | ||
| "path/filepath" | ||
| "strconv" | ||
| "strings" | ||
| "syscall" | ||
| "time" | ||
|
|
||
| "github.com/containers/podman/v3/pkg/cgroups" | ||
| "github.com/cri-o/cri-o/internal/config/node" | ||
| "github.com/cri-o/cri-o/server/cri/types" | ||
| "github.com/pkg/errors" | ||
| "github.com/sirupsen/logrus" | ||
| ) | ||
|
|
||
| func populateSandboxCgroupStatsFromPath(cgroupPath string, stats *types.PodSandboxStats) error { | ||
| cgroupStats, err := cgroupStatsFromPath(cgroupPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| systemNano := time.Now().UnixNano() | ||
| stats.CPU = createCPUStats(systemNano, cgroupStats) | ||
| stats.Process = createProcessUsage(systemNano, cgroupStats) | ||
| stats.Memory, err = createMemoryStats(systemNano, cgroupStats, cgroupPath) | ||
| return err | ||
| } | ||
|
|
||
| func populateContainerCgroupStatsFromPath(cgroupPath string, stats *types.ContainerStats) error { | ||
| // checks cgroup just for the container, not the entire pod | ||
| cgroupStats, err := cgroupStatsFromPath(cgroupPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| systemNano := time.Now().UnixNano() | ||
| stats.CPU = createCPUStats(systemNano, cgroupStats) | ||
| stats.Memory, err = createMemoryStats(systemNano, cgroupStats, cgroupPath) | ||
| return err | ||
| } | ||
|
|
||
| func cgroupStatsFromPath(cgroupPath string) (*cgroups.Metrics, error) { | ||
| cg, err := cgroups.Load(cgroupPath) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "unable to load cgroup at %s", cgroupPath) | ||
| } | ||
|
|
||
| return cg.Stat() | ||
| } | ||
|
|
||
| func createCPUStats(systemNano int64, cgroupStats *cgroups.Metrics) *types.CPUUsage { | ||
| return &types.CPUUsage{ | ||
| Timestamp: systemNano, | ||
| UsageCoreNanoSeconds: &types.UInt64Value{Value: cgroupStats.CPU.Usage.Total}, | ||
| } | ||
| } | ||
|
|
||
| func createMemoryStats(systemNano int64, cgroupStats *cgroups.Metrics, cgroupPath string) (*types.MemoryUsage, error) { | ||
| memUsage := cgroupStats.Memory.Usage.Usage | ||
| memLimit := MemLimitGivenSystem(cgroupStats.Memory.Usage.Limit) | ||
|
|
||
| memory := &types.MemoryUsage{ | ||
| Timestamp: systemNano, | ||
| WorkingSetBytes: &types.UInt64Value{}, | ||
| RssBytes: &types.UInt64Value{}, | ||
| PageFaults: &types.UInt64Value{}, | ||
| MajorPageFaults: &types.UInt64Value{}, | ||
| UsageBytes: &types.UInt64Value{Value: memUsage}, | ||
| AvailableBytes: &types.UInt64Value{Value: memUsage - memLimit}, | ||
| } | ||
|
|
||
| if err := updateWithMemoryStats(cgroupPath, memory, memUsage); err != nil { | ||
| return memory, errors.Wrap(err, "unable to update with memory.stat info") | ||
| } | ||
| return memory, nil | ||
| } | ||
|
|
||
| // MemLimitGivenSystem limit returns the memory limit for a given cgroup | ||
| // If the configured memory limit is larger than the total memory on the sys, the | ||
| // physical system memory size is returned | ||
| func MemLimitGivenSystem(cgroupLimit uint64) uint64 { | ||
| si := &syscall.Sysinfo_t{} | ||
| err := syscall.Sysinfo(si) | ||
| if err != nil { | ||
| return cgroupLimit | ||
| } | ||
|
|
||
| // conversion to uint64 needed to build on 32-bit | ||
| // but lint complains about unnecessary conversion | ||
| // see: pr#2409 | ||
| physicalLimit := uint64(si.Totalram) //nolint:unconvert | ||
| if cgroupLimit > physicalLimit { | ||
| return physicalLimit | ||
| } | ||
| return cgroupLimit | ||
| } | ||
|
|
||
| // updateWithMemoryStats updates the ContainerStats object with info | ||
| // from cgroup's memory.stat. Returns an error if the file does not exists, | ||
| // or not parsable. | ||
| func updateWithMemoryStats(path string, memory *types.MemoryUsage, usage uint64) error { | ||
| const memoryStatFile = "memory.stat" | ||
| var memoryStatPath, inactiveFileSearchString string | ||
| if !node.CgroupIsV2() { | ||
| memoryStatPath = filepath.Join(cgroupMemoryPathV1, path, memoryStatFile) | ||
| inactiveFileSearchString = "total_inactive_file " | ||
| } else { | ||
| memoryStatPath = filepath.Join(cgroupMemoryPathV2, path, memoryStatFile) | ||
| inactiveFileSearchString = "inactive_file " | ||
| } | ||
| return UpdateWithMemoryStatsFromFile(memoryStatPath, inactiveFileSearchString, memory, usage) | ||
| } | ||
|
|
||
| func UpdateWithMemoryStatsFromFile(memoryStatPath, inactiveFileSearchString string, memory *types.MemoryUsage, usage uint64) error { | ||
| var totalInactive uint64 | ||
| f, err := os.Open(memoryStatPath) | ||
|
There was a problem hiding this comment. G304: Potential file inclusion via variable |
||
| if err != nil { | ||
| return err | ||
| } | ||
| defer f.Close() | ||
|
|
||
| toUpdate := []struct { | ||
| prefix string | ||
| field *uint64 | ||
| }{ | ||
| {inactiveFileSearchString, &totalInactive}, | ||
| {"rss ", &memory.RssBytes.Value}, | ||
| {"pgfault ", &memory.PageFaults.Value}, | ||
| {"pgmajfault ", &memory.MajorPageFaults.Value}, | ||
| } | ||
|
|
||
| scanner := bufio.NewScanner(f) | ||
| for scanner.Scan() { | ||
| for _, field := range toUpdate { | ||
| if !strings.HasPrefix(scanner.Text(), field.prefix) { | ||
| continue | ||
| } | ||
| val, err := strconv.ParseUint( | ||
| strings.TrimPrefix(scanner.Text(), field.prefix), 10, 64, | ||
| ) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "unable to parse %s", field.prefix) | ||
| } | ||
| *field.field = val | ||
| } | ||
| } | ||
| if err := scanner.Err(); err != nil { | ||
| return err | ||
| } | ||
| if usage > totalInactive { | ||
| memory.WorkingSetBytes.Value = usage - totalInactive | ||
| } else { | ||
| logrus.Warnf( | ||
| "Unable to account working set stats: total_inactive_file (%d) > memory usage (%d)", | ||
| totalInactive, usage, | ||
| ) | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func createProcessUsage(systemNano int64, cgroupStats *cgroups.Metrics) *types.ProcessUsage { | ||
| return &types.ProcessUsage{ | ||
| Timestamp: systemNano, | ||
| ProcessCount: &types.UInt64Value{Value: cgroupStats.Pids.Current}, | ||
| } | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.