Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor file-metricscollector #1137

Merged
merged 1 commit into from
Apr 13, 2020
Merged

Refactor file-metricscollector #1137

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 13 additions & 16 deletions cmd/metricscollector/v1alpha3/file-metricscollector/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,14 +53,16 @@ import (
filemc "github.com/kubeflow/katib/pkg/metricscollector/v1alpha3/file-metricscollector"
)

var metricsFileName = flag.String("path", "", "Metrics File Path")
var trialName = flag.String("t", "", "Trial Name")
var managerService = flag.String("s", "", "Katib Manager service")
var metricNames = flag.String("m", "", "Metric names")
var filters = flag.String("f", "", "Metric filters")
var pollInterval = flag.Duration("p", common.DefaultPollInterval, "Poll interval to check if main process of worker container exit")
var timeout = flag.Duration("timeout", common.DefaultTimeout, "Timeout to check if main process of worker container exit")
var waitAll = flag.Bool("w", common.DefaultWaitAll, "Whether wait for all other main process of container exiting")
var (
metricsFileName = flag.String("path", "", "Metrics File Path")
trialName = flag.String("t", "", "Trial Name")
managerService = flag.String("s", "", "Katib Manager service")
metricNames = flag.String("m", "", "Metric names")
filters = flag.String("f", "", "Metric filters")
pollInterval = flag.Duration("p", common.DefaultPollInterval, "Poll interval to check if main process of worker container exit")
timeout = flag.Duration("timeout", common.DefaultTimeout, "Timeout to check if main process of worker container exit")
waitAll = flag.Bool("w", common.DefaultWaitAll, "Whether wait for all other main process of container exiting")
)

func printMetricsFile(mFile string) {
for {
Expand Down Expand Up @@ -101,20 +103,16 @@ func main() {
}
defer conn.Close()
c := api.NewManagerClient(conn)
mc, err := filemc.NewFileMetricsCollector()
if err != nil {
klog.Fatalf("Failed to create MetricsCollector: %v", err)
}
ctx := context.Background()
metricList := []string{}
var metricList []string
if len(*metricNames) != 0 {
metricList = strings.Split(*metricNames, ";")
}
filterList := []string{}
var filterList []string
if len(*filters) != 0 {
filterList = strings.Split(*filters, ";")
}
olog, err := mc.CollectObservationLog(*metricsFileName, metricList, filterList)
olog, err := filemc.CollectObservationLog(*metricsFileName, metricList, filterList)
if err != nil {
klog.Fatalf("Failed to collect logs: %v", err)
}
Expand All @@ -127,5 +125,4 @@ func main() {
klog.Fatalf("Failed to Report logs: %v", err)
}
klog.Infof("Metrics reported. :\n%v", olog)
return
}
21 changes: 5 additions & 16 deletions pkg/metricscollector/v1alpha3/file-metricscollector/file-metricscollector.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,30 +12,22 @@ import (
"k8s.io/klog"
)

type FileMetricsCollector struct {
}

func NewFileMetricsCollector() (*FileMetricsCollector, error) {
return &FileMetricsCollector{}, nil
}

func (d *FileMetricsCollector) CollectObservationLog(fileName string, metrics []string, filters []string) (*v1alpha3.ObservationLog, error) {
func CollectObservationLog(fileName string, metrics []string, filters []string) (*v1alpha3.ObservationLog, error) {
file, err := os.Open(fileName)
if err != nil {
return nil, err
}
defer file.Close()
content, err := ioutil.ReadAll(file)
logs := string(content)
olog, err := d.parseLogs(strings.Split(logs, "\n"), metrics, filters)
olog, err := parseLogs(strings.Split(logs, "\n"), metrics, filters)
return olog, err
}

func (d *FileMetricsCollector) parseLogs(logs []string, metrics []string, filters []string) (*v1alpha3.ObservationLog, error) {
var lasterr error
func parseLogs(logs []string, metrics []string, filters []string) (*v1alpha3.ObservationLog, error) {
olog := &v1alpha3.ObservationLog{}
mlogs := []*v1alpha3.MetricLog{}
metricRegList := getFilterRegexpList(filters)
mlogs := make([]*v1alpha3.MetricLog, 0, len(logs))

for _, logline := range logs {
// skip line which doesn't contain any metrics keywords, avoiding unnecessary pattern match
Expand Down Expand Up @@ -87,14 +79,11 @@ func (d *FileMetricsCollector) parseLogs(logs []string, metrics []string, filter
}
}
olog.MetricLogs = mlogs
if lasterr != nil {
return olog, lasterr
}
Comment on lines -90 to -92
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This variable is always nil because it's unused.

return olog, nil
}

func getFilterRegexpList(filters []string) []*regexp.Regexp {
regexpList := []*regexp.Regexp{}
regexpList := make([]*regexp.Regexp, 0, len(filters))
if len(filters) == 0 {
filters = append(filters, common.DefaultFilter)
}
Expand Down