From cfb2d722f86a1d600de762982c00a213cbcff6b2 Mon Sep 17 00:00:00 2001 From: Piotr Skamruk Date: Tue, 27 Feb 2018 19:47:06 +0100 Subject: [PATCH 1/3] Provide initial part of virtletctl with WIP ver of dump-metadata --- build/cmd.sh | 1 + cmd/virtletctl/virtletctl.go | 44 ++++++++++++ pkg/tools/common.go | 126 +++++++++++++++++++++++++++++++++++ pkg/tools/dumpmetadata.go | 93 ++++++++++++++++++++++++++ pkg/tools/subcommands.go | 117 ++++++++++++++++++++++++++++++++ 5 files changed, 381 insertions(+) create mode 100644 cmd/virtletctl/virtletctl.go create mode 100644 pkg/tools/common.go create mode 100644 pkg/tools/dumpmetadata.go create mode 100644 pkg/tools/subcommands.go diff --git a/build/cmd.sh b/build/cmd.sh index 2969970ff..10f1967ff 100755 --- a/build/cmd.sh +++ b/build/cmd.sh @@ -375,6 +375,7 @@ function build_internal { install_vendor_internal mkdir -p "${project_dir}/_output" go build -i -o "${project_dir}/_output/virtlet" ./cmd/virtlet + go build -i -o "${project_dir}/_output/virtletctl" ./cmd/virtletctl go build -i -o "${project_dir}/_output/vmwrapper" ./cmd/vmwrapper go build -i -o "${project_dir}/_output/flexvolume_driver" ./cmd/flexvolume_driver go test -i -c -o "${project_dir}/_output/virtlet-e2e-tests" ./tests/e2e diff --git a/cmd/virtletctl/virtletctl.go b/cmd/virtletctl/virtletctl.go new file mode 100644 index 000000000..f81b0aec8 --- /dev/null +++ b/cmd/virtletctl/virtletctl.go @@ -0,0 +1,44 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/Mirantis/virtlet/pkg/tools" +) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "Error: missing subcommand name") + os.Exit(1) + } + + if err := tools.ParseFlags(os.Args[1]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + args := flag.Args() + + if err := tools.RunSubcommand(args[0], args[1:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/pkg/tools/common.go b/pkg/tools/common.go new file mode 100644 index 000000000..3d3e7fe60 --- /dev/null +++ b/pkg/tools/common.go @@ -0,0 +1,126 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tools + +import ( + "io" + + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand" + // "k8s.io/client-go/kubernetes/scheme" + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" + v1 "k8s.io/client-go/pkg/api/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/client-go/util/exec" + //"k8s.io/kubernetes/pkg/api" +) + +// SubCommandCommon contains attributes and methods useful for all subcommands. +type SubCommandCommon struct { + client *typedv1.CoreV1Client + config *rest.Config +} + +// Setup prepares values for common between subcommands attributes +func (s *SubCommandCommon) Setup(client *typedv1.CoreV1Client, config *rest.Config) { + s.client = client + s.config = config +} + +// GetVirtletPods returns a list of virtlet pod +func (s *SubCommandCommon) GetVirtletPods() ([]v1.Pod, error) { + pods, err := s.client.Pods("kube-system").List(meta_v1.ListOptions{ + LabelSelector: "runtime=virtlet", + }) + if err != nil { + return nil, err + } + + return pods.Items, nil +} + +// ExecCommandOnContainer given a pod, container, namespace and command +// executes that command remotely returning stdout and stderr output +// as strings and error if any occured. +// If there is provided bytes.Buffer as stdin - it's content will be passed +// to remote command. +// Command is executed without a TTY as stdin. +func (s *SubCommandCommon) ExecCommandOnContainer( + pod, container, namespace string, + stdin io.Reader, stdout, stderr io.Writer, + command ...string, +) (int, error) { + + req := s.client.RESTClient().Post(). + Resource("pods"). + Name(pod). + Namespace(namespace). + SubResource("exec"). + Param("container", container) + + if stdin != nil { + req.Param("stdin", "true") + } + if stdout != nil { + req.Param("stdout", "true") + } + if stderr != nil { + req.Param("stderr", "true") + } + for _, cmd := range command { + req.Param("command", cmd) + } + + // Above replaces different below attempts which are producing incorrect + // urls + + // req.VersionedParams(&scheme.PodExecOptions{ + // req.VersionedParams(&v1.PodExecOptions{ + // req.VersionedParams(&api.PodExecOptions{ + // Container: container, + // Command: command, + // Stdin: stdin != nil, + // Stdout: stdout != nil, + // Stderr: stderr != nil, + // TTY: false, + // }, api.ParameterCodec) + // }, scheme.ParameterCodec) + // fmt.Printf("Constructed url: %s\n", req.URL()) + + executor, err := remotecommand.NewExecutor(s.config, "POST", req.URL()) + if err != nil { + return 0, err + } + + exitCode := 0 + err = executor.Stream(remotecommand.StreamOptions{ + SupportedProtocols: remotecommandconsts.SupportedStreamingProtocols, + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + }) + + if err != nil { + if c, ok := err.(exec.CodeExitError); ok { + exitCode = c.Code + err = nil + } + } + + return exitCode, err +} diff --git a/pkg/tools/dumpmetadata.go b/pkg/tools/dumpmetadata.go new file mode 100644 index 000000000..6fec54dd8 --- /dev/null +++ b/pkg/tools/dumpmetadata.go @@ -0,0 +1,93 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tools + +import ( + "bytes" + "fmt" + "io/ioutil" + + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" +) + +const ( + // TODO: pass that as command line arg and use make default as constant + // used there and in cmd/virtlet/virtlet.go + virtletDBPath = "/var/lib/virtlet/virtlet.db" +) + +// DumpMetadata contains data needed by dump-metedata subcommand. +type DumpMetadata struct { + SubCommandCommon +} + +var _ SubCommand = &DumpMetadata{} + +// RegisterFlags implements RegisterFlags method of SubCommand interface. +func (d DumpMetadata) RegisterFlags() { +} + +// Run implements Run method of SubCommand interface. +func (d DumpMetadata) Run(clientset *typedv1.CoreV1Client, config *rest.Config, args []string) error { + d.Setup(clientset, config) + + pods, err := d.GetVirtletPods() + if err != nil { + return err + } + + if len(pods) == 0 { + return fmt.Errorf("Not found any Virtlet pod") + } + + for _, pod := range pods { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + stdin := bytes.NewBufferString("") + + fmt.Printf("pod: %s\n", pod.Name) + exitCode, err := d.ExecCommandOnContainer( + pod.Name, + "virtlet", + "kube-system", + stdin, + stdout, + stderr, + "/bin/cat", + virtletDBPath, + ) + if err != nil { + fmt.Printf(" Error during downloading virtled metadata database: %v\n", err) + } + if exitCode != 0 { + fmt.Printf(" Got different than expected exit code: %d\n", exitCode) + fmt.Printf(" Remote command error output: %s\n", stderr.String()) + continue + } + f, err := ioutil.TempFile("/tmp", "virtlet-") + defer f.Close() + if err != nil { + fmt.Printf(" Got error during opening tempfile: %v\n", err) + continue + } + f.Write(stdout.Bytes()) + fmt.Printf(" Virtlet metadata database for this pod saved in %q location\n", f.Name()) + } + + return nil +} diff --git a/pkg/tools/subcommands.go b/pkg/tools/subcommands.go new file mode 100644 index 000000000..d2c0121fe --- /dev/null +++ b/pkg/tools/subcommands.go @@ -0,0 +1,117 @@ +/* +Copyright 2018 Mirantis + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tools + +import ( + "flag" + "fmt" + "os/user" + "path/filepath" + "strings" + + typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +var ( + kubeconfig string + master string + + // map of all subcommands + subcommands = map[string]SubCommand{ + "dump-metadata": DumpMetadata{}, + } +) + +// SubCommand interface defines an interface for all subcommands. +type SubCommand interface { + // RegisterFlags registers flags for subcommand + RegisterFlags() + // Run is main entry point to subcommand + Run(clientset *typedv1.CoreV1Client, config *rest.Config, args []string) error +} + +// ParseFlags registers additional flags for particular subcommand +// and then parse them with common one defined as vars in this package. +// It returns an error containing message with available commands if +// requested command was not recognized. +func ParseFlags(command string) error { + flag.StringVar(&kubeconfig, "kubeconfig", "~/.kube/config", "absolute path to the kubeconfig file") + flag.StringVar(&master, "master", "http://127.0.0.1:8080", "master url") + + if subcommand, err := getSubcommand(command); err != nil { + return err + } else { + subcommand.RegisterFlags() + } + + flag.Parse() + + return nil +} + +// RunSubcommand creates kubernetes api client, passes it with args +// into subcommand and returns it's error if any. +// It returns an error if it can not create client or an error containing +// message with available commands if requested command was not recognized. +func RunSubcommand(command string, args []string) error { + var subcommand SubCommand + var err error + if subcommand, err = getSubcommand(command); err != nil { + return err + } + + if kubeconfig[:2] == "~/" { + usr, err := user.Current() + if err != nil { + return err + } + kubeconfig = filepath.Join(usr.HomeDir, kubeconfig[2:]) + } + + config, err := clientcmd.BuildConfigFromFlags(master, kubeconfig) + if err != nil { + return fmt.Errorf("Can't create kubernetes api client config: %v", err) + } + + client, err := typedv1.NewForConfig(config) + if err != nil { + return fmt.Errorf("Can't create kubernetes api client: %v", err) + } + + return subcommand.Run(client, config, args) +} + +func getSubcommand(name string) (SubCommand, error) { + if subcommand, ok := subcommands[name]; ok { + return subcommand, nil + } + + commands := make([]string, len(subcommands)) + i := 0 + for cmd := range subcommands { + commands[i] = cmd + i++ + } + + return nil, fmt.Errorf( + "Subcommand %q unrecognized. Available commands: %s", + name, + strings.Join(commands, ", "), + ) +} From 5680d57daa3621cda047fbb1113b53ac95e3ebfa Mon Sep 17 00:00:00 2001 From: Piotr Skamruk Date: Wed, 28 Feb 2018 15:30:52 +0100 Subject: [PATCH 2/3] Implementation of data dump for metadata --- pkg/tools/common.go | 49 +++++-------- pkg/tools/dumpmetadata.go | 144 ++++++++++++++++++++++++++++++-------- 2 files changed, 132 insertions(+), 61 deletions(-) diff --git a/pkg/tools/common.go b/pkg/tools/common.go index 3d3e7fe60..06adde481 100644 --- a/pkg/tools/common.go +++ b/pkg/tools/common.go @@ -21,13 +21,15 @@ import ( meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand" - // "k8s.io/client-go/kubernetes/scheme" typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" v1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" "k8s.io/client-go/util/exec" - //"k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api" + + // register standard k8s types + _ "k8s.io/kubernetes/pkg/api/install" ) // SubCommandCommon contains attributes and methods useful for all subcommands. @@ -57,10 +59,11 @@ func (s *SubCommandCommon) GetVirtletPods() ([]v1.Pod, error) { // ExecCommandOnContainer given a pod, container, namespace and command // executes that command remotely returning stdout and stderr output // as strings and error if any occured. -// If there is provided bytes.Buffer as stdin - it's content will be passed -// to remote command. +// If there is provided io.Reader as stdin - it's content will be passed +// to remote command. Similarly if there will be provided instances of io.Writer +// as stdout/stderr - they also will be passed. // Command is executed without a TTY as stdin. -func (s *SubCommandCommon) ExecCommandOnContainer( +func (s *SubCommandCommon) ExecInContainer( pod, container, namespace string, stdin io.Reader, stdout, stderr io.Writer, command ...string, @@ -73,34 +76,14 @@ func (s *SubCommandCommon) ExecCommandOnContainer( SubResource("exec"). Param("container", container) - if stdin != nil { - req.Param("stdin", "true") - } - if stdout != nil { - req.Param("stdout", "true") - } - if stderr != nil { - req.Param("stderr", "true") - } - for _, cmd := range command { - req.Param("command", cmd) - } - - // Above replaces different below attempts which are producing incorrect - // urls - - // req.VersionedParams(&scheme.PodExecOptions{ - // req.VersionedParams(&v1.PodExecOptions{ - // req.VersionedParams(&api.PodExecOptions{ - // Container: container, - // Command: command, - // Stdin: stdin != nil, - // Stdout: stdout != nil, - // Stderr: stderr != nil, - // TTY: false, - // }, api.ParameterCodec) - // }, scheme.ParameterCodec) - // fmt.Printf("Constructed url: %s\n", req.URL()) + req.VersionedParams(&api.PodExecOptions{ + Container: container, + Command: command, + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + TTY: false, + }, api.ParameterCodec) executor, err := remotecommand.NewExecutor(s.config, "POST", req.URL()) if err != nil { diff --git a/pkg/tools/dumpmetadata.go b/pkg/tools/dumpmetadata.go index 6fec54dd8..a7381b564 100644 --- a/pkg/tools/dumpmetadata.go +++ b/pkg/tools/dumpmetadata.go @@ -20,15 +20,22 @@ import ( "bytes" "fmt" "io/ioutil" + "os" + "strings" + "github.com/davecgh/go-spew/spew" typedv1 "k8s.io/client-go/kubernetes/typed/core/v1" + v1 "k8s.io/client-go/pkg/api/v1" "k8s.io/client-go/rest" + + "github.com/Mirantis/virtlet/pkg/metadata" ) const ( // TODO: pass that as command line arg and use make default as constant // used there and in cmd/virtlet/virtlet.go - virtletDBPath = "/var/lib/virtlet/virtlet.db" + virtletDBPath = "/var/lib/virtlet/virtlet.db" + defaultIndentString = " " ) // DumpMetadata contains data needed by dump-metedata subcommand. @@ -56,38 +63,119 @@ func (d DumpMetadata) Run(clientset *typedv1.CoreV1Client, config *rest.Config, } for _, pod := range pods { - stdout := &bytes.Buffer{} - stderr := &bytes.Buffer{} - stdin := bytes.NewBufferString("") - - fmt.Printf("pod: %s\n", pod.Name) - exitCode, err := d.ExecCommandOnContainer( - pod.Name, - "virtlet", - "kube-system", - stdin, - stdout, - stderr, - "/bin/cat", - virtletDBPath, - ) + fname, err := d.copyOutFile(&pod) if err != nil { - fmt.Printf(" Error during downloading virtled metadata database: %v\n", err) - } - if exitCode != 0 { - fmt.Printf(" Got different than expected exit code: %d\n", exitCode) - fmt.Printf(" Remote command error output: %s\n", stderr.String()) + fmt.Fprintf(os.Stderr, "Can't extract metadata db for pod %q: %v\n", pod.Name, err) continue } - f, err := ioutil.TempFile("/tmp", "virtlet-") - defer f.Close() - if err != nil { - fmt.Printf(" Got error during opening tempfile: %v\n", err) - continue + defer os.Remove(fname) + if err := dumpMetadata(fname); err != nil { + fmt.Fprintf(os.Stderr, "Can't dump metadata for pod %q: %v\n", pod.Name, err) } - f.Write(stdout.Bytes()) - fmt.Printf(" Virtlet metadata database for this pod saved in %q location\n", f.Name()) } return nil } + +func (d *DumpMetadata) copyOutFile(pod *v1.Pod) (string, error) { + fmt.Printf("Virtlet pod name: %s\n", pod.Name) + + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + stdin := bytes.NewBufferString("") + + exitCode, err := d.ExecInContainer( + pod.Name, + "virtlet", + "kube-system", + stdin, + stdout, + stderr, + "/bin/cat", + virtletDBPath, + ) + if err != nil { + return "", err + } + + if exitCode != 0 { + return "", fmt.Errorf("remote command exit code was %d and its stderr output was:\n%s", exitCode, stderr.String()) + } + + f, err := ioutil.TempFile("/tmp", "virtlet-") + defer f.Close() + if err != nil { + return "", fmt.Errorf("error during opening tempfile: %v\n", err) + } + f.Write(stdout.Bytes()) + + return f.Name(), nil +} + +func dumpMetadata(fname string) error { + s, err := metadata.NewMetadataStore(fname) + if err != nil { + return fmt.Errorf("can't open metadata db: %v", err) + } + + printlnIndented(1, "Sandboxes:") + sandboxes, err := s.ListPodSandboxes(nil) + if err != nil { + return fmt.Errorf("can't list sandboxes: %v", err) + } + + for _, smeta := range sandboxes { + if sinfo, err := smeta.Retrieve(); err != nil { + return fmt.Errorf("can't retrieve sandbox: %v", err) + } else if err := dumpSandbox(smeta.GetID(), sinfo, s); err != nil { + return fmt.Errorf("can't dump sandbox: %v", err) + } + } + + printlnIndented(1, "Images:") + images, err := s.ImagesInUse() + if err != nil { + return fmt.Errorf("can't dump images: %v", err) + } + + for image := range images { + printlnIndented(2, image) + } + + return nil +} + +func dumpSandbox(podid string, sandbox *metadata.PodSandboxInfo, s metadata.MetadataStore) error { + printlnIndented(2, "Sandbox id: %s", podid) + printlnIndented(0, spew.Sdump(sandbox)) + + printlnIndented(3, "Containers:") + containers, err := s.ListPodContainers(podid) + if err != nil { + return fmt.Errorf("can't retrieve list of containers: %v", err) + } + + for _, cmeta := range containers { + printIndented(4, "Container id: %s", cmeta.GetID()) + if cinfo, err := cmeta.Retrieve(); err != nil { + return fmt.Errorf("can't retrieve container metadata: %v", err) + } else { + printIndented(0, spew.Sdump(cinfo)) + } + } + + return nil +} + +func printlnIndented(level int, format string, data ...interface{}) { + printIndented(level, format+"\n", data...) +} + +func printIndented(level int, format string, data ...interface{}) { + printIndentedWith(defaultIndentString, 2*level, format, data...) +} + +func printIndentedWith(with string, length int, format string, data ...interface{}) { + indentString := strings.Repeat(with, (length/len(with))+1)[:length] + fmt.Printf("%s%s", indentString, fmt.Sprintf(format, data...)) +} From 9825512f0b2676a87620744a96abcfc131172a12 Mon Sep 17 00:00:00 2001 From: Piotr Skamruk Date: Thu, 1 Mar 2018 14:00:28 +0100 Subject: [PATCH 3/3] Use empty string to give a chance to use value from kubeconfig --- pkg/tools/subcommands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/tools/subcommands.go b/pkg/tools/subcommands.go index d2c0121fe..42f607c1c 100644 --- a/pkg/tools/subcommands.go +++ b/pkg/tools/subcommands.go @@ -52,7 +52,7 @@ type SubCommand interface { // requested command was not recognized. func ParseFlags(command string) error { flag.StringVar(&kubeconfig, "kubeconfig", "~/.kube/config", "absolute path to the kubeconfig file") - flag.StringVar(&master, "master", "http://127.0.0.1:8080", "master url") + flag.StringVar(&master, "master", "", "master url") if subcommand, err := getSubcommand(command); err != nil { return err