Implement virtlet --diag

cmd/virtlet/virtlet.go

@@ -30,6 +30,7 @@ import (
  "github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
  "github.com/Mirantis/virtlet/pkg/cni"
  "github.com/Mirantis/virtlet/pkg/config"
+ "github.com/Mirantis/virtlet/pkg/diag"
  "github.com/Mirantis/virtlet/pkg/libvirttools"
  "github.com/Mirantis/virtlet/pkg/manager"
  "github.com/Mirantis/virtlet/pkg/nsfix"
@@ -41,10 +42,14 @@ import (
 const (
  wantTapManagerEnv = "WANT_TAP_MANAGER"
  nodeNameEnv = "KUBE_NODE_NAME"
+ diagSocket = "/run/virtlet-diag.sock"
+ netnsDiagCommand = `if [ -d /var/run/netns ]; then cd /var/run/netns; for ns in *; do echo "*** ${ns} ***"; ip netns exec "${ns}" ip a; ip netns exec "${ns}" ip r; echo; done; fi`
+ qemuLogDir = "/var/log/libvirt/qemu"
 )
 
 var (
  dumpConfig = flag.Bool("dump-config", false, "Dump node-specific Virtlet config as a shell script and exit")
+ dumpDiag = flag.Bool("diag", false, "Dump diagnostics as JSON and exit")
  displayVersion = flag.Bool("version", false, "Display version and exit")
  versionFormat = flag.String("version-format", "text", "Version format to use (text, short, json, yaml)")
 )
@@ -55,8 +60,8 @@ func configWithDefaults(cfg *v1.VirtletConfig) *v1.VirtletConfig {
  return r
 }
 
-func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig) {
- manager := manager.NewVirtletManager(config, nil, clientCfg)
+func runVirtlet(config *v1.VirtletConfig, clientCfg clientcmd.ClientConfig, diagSet *diag.DiagSet) {
+ manager := manager.NewVirtletManager(config, nil, clientCfg, diagSet)
  if err := manager.Run(); err != nil {
  glog.Errorf("Error: %v", err)
  os.Exit(1)
@@ -106,6 +111,30 @@ func setLogLevel(config *v1.VirtletConfig) {
  })
 }
 
+func runDiagServer() *diag.DiagSet {
+ diagSet := diag.NewDiagSet()
+ diagSet.RegisterDiagSource("ip-a", diag.NewCommandSource("txt", []string{"ip", "a"}))
+ diagSet.RegisterDiagSource("ip-r", diag.NewCommandSource("txt", []string{"ip", "r"}))
+ diagSet.RegisterDiagSource("netns", diag.NewCommandSource("txt", []string{"/bin/bash", "-c", netnsDiagCommand}))
+ diagSet.RegisterDiagSource("libvirt-logs", diag.NewLogDirSource(qemuLogDir))
+ diagSet.RegisterDiagSource("stack", diag.StackDumpSource)
+ server := diag.NewServer(diagSet)
+ go func() {
+ err := server.Serve(diagSocket, nil)
+ glog.V(1).Infof("Diag server returned: %v", err)
+ }()
+ return diagSet
+}
+
+func doDiag() {
+ dr, err := diag.RetrieveDiagnostics(diagSocket)
+ if err != nil {
+ glog.Errorf("Failed to retrieve diagnostics: %v", err)
+ os.Exit(1)
+ }
+ os.Stdout.Write(dr.ToJSON())
+}
+
 func main() {
  nsfix.HandleReexec()
  clientCfg := utils.BindFlags(flag.CommandLine)
@@ -131,9 +160,12 @@ func main() {
  glog.Errorf("Error writing config: %v", err)
  os.Exit(1)
  }
+ case *dumpDiag:
+ doDiag()
  default:
  localConfig = configWithDefaults(localConfig)
  go runTapManager(localConfig)
- runVirtlet(localConfig, clientCfg)
+ diagSet := runDiagServer()
+ runVirtlet(localConfig, clientCfg, diagSet)
  }
 }

deploy/data/virtlet-ds.yaml

@@ -198,6 +198,9 @@ spec:
  name: image-name-translations
  - name: pods-log
  mountPath: /var/log/pods
+ # needed for diagnostic purposes
+ - name: libvirt-log
+ mountPath: /var/log/libvirt
  securityContext:
  privileged: true
  readinessProbe:
@@ -275,9 +278,6 @@ spec:
  - hostPath:
  path: /var/log/pods
  name: pods-log
- - hostPath:
- path: /var/run/netns
- name: netns-dir
  - configMap:
  name: virtlet-image-translations
  name: image-name-translations

pkg/manager/manager.go

@@ -25,6 +25,7 @@ import (
  "k8s.io/client-go/tools/clientcmd"
 
  "github.com/Mirantis/virtlet/pkg/api/virtlet.k8s/v1"
+ "github.com/Mirantis/virtlet/pkg/diag"
  "github.com/Mirantis/virtlet/pkg/image"
  "github.com/Mirantis/virtlet/pkg/imagetranslation"
  "github.com/Mirantis/virtlet/pkg/libvirttools"
@@ -47,6 +48,7 @@ type VirtletManager struct {
  config *v1.VirtletConfig
  metadataStore metadata.Store
  fdManager tapmanager.FDManager
+ diagSet *diag.DiagSet
  clientCfg clientcmd.ClientConfig
  virtTool *libvirttools.VirtualizationTool
  imageStore image.Store
@@ -56,8 +58,8 @@ type VirtletManager struct {
 }
 
 // NewVirtletManager creates a new VirtletManager.
-func NewVirtletManager(config *v1.VirtletConfig, fdManager tapmanager.FDManager, clientCfg clientcmd.ClientConfig) *VirtletManager {
- return &VirtletManager{config: config, fdManager: fdManager}
+func NewVirtletManager(config *v1.VirtletConfig, fdManager tapmanager.FDManager, clientCfg clientcmd.ClientConfig, diagSet *diag.DiagSet) *VirtletManager {
+ return &VirtletManager{config: config, fdManager: fdManager, diagSet: diagSet}
 }
 
 // Run sets up the environment for the runtime and image services and
@@ -83,6 +85,7 @@ func (v *VirtletManager) Run() error {
  if err != nil {
  return fmt.Errorf("failed to create metadata store: %v", err)
  }
+ v.diagSet.RegisterDiagSource("metadata", metadata.GetMetadataDumpSource(v.metadataStore))
 
  downloader := image.NewDownloader(*v.config.DownloadProtocol)
  v.imageStore = image.NewFileStore(*v.config.ImageDir, downloader, nil)
@@ -99,6 +102,7 @@ func (v *VirtletManager) Run() error {
  if err != nil {
  return fmt.Errorf("error establishing libvirt connection: %v", err)
  }
+ v.diagSet.RegisterDiagSource("libvirt-xml", libvirttools.NewLibvirtDiagSource(conn, conn))
 
  virtConfig := libvirttools.VirtualizationConfig{
  DisableKVM: *v.config.DisableKVM,

pkg/tools/TestGenCommand__compat.out.yaml

@@ -100,6 +100,8 @@ spec:
  name: image-name-translations
  - mountPath: /var/log/pods
  name: pods-log
+ - mountPath: /var/log/libvirt
+ name: libvirt-log
  - command:
  - /vms.sh
  image: mirantis/virtlet
@@ -256,9 +258,6 @@ spec:
  - hostPath:
  path: /var/log/pods
  name: pods-log
- - hostPath:
- path: /var/run/netns
- name: netns-dir
  - configMap:
  name: virtlet-image-translations
  name: image-name-translations

pkg/tools/TestGenCommand__compat_dev.out.yaml

@@ -102,6 +102,8 @@ spec:
  name: image-name-translations
  - mountPath: /var/log/pods
  name: pods-log
+ - mountPath: /var/log/libvirt
+ name: libvirt-log
  - mountPath: /dind
  name: dind
  - command:
@@ -264,9 +266,6 @@ spec:
  - hostPath:
  path: /var/log/pods
  name: pods-log
- - hostPath:
- path: /var/run/netns
- name: netns-dir
  - configMap:
  name: virtlet-image-translations
  name: image-name-translations

pkg/tools/TestGenCommand__dev.out.yaml

@@ -105,6 +105,8 @@ spec:
  name: image-name-translations
  - mountPath: /var/log/pods
  name: pods-log
+ - mountPath: /var/log/libvirt
+ name: libvirt-log
  - mountPath: /dind
  name: dind
  - command:
@@ -268,9 +270,6 @@ spec:
  - hostPath:
  path: /var/log/pods
  name: pods-log
- - hostPath:
- path: /var/run/netns
- name: netns-dir
  - configMap:
  name: virtlet-image-translations
  name: image-name-translations

pkg/tools/TestGenCommand__plain.out.yaml

@@ -103,6 +103,8 @@ spec:
  name: image-name-translations
  - mountPath: /var/log/pods
  name: pods-log
+ - mountPath: /var/log/libvirt
+ name: libvirt-log
  - command:
  - /vms.sh
  image: mirantis/virtlet
@@ -260,9 +262,6 @@ spec:
  - hostPath:
  path: /var/log/pods
  name: pods-log
- - hostPath:
- path: /var/run/netns
- name: netns-dir
  - configMap:
  name: virtlet-image-translations
  name: image-name-translations

pkg/tools/TestGenCommand__tag.out.yaml

@@ -103,6 +103,8 @@ spec:
  name: image-name-translations
  - mountPath: /var/log/pods
  name: pods-log
+ - mountPath: /var/log/libvirt
+ name: libvirt-log
  - command:
  - /vms.sh
  image: mirantis/virtlet:0.9.42
@@ -260,9 +262,6 @@ spec:
  - hostPath:
  path: /var/log/pods
  name: pods-log
- - hostPath:
- path: /var/run/netns
- name: netns-dir
  - configMap:
  name: virtlet-image-translations
  name: image-name-translations