Skip to content

Commit

Permalink
Merge pull request #6635 from medyagh/kic_volume
Browse files Browse the repository at this point in the history
improve delete by labeling created docker volumes
  • Loading branch information
medyagh authored Feb 18, 2020
2 parents 61bc2e9 + 1da0a37 commit 52c200a
Show file tree
Hide file tree
Showing 5 changed files with 86 additions and 17 deletions.
15 changes: 11 additions & 4 deletions cmd/minikube/cmd/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
"k8s.io/minikube/pkg/drivers/kic/oci"
"k8s.io/minikube/pkg/minikube/cluster"
"k8s.io/minikube/pkg/minikube/config"
pkg_config "k8s.io/minikube/pkg/minikube/config"
Expand Down Expand Up @@ -96,9 +97,10 @@ func runDelete(cmd *cobra.Command, args []string) {
profileFlag := viper.GetString(config.MachineProfile)

validProfiles, invalidProfiles, err := pkg_config.ListProfiles()
glog.Warningf("Couldn't find any profiles in minikube home %q: %v", localpath.MiniPath(), err)
profilesToDelete := append(validProfiles, invalidProfiles...)

// If the purge flag is set, go ahead and delete the .minikube directory.
// in the case user has more than 1 profile and runs --purge
// to prevent abandoned VMs/containers, force user to run with delete --all
if purge && len(profilesToDelete) > 1 && !deleteAll {
out.ErrT(out.Notice, "Multiple minikube profiles were found - ")
for _, p := range profilesToDelete {
Expand All @@ -112,8 +114,9 @@ func runDelete(cmd *cobra.Command, args []string) {
exit.UsageT("usage: minikube delete --all")
}

if err != nil {
exit.WithError("Error getting profiles to delete", err)
err := oci.DeleteAllVolumesByLabel(oci.Docker, fmt.Sprintf("%s=%s", oci.CreatedByLabelKey, "=true"))
if err != nil { // if there is no volume there won't be any error
glog.Warningf("error deleting left docker volumes. \n%v:\nfor more information please refer to docker documentation: https://docs.docker.com/engine/reference/commandline/volume_prune/", err)
}

errs := DeleteProfiles(profilesToDelete)
Expand Down Expand Up @@ -177,6 +180,10 @@ func DeleteProfiles(profiles []*pkg_config.Profile) []error {

func deleteProfile(profile *pkg_config.Profile) error {
viper.Set(pkg_config.MachineProfile, profile.Name)
err := oci.DeleteAllVolumesByLabel(oci.Docker, fmt.Sprintf("%s=%s", oci.ProfileLabelKey, profile.Name))
if err != nil { // if there is no volume there wont be any error
glog.Warningf("error deleting left docker volumes. To see the list of volumes run: 'docker volume ls' \n:%v", err)
}

api, err := machine.NewAPIClient()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion pkg/drivers/kic/kic.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (d *Driver) Create() error {
params := oci.CreateParams{
Name: d.NodeConfig.MachineName,
Image: d.NodeConfig.ImageDigest,
ClusterLabel: oci.ClusterLabelKey + "=" + d.MachineName,
ClusterLabel: oci.ProfileLabelKey + "=" + d.MachineName,
CPUs: strconv.Itoa(d.NodeConfig.CPU),
Memory: strconv.Itoa(d.NodeConfig.Memory) + "mb",
Envs: d.NodeConfig.Envs,
Expand Down
22 changes: 13 additions & 9 deletions pkg/drivers/kic/oci/oci.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,24 +57,28 @@ func CreateContainerNode(p CreateParams) error {
"-v", "/lib/modules:/lib/modules:ro",
"--hostname", p.Name, // make hostname match container name
"--name", p.Name, // ... and set the container name
"--label", fmt.Sprintf("%s=%s", CreatedByLabelKey, "true"),
// label the node with the cluster ID
"--label", p.ClusterLabel,
// label the node with the role ID
"--label", fmt.Sprintf("%s=%s", nodeRoleKey, p.Role),
}

// volume path in minikube home folder to mount to /var
hostVarVolPath := filepath.Join(localpath.MiniPath(), "machines", p.Name, "var")
if err := os.MkdirAll(hostVarVolPath, 0711); err != nil {
return errors.Wrapf(err, "create var dir %s", hostVarVolPath)
"--label", fmt.Sprintf("%s=%s", nodeRoleLabelKey, p.Role),
}

if p.OCIBinary == Podman { // enable execing in /var
// volume path in minikube home folder to mount to /var
hostVarVolPath := filepath.Join(localpath.MiniPath(), "machines", p.Name, "var")
if err := os.MkdirAll(hostVarVolPath, 0711); err != nil {
return errors.Wrapf(err, "create var dir %s", hostVarVolPath)
}
// podman mounts var/lib with no-exec by default https://github.com/containers/libpod/issues/5103
runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var:exec", hostVarVolPath))
}
if p.OCIBinary == Docker {
runArgs = append(runArgs, "--volume", "/var")
if err := createDockerVolume(p.Name); err != nil {
return errors.Wrapf(err, "creating volume for %s container", p.Name)
}
glog.Infof("Successfully created a docker volume %s", p.Name)
runArgs = append(runArgs, "--volume", fmt.Sprintf("%s:/var", p.Name))
// setting resource limit in privileged mode is only supported by docker
// podman error: "Error: invalid configuration, cannot set resources with rootless containers not using cgroups v2 unified mode"
runArgs = append(runArgs, fmt.Sprintf("--cpus=%s", p.CPUs), fmt.Sprintf("--memory=%s", p.Memory))
Expand Down Expand Up @@ -264,7 +268,7 @@ func ContainerID(ociBinary string, nameOrID string) (string, error) {

// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
func ListOwnedContainers(ociBinary string) ([]string, error) {
return listContainersByLabel(ociBinary, ClusterLabelKey)
return listContainersByLabel(ociBinary, ProfileLabelKey)
}

// inspect return low-level information on containers
Expand Down
9 changes: 6 additions & 3 deletions pkg/drivers/kic/oci/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,15 @@ const (
DefaultBindIPV4 = "127.0.0.1"
Docker = "docker"
Podman = "podman"
// ClusterLabelKey is applied to each node docker container for identification
ClusterLabelKey = "io.x-k8s.kic.cluster"
// ProfileLabelKey is applied to any container or volume created by a specific minikube profile name.minikube.sigs.k8s.io=PROFILE_NAME
ProfileLabelKey = "name.minikube.sigs.k8s.io"
// NodeRoleKey is used to identify if it is control plane or worker
nodeRoleKey = "io.k8s.sigs.kic.role"
nodeRoleLabelKey = "role.minikube.sigs.k8s.io"
// CreatedByLabelKey is applied to any container/volume that is created by minikube created_by.minikube.sigs.k8s.io=true
CreatedByLabelKey = "created_by.minikube.sigs.k8s.io"
)

// CreateParams are parameters needed to create a container
type CreateParams struct {
Name string // used for container name and hostname
Image string // container image to use to create the node.
Expand Down
55 changes: 55 additions & 0 deletions pkg/drivers/kic/oci/volumes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
Copyright 2020 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package oci

import (
"os/exec"

"github.com/golang/glog"
"github.com/pkg/errors"
)

// DeleteAllVolumesByLabel deletes all volumes that have a specific label
// if there is no volume to delete it will return nil
// example: docker volume prune -f --filter label=name.minikube.sigs.k8s.io=minikube
func DeleteAllVolumesByLabel(ociBin string, label string) error {
glog.Infof("trying to prune all %s volumes with label %s", ociBin, label)
if ociBin == Docker {
if err := PointToHostDockerDaemon(); err != nil {
return errors.Wrap(err, "point host docker-daemon")
}
}
cmd := exec.Command(ociBin, "volume", "prune", "-f", "--filter", label)
if out, err := cmd.CombinedOutput(); err != nil {
return errors.Wrapf(err, "output %s", string(out))
}
return nil
}

// createDockerVolume creates a docker volume to be attached to the container with correct labels and prefixes based on profile name
// Caution ! if volume already exists does NOT return an error and will not apply the minikube labels on it.
// TODO: this should be fixed as a part of https://github.com/kubernetes/minikube/issues/6530
func createDockerVolume(name string) error {
if err := PointToHostDockerDaemon(); err != nil {
return errors.Wrap(err, "point host docker-daemon")
}
cmd := exec.Command(Docker, "volume", "create", name, "--label", "name.minikube.sigs.k8s.io="+name, "--label", "craeted_by_minikube.minikube.sigs.k8s.io=true")
if out, err := cmd.CombinedOutput(); err != nil {
return errors.Wrapf(err, "output %s", string(out))
}
return nil
}

0 comments on commit 52c200a

Please sign in to comment.