diff --git a/docs/usage.md b/docs/usage.md index 93b8049a2..40a220b61 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -64,14 +64,17 @@ Now the `weaveworks/ignite-ubuntu` image is imported and ready for VM use. ### Configuring image registries -Ignite's runtime configuration for image registry uses the docker client -configuration. To add a new registry to docker client configuration, run +Ignite's runtime configuration for image registry uses the docker registry +configuration. To add a new registry to docker registry configuration, run `docker login `. This will create `$HOME/.docker/config.json` in the user's home directory. When ignite runs, it'll check the user's home -directory for docker client configuration file, load the registry configuration +directory for docker registry configuration file, load the registry configuration if found and use it. -An example of a docker client configuration file: +!!! Note + On many systems, running `sudo ignite` will set the `$HOME` directory to `/root`. + +An example of a docker registry configuration file: ```json @@ -80,6 +83,9 @@ An example of a docker client configuration file: "https://index.docker.io/v1/": { "auth": "" }, + "http://localhost:5000": { + "auth": "" + }, "gcr.io": { "auth": "" } @@ -92,12 +98,24 @@ the token is a base64 encoded value of `:`. For `gcr.io`, it's a [json key][json-key] file. Using docker [credential helpers][credential-helpers] also works but please ensure that the required credential helper program is installed to handle the credentials. If -the docker client configuration contains `"credHelpers"` block, but the +the docker registry configuration contains `"credHelpers"` block, but the associated helper program isn't installed or not configured properly, ignite image pull will fail with errors related to the specific credential helper. In presence of both auth tokens and credential helpers in a configuration file, credential helper takes precedence. +The `--registry-config-dir` flag can be used to override the default directory(`$HOME/.docker/config.json`). +This can also be done from the ignite [Configuration](./ignite-configuration). + +When using the `containerd` runtime to pull images, TLS verification can be disabled, +and `http://` protocols can be specified by using the client-side `IGNITE_CONTAINERD_INSECURE_REGISTRIES` +environment variable as a comma separate list. +In this list, the protocol is completely ignored, because it's specified by the registry-configuration: + +```shell +IGNITE_CONTAINERD_INSECURE_REGISTRIES="localhost:5000,localhost:5001,example.com,http://example.com" +``` + [json-key]: https://cloud.google.com/container-registry/docs/advanced-authentication#json-key [credential-helpers]: https://docs.docker.com/engine/reference/commandline/login/#credential-helpers diff --git a/e2e/registry_auth_test.go b/e2e/registry_auth_test.go index 046e8ee6d..a9b8b18d2 100644 --- a/e2e/registry_auth_test.go +++ b/e2e/registry_auth_test.go @@ -14,8 +14,10 @@ import ( ) const ( - testOSImage = "localhost:5000/weaveworks/ignite-ubuntu:test" - testKernelImage = "localhost:5000/weaveworks/ignite-kernel:test" + httpTestOSImage = "127.5.0.1:5080/weaveworks/ignite-ubuntu:test" + httpTestKernelImage = "127.5.0.1:5080/weaveworks/ignite-kernel:test" + httpsTestOSImage = "127.5.0.1:5443/weaveworks/ignite-ubuntu:test" + httpsTestKernelImage = "127.5.0.1:5443/weaveworks/ignite-kernel:test" ) // client config with auth info for the registry setup in @@ -24,22 +26,40 @@ const ( // is updated. const clientConfigContent = ` { - "auths": { - "localhost:5000": { - "auth": "dGVzdHVzZXI6dGVzdHBhc3N3b3Jk" - } - } + "auths": { + "http://127.5.0.1:5080": { + "auth": "aHR0cF90ZXN0dXNlcjpodHRwX3Rlc3RwYXNzd29yZA==" + }, + "https://127.5.0.1:5443": { + "auth": "aHR0cHNfdGVzdHVzZXI6aHR0cHNfdGVzdHBhc3N3b3Jk" + } + } } ` func TestPullFromAuthRegistry(t *testing.T) { assert.Assert(t, e2eHome != "", "IGNITE_E2E_HOME should be set") + os.Setenv("IGNITE_CONTAINERD_INSECURE_REGISTRIES", "http://127.5.0.1:5080,https://127.5.0.1:5443") + defer os.Unsetenv("IGNITE_CONTAINERD_INSECURE_REGISTRIES") + + // Create a client config directory to use in test. + emptyDir, err := ioutil.TempDir("", "ignite-test") + assert.NilError(t, err) + defer os.RemoveAll(emptyDir) + // Create a client config directory to use in test. ccDir, err := ioutil.TempDir("", "ignite-test") assert.NilError(t, err) defer os.RemoveAll(ccDir) + // Ensure the directory exists and create a config file in the + // directory. + assert.NilError(t, os.MkdirAll(ccDir, 0755)) + configPath := filepath.Join(ccDir, "config.json") + assert.NilError(t, os.WriteFile(configPath, []byte(clientConfigContent), 0600)) + defer os.Remove(configPath) + templateConfig := `--- apiVersion: ignite.weave.works/v1alpha4 kind: Configuration @@ -50,14 +70,14 @@ spec: ` igniteConfigContent := fmt.Sprintf(templateConfig, ccDir) - cases := []struct { - name string - runtime runtime.Name - configWithAuthPath string - clientConfigFlag string - igniteConfig string - wantErr bool - }{ + type testCase struct { + name string + runtime runtime.Name + clientConfigFlag string + igniteConfig string + wantErr bool + } + cases := []testCase{ { name: "no auth info - containerd", runtime: runtime.RuntimeContainerd, @@ -69,71 +89,62 @@ spec: wantErr: true, }, { - name: "client config flag - containerd", - runtime: runtime.RuntimeContainerd, - configWithAuthPath: ccDir, - clientConfigFlag: ccDir, + name: "client config flag - containerd", + runtime: runtime.RuntimeContainerd, + clientConfigFlag: ccDir, }, { - name: "client config flag - docker", - runtime: runtime.RuntimeDocker, - configWithAuthPath: ccDir, - clientConfigFlag: ccDir, + name: "client config flag - docker", + runtime: runtime.RuntimeDocker, + clientConfigFlag: ccDir, }, { - name: "client config in ignite config - containerd", - runtime: runtime.RuntimeContainerd, - configWithAuthPath: ccDir, - igniteConfig: igniteConfigContent, + name: "client config in ignite config - containerd", + runtime: runtime.RuntimeContainerd, + igniteConfig: igniteConfigContent, }, { - name: "client config in ignite config - docker", - runtime: runtime.RuntimeDocker, - configWithAuthPath: ccDir, - igniteConfig: igniteConfigContent, + name: "client config in ignite config - docker", + runtime: runtime.RuntimeDocker, + igniteConfig: igniteConfigContent, }, // Following sets the client config dir to a location without a valid // client config file, although the client config dir in the ignite // config is correct, the import fails due to bad configuration by the // flag override. { - name: "flag override client config - containerd", - runtime: runtime.RuntimeContainerd, - configWithAuthPath: ccDir, - clientConfigFlag: "/tmp", - igniteConfig: igniteConfigContent, - wantErr: true, + name: "flag override client config - containerd", + runtime: runtime.RuntimeContainerd, + clientConfigFlag: emptyDir, + igniteConfig: igniteConfigContent, + wantErr: true, }, { - name: "flag override client config - docker", - runtime: runtime.RuntimeDocker, - configWithAuthPath: ccDir, - clientConfigFlag: "/tmp", - igniteConfig: igniteConfigContent, - wantErr: true, + name: "flag override client config - docker", + runtime: runtime.RuntimeDocker, + clientConfigFlag: emptyDir, + igniteConfig: igniteConfigContent, + wantErr: true, }, - // Following set the client config dir via flag without any actual + // Following sets the client config dir via flag without any actual // client config. Import fails due to missing auth info in the given // client config dir. { - name: "invalid client config - containerd", - runtime: runtime.RuntimeContainerd, - configWithAuthPath: "", - clientConfigFlag: ccDir, - wantErr: true, + name: "invalid client config - containerd", + runtime: runtime.RuntimeContainerd, + clientConfigFlag: emptyDir, + wantErr: true, }, { - name: "invalid client config - docker", - runtime: runtime.RuntimeDocker, - configWithAuthPath: "", - clientConfigFlag: ccDir, - wantErr: true, + name: "invalid client config - docker", + runtime: runtime.RuntimeDocker, + clientConfigFlag: emptyDir, + wantErr: true, }, } - for _, rt := range cases { - rt := rt - t.Run(rt.name, func(t *testing.T) { + testFunc := func(rt testCase, osImage, kernelImage string) func(t *testing.T) { + return func(t *testing.T) { igniteCmd := util.NewCommand(t, igniteBin) // Remove images from ignite store and runtime store. Remove @@ -141,18 +152,8 @@ spec: // whole command. // TODO: Improve image rm to not fail completely when there are // multiple images and some are not found. - util.RmiCompletely(testOSImage, igniteCmd, rt.runtime) - util.RmiCompletely(testKernelImage, igniteCmd, rt.runtime) - - // Write client config if given. - if len(rt.configWithAuthPath) > 0 { - // Ensure the directory exists and create a config file in the - // directory. - assert.NilError(t, os.MkdirAll(rt.configWithAuthPath, 0755)) - configPath := filepath.Join(rt.configWithAuthPath, "config.json") - assert.NilError(t, os.WriteFile(configPath, []byte(clientConfigContent), 0600)) - defer os.Remove(configPath) - } + util.RmiCompletely(osImage, igniteCmd, rt.runtime) + util.RmiCompletely(kernelImage, igniteCmd, rt.runtime) // Write ignite config if provided. var igniteConfigPath string @@ -181,7 +182,7 @@ spec: // Run image import. _, importErr := igniteCmd.New(). - With("image", "import", testOSImage). + With("image", "import", osImage). With(imageImportCmdArgs...). Cmd.CombinedOutput() if (importErr != nil) != rt.wantErr { @@ -190,12 +191,22 @@ spec: // Run kernel import. _, importErr = igniteCmd.New(). - With("image", "import", testKernelImage). + With("image", "import", kernelImage). With(imageImportCmdArgs...). Cmd.CombinedOutput() if (importErr != nil) != rt.wantErr { t.Error("expected kernel image import to fail") } - }) + } + } + + for _, rt := range cases { + rt := rt + t.Run("http_"+rt.name, testFunc(rt, httpTestOSImage, httpTestKernelImage)) + } + + for _, rt := range cases { + rt := rt + t.Run("https_"+rt.name, testFunc(rt, httpsTestOSImage, httpsTestKernelImage)) } } diff --git a/e2e/util/setup-private-registry.sh b/e2e/util/setup-private-registry.sh index 7d1fcf13c..3443fcab5 100644 --- a/e2e/util/setup-private-registry.sh +++ b/e2e/util/setup-private-registry.sh @@ -1,57 +1,95 @@ #!/usr/bin/env bash -set -e - -# This script runs a local private docker registry with self-signed certificate -# and basic auth. - -REGISTRY_SECRET_PATH=/tmp/ignite-test-registry -PRIVATE_KEY=${REGISTRY_SECRET_PATH}/certs/domain.key -CERT=${REGISTRY_SECRET_PATH}/certs/domain.crt -HTPASSWD=${REGISTRY_SECRET_PATH}/auth/htpasswd -USERNAME=testuser -PASSWORD=testpassword -REGISTRY_ADDRESS=https://localhost:5000 -OS_IMG=weaveworks/ignite-ubuntu:latest -LOCAL_OS_IMG=localhost:5000/weaveworks/ignite-ubuntu:test -KERNEL_IMG=weaveworks/ignite-kernel:5.4.108 -LOCAL_KERNEL_IMG=localhost:5000/weaveworks/ignite-kernel:test +set -eu + +# This script runs two local, private, docker registries, +# one with with self-signed certificates, TLS, and HTTP, +# the other with plain HTTP, both using basic auth. + +HTTP_REGISTRY_SECRET_PATH="$(mktemp -d)/ignite-test-registry-http" +HTTPS_REGISTRY_SECRET_PATH="$(mktemp -d)/ignite-test-registry-https" + +PRIVATE_KEY="${HTTPS_REGISTRY_SECRET_PATH}/certs/domain.key" +CERT="${HTTPS_REGISTRY_SECRET_PATH}/certs/domain.crt" + +HTTP_USERNAME="http_testuser" +HTTP_PASSWORD="http_testpassword" +HTTPS_USERNAME="https_testuser" +HTTPS_PASSWORD="https_testpassword" + +BIND_IP="127.5.0.1" +HTTP_ADDR="${BIND_IP}:5080" +HTTPS_ADDR="${BIND_IP}:5443" + +OS_IMG="weaveworks/ignite-ubuntu:latest" +KERNEL_IMG="weaveworks/ignite-kernel:5.4.108" +HTTP_LOCAL_OS_IMG="${HTTP_ADDR}/weaveworks/ignite-ubuntu:test" +HTTP_LOCAL_KERNEL_IMG="${HTTP_ADDR}/weaveworks/ignite-kernel:test" +HTTPS_LOCAL_OS_IMG="${HTTPS_ADDR}/weaveworks/ignite-ubuntu:test" +HTTPS_LOCAL_KERNEL_IMG="${HTTPS_ADDR}/weaveworks/ignite-kernel:test" # Clear any existing registry secret and create new directories. -rm -rf ${REGISTRY_SECRET_PATH} -mkdir -p ${REGISTRY_SECRET_PATH}/{certs,auth} +mkdir -p "${HTTP_REGISTRY_SECRET_PATH}/auth" +mkdir -p "${HTTPS_REGISTRY_SECRET_PATH}/"{certs,auth} # Generate key and cert. openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \ -subj "/C=US/ST=Foo/L=Bar/O=Weave" \ - -keyout ${PRIVATE_KEY} -out ${CERT} -chmod 400 ${PRIVATE_KEY} + -keyout "${PRIVATE_KEY}" -out "${CERT}" +chmod 400 "${PRIVATE_KEY}" -# Create htpasswd file. +# Create htpasswd files. docker run --rm \ --entrypoint htpasswd \ - httpd:2 -Bbn ${USERNAME} ${PASSWORD} > ${HTPASSWD} + httpd:2 -Bbn "${HTTP_USERNAME}" "${HTTP_PASSWORD}" > "${HTTP_REGISTRY_SECRET_PATH}/auth/htpasswd" + +docker run --rm \ + --entrypoint htpasswd \ + httpd:2 -Bbn "${HTTPS_USERNAME}" "${HTTPS_PASSWORD}" > "${HTTPS_REGISTRY_SECRET_PATH}/auth/htpasswd" + +# Run the registries +docker run -d --rm \ + --name ignite-test-http-registry \ + -v "${HTTP_REGISTRY_SECRET_PATH}/auth":/auth \ + -e REGISTRY_AUTH=htpasswd \ + -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" \ + -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + -p "${HTTP_ADDR}":5000 \ + registry:2 -# Run the registry. docker run -d --rm \ - --name registry \ - -v ${REGISTRY_SECRET_PATH}/auth:/auth \ - -v ${REGISTRY_SECRET_PATH}/certs:/certs \ + --name ignite-test-https-registry \ + -v "${HTTPS_REGISTRY_SECRET_PATH}/auth":/auth \ + -v "${HTTPS_REGISTRY_SECRET_PATH}/certs":/certs \ -e REGISTRY_AUTH=htpasswd \ -e REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" \ -e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \ -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \ - -p 5000:5000 \ + -p "${HTTPS_ADDR}":5000 \ registry:2 # Login, push test images to download in tests and logout. -docker login -u ${USERNAME} -p ${PASSWORD} ${REGISTRY_ADDRESS} -docker pull ${OS_IMG} -docker pull ${KERNEL_IMG} -docker tag ${OS_IMG} ${LOCAL_OS_IMG} -docker tag ${KERNEL_IMG} ${LOCAL_KERNEL_IMG} -docker push ${LOCAL_OS_IMG} -docker push ${LOCAL_KERNEL_IMG} -docker rmi ${LOCAL_OS_IMG} ${LOCAL_KERNEL_IMG} -docker logout ${REGISTRY_ADDRESS} +docker pull "${OS_IMG}" +docker pull "${KERNEL_IMG}" + +docker tag "${OS_IMG}" "${HTTP_LOCAL_OS_IMG}" +docker tag "${KERNEL_IMG}" "${HTTP_LOCAL_KERNEL_IMG}" +docker tag "${OS_IMG}" "${HTTPS_LOCAL_OS_IMG}" +docker tag "${KERNEL_IMG}" "${HTTPS_LOCAL_KERNEL_IMG}" + +docker login -u "${HTTP_USERNAME}" -p "${HTTP_PASSWORD}" "https://${HTTP_ADDR}" +docker login -u "${HTTPS_USERNAME}" -p "${HTTPS_PASSWORD}" "https://${HTTPS_ADDR}" + +# push in parallel, block until all finished +docker push "${HTTP_LOCAL_OS_IMG}" & +docker push "${HTTP_LOCAL_KERNEL_IMG}" & +docker push "${HTTPS_LOCAL_OS_IMG}" & +docker push "${HTTPS_LOCAL_KERNEL_IMG}" & +wait + +docker logout "http://${HTTP_ADDR}" +docker logout "https://${HTTPS_ADDR}" + +docker rmi "${HTTP_LOCAL_OS_IMG}" "${HTTP_LOCAL_KERNEL_IMG}" +docker rmi "${HTTPS_LOCAL_OS_IMG}" "${HTTPS_LOCAL_KERNEL_IMG}" diff --git a/pkg/runtime/auth/auth.go b/pkg/runtime/auth/auth.go index eef0e53e8..6f920d2b0 100644 --- a/pkg/runtime/auth/auth.go +++ b/pkg/runtime/auth/auth.go @@ -18,23 +18,24 @@ type AuthCreds func(string) (string, string, error) // NewAuthCreds returns an AuthCreds which loads the credentials from the // docker client config. -func NewAuthCreds(refHostname string, configPath string) (AuthCreds, error) { +func NewAuthCreds(refHostname string, configPath string) (AuthCreds, string, error) { log.Debugf("runtime.auth: client config dir path: %q", configPath) // Load does not raise an error on ENOENT dockerConfigFile, err := dockercliconfig.Load(configPath) if err != nil { - return nil, err + return nil, "", err } // DefaultHost converts "docker.io" to "registry-1.docker.io", // which is wanted by credFunc . credFuncExpectedHostname, err := docker.DefaultHost(refHostname) if err != nil { - return nil, err + return nil, "", err } var credFunc AuthCreds + var serverAddress string authConfigHostnames := []string{refHostname} if refHostname == "docker.io" || refHostname == "registry-1.docker.io" { @@ -62,7 +63,7 @@ func NewAuthCreds(refHostname string, configPath string) (AuthCreds, error) { } else { acsaHostname := credentials.ConvertToHostname(ac.ServerAddress) if acsaHostname != authConfigHostname { - return nil, fmt.Errorf("expected the hostname part of ac.ServerAddress (%q) to be authConfigHostname=%q, got %q", + return nil, "", fmt.Errorf("expected the hostname part of ac.ServerAddress (%q) to be authConfigHostname=%q, got %q", ac.ServerAddress, authConfigHostname, acsaHostname) } } @@ -83,12 +84,13 @@ func NewAuthCreds(refHostname string, configPath string) (AuthCreds, error) { } return ac.Username, ac.Password, nil } + serverAddress = ac.ServerAddress break } } } // credFunc can be nil here. - return credFunc, nil + return credFunc, serverAddress, nil } func isAuthConfigEmpty(ac dockercliconfigtypes.AuthConfig) bool { diff --git a/pkg/runtime/containerd/client.go b/pkg/runtime/containerd/client.go index 2c48377d7..7942764f2 100644 --- a/pkg/runtime/containerd/client.go +++ b/pkg/runtime/containerd/client.go @@ -11,9 +11,11 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" "syscall" "time" + "github.com/docker/cli/cli/config/credentials" meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1" "github.com/weaveworks/ignite/pkg/constants" "github.com/weaveworks/ignite/pkg/preflight" @@ -149,27 +151,44 @@ func GetContainerdClient() (*ctdClient, error) { // host name. func newRemoteResolver(refHostname string, configPath string) (remotes.Resolver, error) { var authzOpts []docker.AuthorizerOpt - if authCreds, err := auth.NewAuthCreds(refHostname, configPath); err != nil { + regOpts := []docker.RegistryOpt{} + insecureAllowed := false + client := &http.Client{} + + // Allow setting insecure_registries through a client-side ENV variable. + // dockerconfig.json does not have a place to set this. + // We would have to override the parser to add a field otherwise. + for _, reg := range strings.Split(os.Getenv("IGNITE_CONTAINERD_INSECURE_REGISTRIES"), ",") { + // image hostnames don't have protocols, this is the most forgiving parsing logic. + if credentials.ConvertToHostname(reg) == refHostname { + insecureAllowed = true + } + } + + if authCreds, serverAddress, err := auth.NewAuthCreds(refHostname, configPath); err != nil { return nil, err } else { authzOpts = append(authzOpts, docker.WithAuthCreds(authCreds)) + // Allow the dockerconfig.json to specify HTTP as a specific protocol override, defaults to HTTPS + if strings.HasPrefix(serverAddress, "http://") { + if !insecureAllowed { + return nil, fmt.Errorf("Registry %q uses plain HTTP, but is not in the IGNITE_CONTAINERD_INSECURE_REGISTRIES env var", serverAddress) + } + regOpts = append(regOpts, docker.WithPlainHTTP(docker.MatchAllHosts)) + } else { + if insecureAllowed { + log.Warnf("Disabling TLS Verification for %q via IGNITE_CONTAINERD_INSECURE_REGISTRIES env var", serverAddress) + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + } + } } authz := docker.NewDockerAuthorizer(authzOpts...) - // TODO: Add plain http option. - regOpts := []docker.RegistryOpt{ - docker.WithAuthorizer(authz), - } - - // TODO: Make this opt-in via a flag option. - tr := &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - client := &http.Client{ - Transport: tr, - } + regOpts = append(regOpts, docker.WithAuthorizer(authz)) regOpts = append(regOpts, docker.WithClient(client)) // TODO: Add option to skip verifying HTTPS cert. diff --git a/pkg/runtime/docker/client.go b/pkg/runtime/docker/client.go index 473d81a43..f35fc7743 100644 --- a/pkg/runtime/docker/client.go +++ b/pkg/runtime/docker/client.go @@ -69,7 +69,7 @@ func (dc *dockerClient) PullImage(image meta.OCIImageRef) (err error) { } // Get available credentials from docker cli config. - authCreds, err := auth.NewAuthCreds(refDomain, providers.ClientConfigDir) + authCreds, _, err := auth.NewAuthCreds(refDomain, providers.ClientConfigDir) if err != nil { return err }