cmd/cue: cue mod upload command

This adds an experimental (and perhaps temporary) command
that can be used to upload a module to an OCI registry.

To repeat: this command is EXPERIMENTAL; usage and
name will change in the future.

Signed-off-by: Roger Peppe <rogpeppe@gmail.com>
Change-Id: I5aa976b528a1d7c9ae50335adb15921cdf8e5465
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1171801
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>

cmd/cue/cmd/mod.go

@@ -45,21 +45,21 @@ func newModCmd(c *Command) *cobra.Command {
 	}
 
 	cmd.AddCommand(newModInitCmd(c))
+	cmd.AddCommand(newModUploadCmd(c))
 	return cmd
 }
 
 func newModInitCmd(c *Command) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "init [module]",
 		Short: "initialize new module in current directory",
-		Long: `Init initializes a cue.mod directory in the current directory,
-in effect creating a new module rooted at the current directory.
-The cue.mod directory must not already exist.
-A legacy cue.mod file in the current directory is moved
-to the new subdirectory.
-
-A module name is optional, but if it is not given a packages
-within the module cannot imported another package defined
+		Long: `Init initializes a cue.mod directory in the current directory, in effect
+creating a new module rooted at the current directory. The cue.mod
+directory must not already exist. A legacy cue.mod file in the current
+directory is moved to the new subdirectory.
+
+A module name is optional, but if it is not given, a package
+within the module cannot import another package defined
 in the module.
 `,
 		RunE: mkRunE(c, runModInit),

cmd/cue/cmd/modupload.go

@@ -0,0 +1,96 @@
+// Copyright 2023 The CUE Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+
+	"cuelang.org/go/internal/mod/modfile"
+	"cuelang.org/go/internal/mod/modregistry"
+	"cuelang.org/go/internal/mod/module"
+	"cuelang.org/go/internal/mod/modzip"
+)
+
+func newModUploadCmd(c *Command) *cobra.Command {
+	cmd := &cobra.Command{
+		// TODO: this command is still experimental, don't show it in
+		// the documentation just yet.
+		Hidden: true,
+
+		Use:   "upload <version>",
+		Short: "upload the current module to a registry",
+		Long: `WARNING: THIS COMMAND IS EXPERIMENTAL.
+
+Upload the current module to an OCI registry.
+Currently this command must be run in the module's root directory.
+Also note that this command does no dependency or other checks at the moment.
+`,
+		RunE: mkRunE(c, runModUpload),
+		Args: cobra.ExactArgs(1),
+	}
+
+	return cmd
+}
+
+func runModUpload(cmd *Command, args []string) error {
+	reg, err := getRegistry()
+	if err != nil {
+		return err
+	}
+	if reg == nil {
+		return fmt.Errorf("no registry configured to upload to")
+	}
+	modfileData, err := os.ReadFile("cue.mod/module.cue")
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("no cue.mod/module.cue file found; cue mod upload must be run in the module's root directory")
+		}
+		return err
+	}
+	mf, err := modfile.Parse(modfileData, "cue.mod/module.cue")
+	if err != nil {
+		return err
+	}
+	mv, err := module.NewVersion(mf.Module, args[0])
+	if err != nil {
+		return fmt.Errorf("cannot form module version: %v", err)
+	}
+	zf, err := os.CreateTemp("", "cue-upload-")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(zf.Name())
+	defer zf.Close()
+
+	// TODO verify that all dependencies exist in the registry.
+	if err := modzip.CreateFromDir(zf, mv, "."); err != nil {
+		return err
+	}
+	info, err := zf.Stat()
+	if err != nil {
+		return err
+	}
+
+	rclient := modregistry.NewClient(reg)
+	if err := rclient.PutModule(context.Background(), mv, zf, info.Size()); err != nil {
+		return fmt.Errorf("cannot put module: %v", err)
+	}
+	fmt.Printf("uploaded %s\n", mv)
+	return nil
+}

cmd/cue/cmd/script_test.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 	"io/fs"
+	"net/http/httptest"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -28,6 +30,8 @@ import (
 	"testing"
 	"time"
 
+	"cuelabs.dev/go/oci/ociregistry/ocimem"
+	"cuelabs.dev/go/oci/ociregistry/ociserver"
 	"github.com/google/shlex"
 	"github.com/rogpeppe/go-internal/goproxytest"
 	"github.com/rogpeppe/go-internal/gotooltest"
@@ -109,6 +113,40 @@ func TestScript(t *testing.T) {
 					ts.Check(os.WriteFile(path, []byte(data), 0o666))
 				}
 			},
+			// memregistry starts an in-memory OCI server and sets the argument
+			// environment variable name to its hostname.
+			"memregistry": func(ts *testscript.TestScript, neg bool, args []string) {
+				usage := func() {
+					ts.Fatalf("usage: memregistry [-auth=username:password] <envvar-name>")
+				}
+				if neg {
+					usage()
+				}
+				var auth *registrytest.AuthConfig
+				if len(args) > 0 && strings.HasPrefix(args[0], "-") {
+					userPass, ok := strings.CutPrefix(args[0], "-auth=")
+					if !ok {
+						usage()
+					}
+					user, pass, ok := strings.Cut(userPass, ":")
+					if !ok {
+						usage()
+					}
+					auth = &registrytest.AuthConfig{
+						Username: user,
+						Password: pass,
+					}
+					args = args[1:]
+				}
+				if len(args) != 1 {
+					usage()
+				}
+
+				srv := httptest.NewServer(registrytest.AuthHandler(ociserver.New(ocimem.New(), nil), auth))
+				u, _ := url.Parse(srv.URL)
+				ts.Setenv(args[0], u.Host)
+				ts.Defer(srv.Close)
+			},
 		},
 		Setup: func(e *testscript.Env) error {
 			// Set up a home dir within work dir with a . prefix so that the

cmd/cue/cmd/testdata/script/registry_upload.txtar

@@ -0,0 +1,122 @@
+# Check that we can use the cue mod upload command to upload to a registry.
+memregistry MEMREGISTRY
+env ORIG_CUE_REGISTRY=$CUE_REGISTRY
+env CUE_REGISTRY=example.com=$MEMREGISTRY+insecure,$CUE_REGISTRY
+cd example
+exec cue mod upload v0.0.1
+cmp stdout ../expect-upload-stdout
+cd ../main
+exec cue eval .
+cmp stdout ../expect-eval-stdout
+
+# Sanity check that the module isn't present in the fallback registry.
+env CUE_REGISTRY=$ORIG_CUE_REGISTRY
+! exec cue eval
+stderr 'repository name not known to registry'
+
+-- expect-upload-stdout --
+uploaded example.com@v0.0.1
+-- expect-eval-stdout --
+main:                   "main"
+"foo.com/bar/hello@v0": "v0.2.3"
+"bar.com@v0":           "v0.5.0"
+"baz.org@v0":           "v0.10.1"
+"example.com@v0":       "v0.0.1"
+-- main/cue.mod/module.cue --
+module: "main.org"
+
+deps: "example.com@v0": v: "v0.0.1"
+
+-- main/main.cue --
+package main
+import "example.com@v0:main"
+
+main
+
+-- example/cue.mod/module.cue --
+module: "example.com@v0"
+deps: {
+	"foo.com/bar/hello@v0": v: "v0.2.3"
+	"bar.com@v0": v: "v0.5.0"
+}
+
+-- example/top.cue --
+package main
+
+// Note: import without a major version takes
+// the major version from the module.cue file.
+import a "foo.com/bar/hello"
+a
+main: "main"
+"example.com@v0": "v0.0.1"
+
+-- _registry/foo.com_bar_hello_v0.2.3/cue.mod/module.cue --
+module: "foo.com/bar/hello@v0"
+deps: {
+	"bar.com@v0": v: "v0.0.2"
+	"baz.org@v0": v: "v0.10.1"
+}
+
+-- _registry/foo.com_bar_hello_v0.2.3/x.cue --
+package hello
+import (
+	a "bar.com/bar@v0"
+	b "baz.org@v0:baz"
+)
+"foo.com/bar/hello@v0": "v0.2.3"
+a
+b
+
+
+-- _registry/bar.com_v0.0.2/cue.mod/module.cue --
+module: "bar.com@v0"
+deps: "baz.org@v0": v: "v0.0.2"
+
+-- _registry/bar.com_v0.0.2/bar/x.cue --
+package bar
+import a "baz.org@v0:baz"
+"bar.com@v0": "v0.0.2"
+a
+
+
+-- _registry/bar.com_v0.5.0/cue.mod/module.cue --
+module: "bar.com@v0"
+deps: "baz.org@v0": v: "v0.5.0"
+
+-- _registry/bar.com_v0.5.0/bar/x.cue --
+package bar
+import a "baz.org@v0:baz"
+"bar.com@v0": "v0.5.0"
+a
+
+
+-- _registry/baz.org_v0.0.2/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.0.2/baz.cue --
+package baz
+"baz.org@v0": "v0.0.2"
+
+
+-- _registry/baz.org_v0.1.2/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.1.2/baz.cue --
+package baz
+"baz.org@v0": "v0.1.2"
+
+
+-- _registry/baz.org_v0.5.0/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.5.0/baz.cue --
+package baz
+"baz.org@v0": "v0.5.0"
+
+
+-- _registry/baz.org_v0.10.1/cue.mod/module.cue --
+module: "baz.org@v0"
+
+-- _registry/baz.org_v0.10.1/baz.cue --
+package baz
+"baz.org@v0": "v0.10.1"

cmd/cue/cmd/testdata/script/registry_upload_auth.txtar

@@ -0,0 +1,49 @@
+# Check that we can use the cue mod upload command to upload to a registry
+# that's protected by authorization.
+
+memregistry -auth=foo:bar MEMREGISTRY
+env CUE_EXPERIMENT=modules
+env CUE_REGISTRY=$MEMREGISTRY+insecure
+env DOCKER_CONFIG=$WORK/dockerconfig
+env-fill $DOCKER_CONFIG/config.json
+
+cd example
+exec cue mod upload v0.0.1
+cmp stdout ../expect-upload-stdout
+cd ../main
+exec cue eval .
+cmp stdout ../expect-eval-stdout
+
+-- dockerconfig/config.json --
+{
+	"auths": {
+		"${MEMREGISTRY}": {
+			"username": "foo",
+			"password": "bar"
+		}
+	}
+}
+
+-- expect-upload-stdout --
+uploaded example.com@v0.0.1
+-- expect-eval-stdout --
+main:             "main"
+"example.com@v0": "v0.0.1"
+-- main/cue.mod/module.cue --
+module: "main.org"
+deps: "example.com@v0": v: "v0.0.1"
+
+-- main/main.cue --
+package main
+import "example.com@v0:main"
+
+main
+"main": "main"
+
+-- example/cue.mod/module.cue --
+module: "example.com@v0"
+
+-- example/top.cue --
+package main
+
+"example.com@v0": "v0.0.1"

internal/registrytest/registry.go

@@ -60,13 +60,13 @@ func New(fsys fs.FS, prefix string) (*Registry, error) {
 	if err := pushContent(client, mods); err != nil {
 		return nil, fmt.Errorf("cannot push modules: %v", err)
 	}
-	var handler http.Handler = ociserver.New(r, nil)
+	var handler http.Handler = ociserver.New(ocifilter.ReadOnly(r), nil)
 	if authConfigData != nil {
 		var cfg AuthConfig
 		if err := json.Unmarshal(authConfigData, &cfg); err != nil {
 			return nil, fmt.Errorf("invalid auth.json: %v", err)
 		}
-		handler = authMiddleware(handler, &cfg)
+		handler = AuthHandler(handler, &cfg)
 	}
 	srv := httptest.NewServer(handler)
 	u, err := url.Parse(srv.URL)
@@ -79,8 +79,12 @@ func New(fsys fs.FS, prefix string) (*Registry, error) {
 	}, nil
 }
 
-func authMiddleware(handler http.Handler, cfg *AuthConfig) http.Handler {
-	if cfg.Username == "" {
+// AuthHandler wraps the given handler with logic that checks
+// that the incoming requests fulfil the auth requirements defined
+// in cfg. If cfg is nil or there are no auth requirements, it returns handler
+// unchanged.
+func AuthHandler(handler http.Handler, cfg *AuthConfig) http.Handler {
+	if cfg == nil || cfg.Username == "" {
 		return handler
 	}
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {