feat(vscode): Use openvsx (#174)

* feat(vscode): Use openvsx

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Check cache first

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

* fix: Remove version in README

Signed-off-by: Ce Gao <cegao@tensorchord.ai>

README.md

@@ -47,7 +47,7 @@ Checkout the [examples](./examples/mnist), and configure envd with the manifest
 
 ```python
 vscode(plugins=[
-    "ms-python.python-2021.12.1559732655",
+    "ms-python.python",
 ])
 
 base(os="ubuntu20.04", language="python3")

pkg/editor/vscode/types.go

@@ -17,15 +17,26 @@ package vscode
 import "fmt"
 
 const (
-	vscodePackageURLTemplate = "https://%s.gallery.vsassets.io/_apis/public/gallery/publisher/%s/extension/%s/%s/assetbyname/Microsoft.VisualStudio.Services.VSIXPackage"
+	vendorVSCodeTemplate  = "https://%s.gallery.vsassets.io/_apis/public/gallery/publisher/%s/extension/%s/%s/assetbyname/Microsoft.VisualStudio.Services.VSIXPackage"
+	vendorOpenVSXTemplate = "https://open-vsx.org/api/%s/%s/latest"
+)
+
+type MarketplaceVendor string
+
+const (
+	MarketplaceVendorVSCode  MarketplaceVendor = "vscode"
+	MarketplaceVendorOpenVSX MarketplaceVendor = "openvsx"
 )
 
 type Plugin struct {
 	Publisher string
 	Extension string
-	Version   string
+	Version   *string
 }
 
 func (p Plugin) String() string {
-	return fmt.Sprintf("%s.%s-%s", p.Publisher, p.Extension, p.Version)
+	if p.Version != nil {
+		return fmt.Sprintf("%s.%s-%s", p.Publisher, p.Extension, *p.Version)
+	}
+	return fmt.Sprintf("%s.%s", p.Publisher, p.Extension)
 }

pkg/editor/vscode/util.go

@@ -0,0 +1,84 @@
+package vscode
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/cockroachdb/errors"
+	"github.com/sirupsen/logrus"
+)
+
+func GetLatestVersionURL(p Plugin) (string, error) {
+	// Auto-detect the version.
+	// Refer to https://github.com/tensorchord/envd/issues/161#issuecomment-1129475975
+	latestURL := fmt.Sprintf(vendorOpenVSXTemplate, p.Publisher, p.Extension)
+	resp, err := http.Get(latestURL)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get latest version")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return "", errors.Errorf("failed to get latest version: %s", resp.Status)
+	}
+	jsonResp := make(map[string]interface{})
+	if err := json.NewDecoder(resp.Body).Decode(&jsonResp); err != nil {
+		return "", errors.Wrap(err, "failed to decode response")
+	}
+	if jsonResp["files"] == nil {
+		return "", errors.New("failed to get latest version: no files")
+	}
+	files := jsonResp["files"].(map[string]interface{})
+	if files["download"] == nil {
+		return "", errors.New("failed to get latest version: no download url")
+	}
+	return files["download"].(string), nil
+}
+
+func ParsePlugin(p string) (*Plugin, error) {
+	indexPublisher := strings.Index(p, ".")
+	if indexPublisher == -1 {
+		return nil, errors.New("invalid publisher")
+	}
+	publisher := p[:indexPublisher]
+
+	indexExtension := strings.LastIndex(p[indexPublisher:], "-")
+	if indexExtension == -1 {
+		extension := p[indexPublisher+1:]
+		logrus.WithFields(logrus.Fields{
+			"publisher": publisher,
+			"extension": extension,
+		}).Debug("vscode plugin is parsed without version")
+		return &Plugin{
+			Publisher: publisher,
+			Extension: extension,
+		}, nil
+	}
+
+	indexExtension = indexPublisher + indexExtension
+	extension := p[indexPublisher+1 : indexExtension]
+	version := p[indexExtension+1:]
+	if _, err := strconv.Atoi(version[0:1]); err != nil {
+		extension := p[indexPublisher+1:]
+		logrus.WithFields(logrus.Fields{
+			"publisher": publisher,
+			"extension": extension,
+		}).Debug("vscode plugin is parsed without version")
+		return &Plugin{
+			Publisher: publisher,
+			Extension: extension,
+		}, nil
+	}
+	logrus.WithFields(logrus.Fields{
+		"publisher": publisher,
+		"extension": extension,
+		"version":   version,
+	}).Debug("vscode plugin is parsed")
+	return &Plugin{
+		Publisher: publisher,
+		Extension: extension,
+		Version:   &version,
+	}, nil
+}

pkg/editor/vscode/vscode_test.go

@@ -20,6 +20,14 @@ import (
 
 var _ = Describe("Visual Studio Code", func() {
 	Describe("Plugin", func() {
+		It("should get the latest version successfully", func() {
+			url, err := GetLatestVersionURL(Plugin{
+				Publisher: "redhat",
+				Extension: "java",
+			})
+			Expect(err).To(BeNil())
+			Expect(url).NotTo(Equal(""))
+		})
 		It("should be able to parse", func() {
 			tcs := []struct {
 				name              string
@@ -57,11 +65,14 @@ var _ = Describe("Visual Studio Code", func() {
 					expectedErr:       false,
 				},
 				{
-					name:        "test",
-					expectedErr: true,
+					name:              "dbaeumer.vscode-eslint",
+					expectedPublisher: "dbaeumer",
+					expectedExtension: "vscode-eslint",
+					expectedVersion:   "",
+					expectedErr:       false,
 				},
 				{
-					name:        "test.test",
+					name:        "test",
 					expectedErr: true,
 				},
 			}
@@ -73,7 +84,10 @@ var _ = Describe("Visual Studio Code", func() {
 					Expect(err).ToNot(HaveOccurred())
 					Expect(p.Publisher).To(Equal(tc.expectedPublisher))
 					Expect(p.Extension).To(Equal(tc.expectedExtension))
-					Expect(p.Version).To(Equal(tc.expectedVersion))
+					if tc.expectedVersion != "" {
+						Expect(p.Version).NotTo(BeNil())
+						Expect(*p.Version).To(Equal(tc.expectedVersion))
+					}
 				}
 			}
 		})

pkg/editor/vscode/vsocde.go

@@ -19,7 +19,6 @@ import (
 	"io"
 	"net/http"
 	"os"
-	"strings"
 
 	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
@@ -38,29 +37,75 @@ type Client interface {
 }
 
 type generalClient struct {
+	vendor MarketplaceVendor
+	logger *logrus.Entry
 }
 
-func NewClient() Client {
-	return &generalClient{}
+func NewClient(vendor MarketplaceVendor) (Client, error) {
+	switch vendor {
+	case MarketplaceVendorOpenVSX:
+		return &generalClient{
+			vendor: vendor,
+			logger: logrus.WithField("vendor", MarketplaceVendorOpenVSX),
+		}, nil
+	case MarketplaceVendorVSCode:
+		return &generalClient{
+			vendor: vendor,
+			logger: logrus.WithField("vendor", MarketplaceVendorVSCode),
+		}, nil
+	default:
+		return nil, errors.Errorf("unknown marketplace vendor %s", vendor)
+	}
 }
 
 func (c generalClient) PluginPath(p Plugin) string {
-	return fmt.Sprintf("%s.%s-%s/extension/", p.Publisher, p.Extension, p.Version)
+	if p.Version != nil {
+		return fmt.Sprintf("%s.%s-%s/extension/", p.Publisher, p.Extension, *p.Version)
+
+	}
+	return fmt.Sprintf("%s.%s/extension/", p.Publisher, p.Extension)
 }
 
 func unzipPath(p Plugin) string {
-	return fmt.Sprintf("%s/%s.%s-%s", home.GetManager().CacheDir(),
-		p.Publisher, p.Extension, p.Version)
+	if p.Version != nil {
+		return fmt.Sprintf("%s/%s.%s-%s", home.GetManager().CacheDir(),
+			p.Publisher, p.Extension, *p.Version)
+	}
+	return fmt.Sprintf("%s/%s.%s", home.GetManager().CacheDir(),
+		p.Publisher, p.Extension)
 }
 
 // DownloadOrCache downloads or cache the plugin.
 // If the plugin is already downloaded, it returns true.
 func (c generalClient) DownloadOrCache(p Plugin) (bool, error) {
-	url := fmt.Sprintf(vscodePackageURLTemplate,
-		p.Publisher, p.Publisher, p.Extension, p.Version)
+	cacheKey := fmt.Sprintf("%s-%s", cachekeyPrefix, p)
+	if home.GetManager().Cached(cacheKey) {
+		logrus.WithFields(logrus.Fields{
+			"cache": cacheKey,
+		}).Debugf("vscode plugin %s already exists in cache", p)
+		return true, nil
+	}
+
+	var url, filename string
+	if c.vendor == MarketplaceVendorVSCode {
+		if p.Version == nil {
+			return false, errors.New("version is required for vscode marketplace")
+		}
+		// TODO(gaocegege): Support version auto-detection.
+		url = fmt.Sprintf(vendorVSCodeTemplate,
+			p.Publisher, p.Publisher, p.Extension, *p.Version)
+		filename = fmt.Sprintf("%s/%s.%s-%s.vsix", home.GetManager().CacheDir(),
+			p.Publisher, p.Extension, *p.Version)
+	} else {
+		var err error
+		url, err = GetLatestVersionURL(p)
+		if err != nil {
+			return false, errors.Wrap(err, "failed to get latest version url")
+		}
+		filename = fmt.Sprintf("%s/%s.%s.vsix", home.GetManager().CacheDir(),
+			p.Publisher, p.Extension)
+	}
 
-	filename := fmt.Sprintf("%s/%s.%s-%s.vsix", home.GetManager().CacheDir(),
-		p.Publisher, p.Extension, p.Version)
 	logger := logrus.WithFields(logrus.Fields{
 		"publisher": p.Publisher,
 		"extension": p.Extension,
@@ -69,14 +114,6 @@ func (c generalClient) DownloadOrCache(p Plugin) (bool, error) {
 		"file":      filename,
 	})
 
-	cacheKey := fmt.Sprintf("%s-%s", cachekeyPrefix, p)
-	if home.GetManager().Cached(cacheKey) {
-		logger.WithFields(logrus.Fields{
-			"cache": cacheKey,
-		}).Debugf("vscode plugin %s already exists in cache", p)
-		return true, nil
-	}
-
 	logger.Debugf("downloading vscode plugin %s", p)
 	out, err := os.Create(filename)
 
@@ -107,30 +144,3 @@ func (c generalClient) DownloadOrCache(p Plugin) (bool, error) {
 	}
 	return false, nil
 }
-
-func ParsePlugin(p string) (*Plugin, error) {
-	indexPublisher := strings.Index(p, ".")
-	if indexPublisher == -1 {
-		return nil, errors.New("invalid publisher")
-	}
-	publisher := p[:indexPublisher]
-
-	indexExtension := strings.LastIndex(p[indexPublisher:], "-")
-	if indexExtension == -1 {
-		return nil, errors.New("invalid extension")
-	}
-
-	indexExtension = indexPublisher + indexExtension
-	extension := p[indexPublisher+1 : indexExtension]
-	version := p[indexExtension+1:]
-	logrus.WithFields(logrus.Fields{
-		"publisher": publisher,
-		"extension": extension,
-		"version":   version,
-	}).Debug("vscode plugin is parsed")
-	return &Plugin{
-		Publisher: publisher,
-		Extension: extension,
-		Version:   version,
-	}, nil
-}

pkg/lang/ir/editor.go

@@ -1,7 +1,9 @@
 package ir
 
 import (
+	"github.com/cockroachdb/errors"
 	"github.com/moby/buildkit/client/llb"
+
 	"github.com/tensorchord/envd/pkg/editor/vscode"
 	"github.com/tensorchord/envd/pkg/flag"
 	"github.com/tensorchord/envd/pkg/progress/compileui"
@@ -13,7 +15,10 @@ func (g Graph) compileVSCode() (*llb.State, error) {
 	}
 	inputs := []llb.State{}
 	for _, p := range g.VSCodePlugins {
-		vscodeClient := vscode.NewClient()
+		vscodeClient, err := vscode.NewClient(vscode.MarketplaceVendorOpenVSX)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to create vscode client")
+		}
 		g.Writer.LogVSCodePlugin(p, compileui.ActionStart, false)
 		if cached, err := vscodeClient.DownloadOrCache(p); err != nil {
 			return nil, err