feat: 初始化时加载代码，支持不同类型的代码获取方式

cnb-builder-shim/cmd/dev-entrypoint/init.go

@@ -20,14 +20,18 @@ package main
 
 import (
 	"fmt"
+	"net/url"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 
 	"github.com/go-logr/logr"
 	"github.com/pelletier/go-toml/v2"
 	"github.com/pkg/errors"
 
+	"github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config"
+	"github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/fetcher/fs"
 	"github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils"
 )
 
@@ -134,3 +138,71 @@ func setupBuildpacksOrder(logger logr.Logger, buildpacks string, cnbDir string)
 	}
 	return nil
 }
+
+// preFetchSourceCode: 根据配置初始化源码
+func preFetchSourceCode() error {
+	logger.Info(fmt.Sprintf("Downloading source code to %s...", config.G.SourceCode.SourceFetchMethod))
+	// TODO: 源码初始化不同的方式抽象成一个接口
+	workspace := config.G.SourceCode.Workspace
+	fetchSourceCode, err := ensureWorkspace(workspace)
+	if err != nil {
+		return err
+	}
+	// 源码已经下载到工作目录，无需初始化
+	if !fetchSourceCode {
+		return nil
+	}
+	switch config.G.SourceCode.SourceFetchMethod {
+	case config.BKREPO:
+		downloadUrl, err := url.Parse(config.G.SourceCode.SourceGetUrl)
+		if err != nil {
+			return err
+		}
+		// 创建临时文件夹存放源码压缩包
+		tmpDir, err := os.MkdirTemp("", "source-packages-*")
+		if err != nil {
+			return errors.Wrap(err, "create tmp dir")
+		}
+		defer os.RemoveAll(tmpDir)
+
+		// 下载源码压缩包
+		if err = fs.NewFetcher(logger).Fetch(downloadUrl.Path, tmpDir); err != nil {
+			return errors.Wrap(err, "download source code")
+		}
+		// 解压源码至工作目录
+		srcFilePath := path.Join(tmpDir, "tar")
+		if err = utils.ExtractTarGz(srcFilePath, workspace); err != nil {
+			return err
+		}
+	case config.GIT:
+		return fmt.Errorf("TODO: clone git from revision")
+	}
+	return nil
+}
+
+// ensureWorkspace 确保 workspace 文件夹存在，如果里面有文件则表示不需要预加载源码
+func ensureWorkspace(workspace string) (bool, error) {
+	// 检查文件夹是否存在
+	if _, err := os.Stat(workspace); os.IsNotExist(err) {
+		// 文件夹不存在，创建文件夹
+		logger.Info("creat workspace directory")
+		if err := os.MkdirAll(workspace, 0750); err != nil {
+			return false, errors.Wrap(err, "create workspace directory")
+		}
+		return true, nil
+	}
+
+	// 文件夹存在，检查文件夹里面是否有文件
+	files, err := os.ReadDir(workspace)
+	if err != nil {
+		return false, errors.Wrap(err, "read workspace directory")
+	}
+
+	if len(files) > 0 {
+		// 文件夹不为空，返回 nil
+		return false, nil
+	}
+
+	// 文件夹为空
+	return true, nil
+}

cnb-builder-shim/cmd/dev-entrypoint/main.go

@@ -34,6 +34,11 @@ func main() {
 		os.Exit(1)
 	}
 
+	if err := preFetchSourceCode(); err != nil {
+		logger.Error(err, "PreFetch Source Code Failed")
+		os.Exit(1)
+	}
+
 	runDevContainerServer()
 }
 

cnb-builder-shim/internal/devsandbox/config/config.go

@@ -0,0 +1 @@
+package config

cnb-builder-shim/internal/devsandbox/config/loader.go

@@ -0,0 +1,29 @@
+package config
+
+import "github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils"
+
+var G *Config
+
+func init() {
+	cfg, _ := loadConfigFromEnv()
+
+	// 设置全局变量
+	G = cfg
+}
+func loadSourceConfigFromEnv() (SourceCodeConfig, error) {
+	sourceFetchMethod := utils.EnvOrDefault("SOURCE_FETCH_METHOD", HTTP)
+	sourceGetUrl := utils.EnvOrDefault("SOURCE_GET_URL", "")
+	gitRevision := utils.EnvOrDefault("GIT_REVISION", "")
+	workspace := utils.EnvOrDefault("WORKSPACE", "/cnb/devsandbox/src")
+
+	return SourceCodeConfig{SourceFetchMethod: sourceFetchMethod, SourceGetUrl: sourceGetUrl, GitRevision: gitRevision, Workspace: workspace}, nil
+}
+
+func loadConfigFromEnv() (*Config, error) {
+	sourceCodeConfig, err := loadSourceConfigFromEnv()
+	if err != nil {
+		return nil, err
+	}
+
+	return &Config{SourceCode: sourceCodeConfig}, nil
+}

cnb-builder-shim/internal/devsandbox/config/type.go

@@ -0,0 +1,27 @@
+package config
+
+// 源码获取方式
+const (
+	HTTP   = "HTTP"
+	BKREPO = "BKREPO"
+	GIT    = "GIT"
+)
+
+// SourceCodeConfig 源码配置
+type SourceCodeConfig struct {
+	// 源码获取方式
+	SourceFetchMethod string
+	// 源码地址
+	SourceGetUrl string
+	// Git 仓库版本
+	GitRevision string
+	// 工作目录
+	Workspace string
+}
+
+// Config  全局配置
+type Config struct {
+	// 源码配置
+	SourceCode SourceCodeConfig
+	// ...
+}

cnb-builder-shim/internal/devsandbox/webserver/server.go

@@ -20,7 +20,10 @@ package webserver
 
 import (
 	"fmt"
+	"github.com/TencentBlueking/bkpaas/cnb-builder-shim/internal/devsandbox/config"
+	"github.com/TencentBlueking/bkpaas/cnb-builder-shim/pkg/utils"
 	"net/http"
+	"os"
 	"path"
 	"path/filepath"
 	"strconv"
@@ -70,6 +73,7 @@ func New(lg *logr.Logger) (*WebServer, error) {
 
 	mgr := service.NewDeployManager()
 	r.POST("/deploys", DeployHandler(s, mgr))
+	r.POST("/deploys_without_file", DeployWithoutFileHandler(s, mgr))
 	r.GET("/deploys/:deployID/results", ResultHandler(mgr))
 
 	return s, nil
@@ -110,6 +114,20 @@ func tokenAuthMiddleware(token string) gin.HandlerFunc {
 // DeployHandler handles the deployment of a file to the web server.
 func DeployHandler(s *WebServer, svc service.DeployServiceHandler) gin.HandlerFunc {
 	return func(c *gin.Context) {
+		// 在其他模式下，不允许使用该接口
+		if config.G.SourceCode.SourceFetchMethod != config.HTTP {
+			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("unsupported source fetch method: %s", config.G.SourceCode.SourceFetchMethod)})
+			return
+		}
+		var tmpAppDir string
+		// 创建临时文件夹
+		tmpDir, err := os.MkdirTemp("", "source-*")
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("create tmp dir err: %s", err.Error())})
+			return
+		}
+		defer os.RemoveAll(tmpDir)
+
 		file, err := c.FormFile("file")
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("get form err: %s", err.Error())})
@@ -122,8 +140,40 @@ func DeployHandler(s *WebServer, svc service.DeployServiceHandler) gin.HandlerFu
 			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("upload file err: %s", err.Error())})
 			return
 		}
+		// 解压文件到临时目录
+		if err = utils.Unzip(srcFilePath, tmpDir); err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("unzip file err: %s", err.Error())})
+			return
+		}
+		tmpAppDir = path.Join(tmpDir, strings.Split(fileName, ".")[0])
+
+		status, err := svc.Deploy(tmpAppDir)
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("deploy error: %s", err.Error())})
+			return
+		}
+
+		select {
+		case s.ch <- dc.AppReloadEvent{ID: status.DeployID, Rebuild: status.StepOpts.Rebuild, Relaunch: status.StepOpts.Relaunch}:
+			c.JSON(http.StatusOK, gin.H{"deployID": status.DeployID})
+		default:
+			c.JSON(
+				http.StatusTooManyRequests,
+				gin.H{"message": "app is deploying, please wait for a while and try again."},
+			)
+		}
+	}
+}
+
+// DeployWithoutFileHandler handles the deployment without file.
+func DeployWithoutFileHandler(s *WebServer, svc service.DeployServiceHandler) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if config.G.SourceCode.SourceFetchMethod == config.HTTP {
+			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("unsupported source fetch method: %s", config.G.SourceCode.SourceFetchMethod)})
+			return
+		}
 
-		status, err := svc.Deploy(srcFilePath)
+		status, err := svc.Deploy(config.G.SourceCode.Workspace)
 		if err != nil {
 			c.JSON(http.StatusBadRequest, gin.H{"message": fmt.Sprintf("deploy error: %s", err.Error())})
 			return

cnb-builder-shim/internal/devsandbox/webserver/service/deploy.go

@@ -73,16 +73,16 @@ type DeployManager struct {
 // It analyzes the source code and copies it to the application directory.
 //
 // Parameters:
-// - srcFilePath: The path of the source file to be deployed.
+// - tmpAppDir: The tmp path of the source file to be deployed.
 //
 // Returns:
 // - *DeployResult: The deployment result.
 // - error: Any error that occurred during the deployment process.
-func (m *DeployManager) Deploy(srcFilePath string) (*DeployResult, error) {
+func (m *DeployManager) Deploy(tmpAppDir string) (*DeployResult, error) {
 	deployID := m.newDeployID()
 
 	// 分析源码并拷贝到应用目录
-	if err := m.analyzeAndSyncToAppDir(srcFilePath, dc.DefaultAppDir); err != nil {
+	if err := m.analyzeAndSyncToAppDir(tmpAppDir, dc.DefaultAppDir); err != nil {
 		return nil, err
 	}
 
@@ -134,26 +134,12 @@ func (m DeployManager) newDeployID() string {
 	return strings.Replace(uuidString, "-", "", -1)
 }
 
-func (m *DeployManager) analyzeAndSyncToAppDir(srcFilePath, appDir string) error {
-	tmpDir, err := os.MkdirTemp("", "source-*")
-	if err != nil {
-		return err
-	}
-	defer os.RemoveAll(tmpDir)
-
-	// 1. 解压文件到临时目录
-	if err = utils.Unzip(srcFilePath, tmpDir); err != nil {
-		return err
-	}
-
-	fileName := filepath.Base(srcFilePath)
-	tmpAppDir := path.Join(tmpDir, strings.Split(fileName, ".")[0])
-
-	// 2. 通过对比新旧文件的变化, 确定哪些步骤需要执行
+func (m *DeployManager) analyzeAndSyncToAppDir(tmpAppDir, appDir string) error {
+	// 1. 通过对比新旧文件的变化, 确定哪些步骤需要执行
 	m.stepOpts = parseDeployStepOpts(appDir, tmpAppDir)
 
-	// 3. 将 tmpAppDir 中的文件拷贝到 appDir
-	if err = m.syncFiles(tmpAppDir, appDir); err != nil {
+	// 2. 将 tmpAppDir 中的文件拷贝到 appDir
+	if err := m.syncFiles(tmpAppDir, appDir); err != nil {
 		return err
 	}
 
@@ -233,7 +219,7 @@ var _ DeployServiceHandler = (*DeployManager)(nil)
 type FakeDeployManger struct{}
 
 // Deploy fake deploy for unit test
-func (m *FakeDeployManger) Deploy(srcFilePath string) (*DeployResult, error) {
+func (m *FakeDeployManger) Deploy(tmpAppDir string) (*DeployResult, error) {
 	return &DeployResult{
 		DeployID: uuid.NewString(),
 		Status:   dc.ReloadProcessing,

cnb-builder-shim/internal/devsandbox/webserver/service/deploy_test.go

@@ -34,7 +34,7 @@ var _ = Describe("Test DeployManager", func() {
 	var m *DeployManager
 	var tmpAppDir string
 
-	testSrcFilePath := filepath.Join("testdata", "django-helloworld.zip")
+	testSrcFilePath := filepath.Join("testdata", "tmpDir", "django-helloworld")
 	oldAppDir := dc.DefaultAppDir
 
 	BeforeEach(func() {

cnb-builder-shim/internal/devsandbox/webserver/service/testdata/tmpDir/django-helloworld/Procfile

@@ -0,0 +1,2 @@
+web1: python manage.py runserver 0.0.0.0:8080
+web: python manage.py runserver 0.0.0.0:8000