Skip to content

Commit

Permalink
tiltfile: load compose services from multiple projects/directories
Browse files Browse the repository at this point in the history 
Fixes #5170.

Signed-off-by: Nick Sieger <nick@nicksieger.com>
  • Loading branch information
@nicksieger
nicksieger committed Oct 11, 2022
1 parent ac882d4 commit 06b32ae
Show file tree
Hide file tree
Showing 7 changed files with 276 additions and 112 deletions.
24 changes: 15 additions & 9 deletions internal/cli/down.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,19 +89,25 @@ func (c *downCmd) down(ctx context.Context, downDeps DownDeps, args []string) er
return err
}

var dcProject v1alpha1.DockerComposeProject
dcProjects := make(map[string]v1alpha1.DockerComposeProject)
for _, m := range sortedManifests {
if m.IsDC() {
dcProject = m.DockerComposeTarget().Spec.Project
break
if !m.IsDC() {
continue
}
proj := m.DockerComposeTarget().Spec.Project

if _, exists := dcProjects[proj.Name]; !exists {
dcProjects[proj.Name] = proj
}
}

if !model.IsEmptyDockerComposeProject(dcProject) {
dcc := downDeps.dcClient
err = dcc.Down(ctx, dcProject, logger.Get(ctx).Writer(logger.InfoLvl), logger.Get(ctx).Writer(logger.InfoLvl))
if err != nil {
return errors.Wrap(err, "Running `docker-compose down`")
for _, dcProject := range dcProjects {
if !model.IsEmptyDockerComposeProject(dcProject) {
dcc := downDeps.dcClient
err = dcc.Down(ctx, dcProject, logger.Get(ctx).Writer(logger.InfoLvl), logger.Get(ctx).Writer(logger.InfoLvl))
if err != nil {
return errors.Wrap(err, "Running `docker-compose down`")
}
}
}

Expand Down
11 changes: 9 additions & 2 deletions internal/tiltfile/api/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,9 @@ def docker_compose(configPaths: Union[str, Blob, List[Union[str, Blob]]], env_fi
Args:
configPaths: Path(s) and/or Blob(s) to Docker Compose yaml files or content.
env_file: Path to env file to use; defaults to ``.env`` in current directory.
project_name: The Docker Compose project name. If unspecified, the main Tiltfile's directory name is used.
project_name: The Docker Compose project name. If unspecified, uses either the
name of the directory containing the first compose file, or, in the case of
inline YAML, the current Tiltfile's directory name.
"""


Expand Down Expand Up @@ -428,7 +430,9 @@ def dc_resource(name: str,
resource_deps: List[str] = [],
links: Union[str, Link, List[Union[str, Link]]] = [],
labels: Union[str, List[str]] = [],
auto_init: bool = True) -> None:
auto_init: bool = True,
project_name: str = "",
new_name: str = "") -> None:
"""Configures the Docker Compose resource of the given name. Note: Tilt does an amount of resource configuration
for you(for more info, see `Tiltfile Concepts: Resources <tiltfile_concepts.html#resources>`_); you only need
to invoke this function if you want to configure your resource beyond what Tilt does automatically.
Expand All @@ -444,6 +448,9 @@ def dc_resource(name: str,
labels: used to group resources in the Web UI, (e.g. you want all frontend services displayed together, while test and backend services are displayed seperately). A label must start and end with an alphanumeric character, can include ``_``, ``-``, and ``.``, and must be 63 characters or less. For an example, see `Resource Grouping <tiltfile_concepts.html#resource-groups>`_.
auto_init: whether this resource runs on ``tilt up``. Defaults to ``True``. For more info, see the
`Manual Update Control docs <manual_update_control.html>`_.
project_name: The Docker Compose project name to match the corresponding project loaded by
``docker_compose``, if necessary for disambiguation.
new_name: If non-empty, will be used as the new name for this resource.
"""

pass
Expand Down
180 changes: 131 additions & 49 deletions internal/tiltfile/docker_compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,26 @@ type dcResourceSet struct {
Project v1alpha1.DockerComposeProject

configPaths []string
services []*dcService
tiltfilePath string
services map[string]*dcService
serviceNames []string
resOptions map[string]*dcResourceOptions
}

type dcResourceMap map[string]*dcResourceSet

func (dc dcResourceSet) Empty() bool { return reflect.DeepEqual(dc, dcResourceSet{}) }

func (dc dcResourceSet) ServiceCount() int { return len(dc.services) }

func (dcm dcResourceMap) ServiceCount() int {
svcCount := 0
for _, dc := range dcm {
svcCount += dc.ServiceCount()
}
return svcCount
}

func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var configPaths starlark.Value
var projectName string
Expand All @@ -63,24 +77,9 @@ func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Buil
return nil, fmt.Errorf("Nothing to compose")
}

dc := s.dc

currentTiltfilePath := starkit.CurrentExecPath(thread)
if dc.tiltfilePath != "" && dc.tiltfilePath != currentTiltfilePath {
return starlark.None, fmt.Errorf("Cannot load docker-compose files from two different Tiltfiles.\n"+
"docker-compose must have a single working directory:\n"+
"(%s, %s)", dc.tiltfilePath, currentTiltfilePath)
}

if projectName == "" {
projectName = model.NormalizeName(filepath.Base(filepath.Dir(currentTiltfilePath)))
}

project := v1alpha1.DockerComposeProject{
ConfigPaths: dc.configPaths,
ProjectPath: dc.Project.ProjectPath,
Name: projectName,
EnvFile: envFile.Value,
Name: projectName,
EnvFile: envFile.Value,
}

if project.EnvFile != "" {
Expand Down Expand Up @@ -121,10 +120,13 @@ func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Buil
return starlark.None, fmt.Errorf("expected blob | path (string). Actual type: %T", val)
}

// Set project path to dir of first compose file, like DC CLI does
// Set project path/name to dir of first compose file, like DC CLI does
if project.ProjectPath == "" {
project.ProjectPath = filepath.Dir(path)
}
if project.Name == "" {
project.Name = filepath.Base(filepath.Dir(path))
}

project.ConfigPaths = append(project.ConfigPaths, path)
err = io.RecordReadPath(thread, io.WatchFileOnly, path)
Expand All @@ -134,26 +136,57 @@ func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Buil
}
}

currentTiltfilePath := starkit.CurrentExecPath(thread)

if project.Name == "" {
project.Name = model.NormalizeName(filepath.Base(filepath.Dir(currentTiltfilePath)))
}

// Set to tiltfile directory for YAML blob tempfiles
if project.ProjectPath == "" {
project.ProjectPath = filepath.Dir(currentTiltfilePath)
}

dc := s.dc[project.Name]
if dc == nil {
dc = &dcResourceSet{
Project: project,
services: make(map[string]*dcService),
resOptions: make(map[string]*dcResourceOptions),
configPaths: project.ConfigPaths,
tiltfilePath: currentTiltfilePath,
}
s.dc[project.Name] = dc
} else {
for _, path := range project.ConfigPaths {
exists := false
for _, extPath := range dc.configPaths {
if path == extPath {
exists = true
break
}
}
if !exists {
dc.configPaths = append(dc.configPaths, path)
}
}
dc.Project.ConfigPaths = dc.configPaths
if project.EnvFile != "" {
dc.Project.EnvFile = project.EnvFile
}
project = dc.Project
}

services, err := parseDCConfig(s.ctx, s.dcCli, project)
if err != nil {
return nil, err
}

dc.services = make(map[string]*dcService)
dc.serviceNames = []string{}
for _, svc := range services {
previousSvc := s.dcByName[svc.Name]
if previousSvc != nil {
delete(s.dcByName, svc.Name)
}
err := s.checkResourceConflict(svc.Name)
if err != nil {
return nil, err
}
svc.Options = s.dcResOptions[svc.Name]
dc.serviceNames = append(dc.serviceNames, svc.Name)
svc.Options = dc.resOptions[svc.Name]
for _, f := range svc.ServiceConfig.EnvFile {
if !filepath.IsAbs(f) {
f = filepath.Join(project.ProjectPath, f)
Expand All @@ -163,14 +196,7 @@ func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Buil
return nil, err
}
}
s.dcByName[svc.Name] = svc
}

s.dc = dcResourceSet{
Project: project,
configPaths: project.ConfigPaths,
services: services,
tiltfilePath: currentTiltfilePath,
dc.services[svc.Name] = svc
}

return starlark.None, nil
Expand All @@ -180,6 +206,8 @@ func (s *tiltfileState) dockerCompose(thread *starlark.Thread, fn *starlark.Buil
// to be defined in a `docker_compose.yml`
func (s *tiltfileState) dcResource(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var name string
var projectName string
var newName string
var imageVal starlark.Value
var triggerMode triggerMode
var resourceDepsVal starlark.Sequence
Expand All @@ -197,6 +225,8 @@ func (s *tiltfileState) dcResource(thread *starlark.Thread, fn *starlark.Builtin
"links?", &links,
"labels?", &labels,
"auto_init?", &autoInit,
"project_name?", &projectName,
"new_name?", &newName,
); err != nil {
return nil, err
}
Expand All @@ -215,12 +245,19 @@ func (s *tiltfileState) dcResource(thread *starlark.Thread, fn *starlark.Builtin
return nil, fmt.Errorf("image arg must be a string; got %T", imageVal)
}

svc, err := s.getDCService(name)
projectName, svc, err := s.getDCService(name, projectName)
if err != nil {
return nil, err
}

options := s.dcResOptions[name]
if newName != "" {
name, err = s.renameDCService(projectName, name, newName, svc)
if err != nil {
return nil, err
}
}

options := s.dc[projectName].resOptions[name]
if options == nil {
options = newDcResourceOptions()
}
Expand Down Expand Up @@ -253,27 +290,71 @@ func (s *tiltfileState) dcResource(thread *starlark.Thread, fn *starlark.Builtin
options.AutoInit = autoInit
}

s.dcResOptions[name] = options
s.dc[projectName].resOptions[name] = options
svc.Options = options
return starlark.None, nil
}

func (s *tiltfileState) getDCService(name string) (*dcService, error) {
allNames := make([]string, len(s.dc.services))
for i, svc := range s.dc.services {
if svc.Name == name {
return svc, nil
func (s *tiltfileState) getDCService(svcName, projName string) (string, *dcService, error) {
allNames := []string{}
foundProjName := ""
var foundSvc *dcService

for _, dc := range s.dc {
if projName != "" && dc.Project.Name != projName {
continue
}

for key, svc := range dc.services {
if key == svcName {
if foundSvc != nil {
return "", nil, fmt.Errorf("found multiple resources named %q, "+
"please specify which one with project_name= argument", svcName)
}
foundProjName = dc.Project.Name
foundSvc = svc
}
allNames = append(allNames, key)
}
}

if foundSvc == nil {
return "", nil, fmt.Errorf("no Docker Compose service found with name %q. "+
"Found these instead:\n\t%s", svcName, strings.Join(allNames, "; "))
}

return foundProjName, foundSvc, nil
}

func (s *tiltfileState) renameDCService(projectName, name, newName string, svc *dcService) (string, error) {
if _, existingSvc, _ := s.getDCService(newName, ""); existingSvc != nil {
return "", fmt.Errorf("dc_resource named %q already exists", newName)
}
s.dc[projectName].services[newName] = svc
delete(s.dc[projectName].services, name)
if opts, exists := s.dc[projectName].resOptions[name]; exists {
s.dc[projectName].resOptions[newName] = opts
delete(s.dc[projectName].resOptions, name)
}
index := -1
for i, n := range s.dc[projectName].serviceNames {
if n == name {
index = i
break
}
allNames[i] = svc.Name
}
return nil, fmt.Errorf("no Docker Compose service found with name '%s'. "+
"Found these instead:\n\t%s", name, strings.Join(allNames, "; "))
s.dc[projectName].serviceNames[index] = newName
svc.Name = newName
return newName, nil
}

// A docker-compose service, according to Tilt.
type dcService struct {
Name string

// Contains the name of the service as referenced in the compose file where it was loaded.
ServiceName string

// these are the host machine paths that DC will sync from the local volume into the container
// https://docs.docker.com/compose/compose-file/#volumes
MountedLocalDirs []string
Expand Down Expand Up @@ -354,6 +435,7 @@ func dockerComposeConfigToService(projectName string, svcConfig types.ServiceCon

svc := dcService{
Name: svcConfig.Name,
ServiceName: svcConfig.Name,
ServiceConfig: svcConfig,
MountedLocalDirs: mountedLocalDirs,
ServiceYAML: rawConfig,
Expand Down Expand Up @@ -386,7 +468,7 @@ func parseDCConfig(ctx context.Context, dcc dockercompose.DockerComposeClient, s
return services, nil
}

func (s *tiltfileState) dcServiceToManifest(service *dcService, dcSet dcResourceSet, iTargets []model.ImageTarget) (model.Manifest, error) {
func (s *tiltfileState) dcServiceToManifest(service *dcService, dcSet *dcResourceSet, iTargets []model.ImageTarget) (model.Manifest, error) {
options := service.Options
if options == nil {
options = newDcResourceOptions()
Expand All @@ -395,7 +477,7 @@ func (s *tiltfileState) dcServiceToManifest(service *dcService, dcSet dcResource
dcInfo := model.DockerComposeTarget{
Name: model.TargetName(service.Name),
Spec: v1alpha1.DockerComposeServiceSpec{
Service: service.Name,
Service: service.ServiceName,
Project: dcSet.Project,
},
ServiceYAML: string(service.ServiceYAML),
Expand Down
Loading

0 comments on commit 06b32ae

Please sign in to comment.