feat!: allow to mount multiple artifacts to the same folder in a service. Users will need to replace the Directory.artifac_name field key with Directory.artifac_names

container-engine-lib/lib/backend_impls/docker/docker_manager/docker_manager.go

@@ -1738,7 +1738,7 @@ func (manager *DockerManager) getContainerHostConfig(
  }
 
  // NOTE: Do NOT use PublishAllPorts here!!!! This will work if a Dockerfile doesn't have an EXPOSE directive, but
- // if the Dockerfile *does* have an EXPOSE directive then _only_ the ports with EXPOSE will be published
+ // if the Dockerfile *does* have and EXPOSE directive then _only_ the ports with EXPOSE will be published
  // See also: https://www.ctl.io/developers/blog/post/docker-networking-rules/
 
  containerHostConfigPtr := &container.HostConfig{

container-engine-lib/lib/backend_interface/objects/service/service_config_test.go

@@ -97,9 +97,9 @@ func testFilesArtifactExpansion() *service_directory.FilesArtifactsExpansion {
  "ENV_VAR1": "env_var1_value",
  "ENV_VAR2": "env_var2_value",
  },
- ServiceDirpathsToArtifactIdentifiers: map[string]string{
- "/pahth/number1": "first_identifier",
- "/path/number2":  "second_idenfifier",
+ ServiceDirpathsToArtifactIdentifiers: map[string][]string{
+ "/path/number1": {"first_identifier"},
+ "/path/number2": {"second_identifier"},
  },
  ExpanderDirpathsToServiceDirpaths: map[string]string{
  "/expander/dir1": "/service/dir1",

container-engine-lib/lib/backend_interface/objects/service_directory/files_artifacts_expansion.go

@@ -9,8 +9,8 @@ type FilesArtifactsExpansion struct {
  // its operation
  ExpanderEnvVars map[string]string
 
- // Map of dirpaths on the target service to the artificat identifier that will be mounted at this location
- ServiceDirpathsToArtifactIdentifiers map[string]string
+ // Map of dirpaths on the target service to the artifact identifier that will be mounted at this location
+ ServiceDirpathsToArtifactIdentifiers map[string][]string
 
  // Map of dirpaths that the expander container expects (which the expander will expand into), mapped to
  // dirpaths on the user service container where those same directories should be made available

container-engine-lib/lib/database_accessors/enclave_db/service_registration/repository_test.go

@@ -342,9 +342,9 @@ func testFilesArtifactExpansion() *service_directory.FilesArtifactsExpansion {
  "ENV_VAR1": "env_var1_value",
  "ENV_VAR2": "env_var2_value",
  },
- ServiceDirpathsToArtifactIdentifiers: map[string]string{
- "/pahth/number1": "first_identifier",
- "/path/number2":  "second_idenfifier",
+ ServiceDirpathsToArtifactIdentifiers: map[string][]string{
+ "/path/number1": {"first_identifier"},
+ "/path/number2": {"second_identifier"},
  },
  ExpanderDirpathsToServiceDirpaths: map[string]string{
  "/expander/dir1": "/service/dir1",

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service.go

@@ -209,10 +209,12 @@ func (builtin *AddServiceCapabilities) TryResolveWith(instructionsAreEqual bool,
  // We check if there has been some updates to the files it's mounting. If that's the case, it should be rerun
  filesArtifactsExpansion := builtin.serviceConfig.GetFilesArtifactsExpansion()
  if filesArtifactsExpansion != nil {
- for _, filesArtifactName := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
- if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
- enclaveComponents.AddService(builtin.serviceName, enclave_structure.ComponentIsUpdated)
- return enclave_structure.InstructionIsUpdate
+ for _, filesArtifactNames := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
+ for _, filesArtifactName := range filesArtifactNames {
+ if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
+ enclaveComponents.AddService(builtin.serviceName, enclave_structure.ComponentIsUpdated)
+ return enclave_structure.InstructionIsUpdate
+ }
  }
  }
  }

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_service_shared.go

@@ -87,9 +87,11 @@ func validateSingleService(validatorEnvironment *startosis_validator.ValidatorEn
  return startosis_errors.NewValidationError("There was an error validating '%s' as service with the name '%s' already exists inside the package. Adding two different services with the same name isn't allowed; we recommend prefixing/suffixing the two service names or using two different names entirely.", AddServiceBuiltinName, serviceName)
  }
  if serviceConfig.GetFilesArtifactsExpansion() != nil {
- for _, artifactName := range serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers {
- if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
- return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", AddServiceBuiltinName, artifactName)
+ for _, artifactNames := range serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers {
+ for _, artifactName := range artifactNames {
+ if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
+ return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", AddServiceBuiltinName, artifactName)
+ }
  }
  }
  }

core/server/api_container/server/startosis_engine/kurtosis_instruction/add_service/add_services.go

@@ -259,9 +259,11 @@ func (builtin *AddServicesCapabilities) TryResolveWith(instructionsAreEqual bool
  // Check whether one file as been updated - if yes the instruction will need to be rerun
  filesArtifactsExpansion := serviceConfig.GetFilesArtifactsExpansion()
  if filesArtifactsExpansion != nil {
- for _, filesArtifactName := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
- if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
- atLeastOneFileHasBeenUpdated = true
+ for _, filesArtifactNames := range filesArtifactsExpansion.ServiceDirpathsToArtifactIdentifiers {
+ for _, filesArtifactName := range filesArtifactNames {
+ if enclaveComponents.HasFilesArtifactBeenUpdated(filesArtifactName) {
+ atLeastOneFileHasBeenUpdated = true
+ }
  }
  }
  }

core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_python.go

@@ -192,7 +192,11 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
  if interpretationErr != nil {
  return nil, interpretationErr
  }
- filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(filesArtifactMountDirPaths, builtin.serviceNetwork)
+ multipleFilesArtifactsMountDirPaths := map[string][]string{}
+ for pathToFile, fileArtifactName := range filesArtifactMountDirPaths {
+ multipleFilesArtifactsMountDirPaths[pathToFile] = []string{fileArtifactName}
+ }
+ filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(multipleFilesArtifactsMountDirPaths, builtin.serviceNetwork)
  if interpretationErr != nil {
  return nil, interpretationErr
  }
@@ -233,7 +237,7 @@ func (builtin *RunPythonCapabilities) Interpret(_ string, arguments *builtin_arg
 
 func (builtin *RunPythonCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet, validatorEnvironment *startosis_validator.ValidatorEnvironment) *startosis_errors.ValidationError {
  // TODO add validation for python script
- var serviceDirpathsToArtifactIdentifiers map[string]string
+ var serviceDirpathsToArtifactIdentifiers map[string][]string
  if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
  serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
  }

core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/run_sh.go

@@ -136,7 +136,11 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen
  if interpretationErr != nil {
  return nil, interpretationErr
  }
- filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(filesArtifactMountDirPaths, builtin.serviceNetwork)
+ multipleFilesArtifactsMountDirPaths := map[string][]string{}
+ for pathToFile, fileArtifactName := range filesArtifactMountDirPaths {
+ multipleFilesArtifactsMountDirPaths[pathToFile] = []string{fileArtifactName}
+ }
+ filesArtifactExpansion, interpretationErr = service_config.ConvertFilesArtifactsMounts(multipleFilesArtifactsMountDirPaths, builtin.serviceNetwork)
  if interpretationErr != nil {
  return nil, interpretationErr
  }
@@ -179,7 +183,7 @@ func (builtin *RunShCapabilities) Interpret(_ string, arguments *builtin_argumen
 
 func (builtin *RunShCapabilities) Validate(_ *builtin_argument.ArgumentValuesSet, validatorEnvironment *startosis_validator.ValidatorEnvironment) *startosis_errors.ValidationError {
  // TODO validate bash
- var serviceDirpathsToArtifactIdentifiers map[string]string
+ var serviceDirpathsToArtifactIdentifiers map[string][]string
  if builtin.serviceConfig.GetFilesArtifactsExpansion() != nil {
  serviceDirpathsToArtifactIdentifiers = builtin.serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers
  }

core/server/api_container/server/startosis_engine/kurtosis_instruction/tasks/tasks_shared.go

@@ -134,7 +134,7 @@ func createInterpretationResult(resultUuid string, storeSpecList []*store_spec.S
  return result
 }
 
-func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string]string, imageName string) *startosis_errors.ValidationError {
+func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvironment, storeSpecList []*store_spec.StoreSpec, serviceDirpathsToArtifactIdentifiers map[string][]string, imageName string) *startosis_errors.ValidationError {
  if storeSpecList != nil {
  if err := validatePathIsUniqueWhileCreatingFileArtifact(storeSpecList); err != nil {
  return startosis_errors.WrapWithValidationError(err, "error occurred while validating file paths to copy into file artifact")
@@ -145,9 +145,11 @@ func validateTasksCommon(validatorEnvironment *startosis_validator.ValidatorEnvi
  }
  }
 
- for _, artifactName := range serviceDirpathsToArtifactIdentifiers {
- if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
- return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", RunPythonBuiltinName, artifactName)
+ for _, artifactNames := range serviceDirpathsToArtifactIdentifiers {
+ for _, artifactName := range artifactNames {
+ if validatorEnvironment.DoesArtifactNameExist(artifactName) == startosis_validator.ComponentNotFound {
+ return startosis_errors.NewValidationError("There was an error validating '%s' as artifact name '%s' does not exist", RunPythonBuiltinName, artifactName)
+ }
  }
  }
 

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument/validators.go

@@ -1,6 +1,7 @@
 package builtin_argument
 
 import (
+ "fmt"
  "github.com/kurtosis-tech/kurtosis/container-engine-lib/lib/backend_interface/objects/service"
  "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_errors"
  "go.starlark.net/starlark"
@@ -21,6 +22,25 @@ func NonEmptyString(value starlark.Value, argNameForLogging string) *startosis_e
  return nil
 }
 
+func StringListWithNotEmptyValues(value starlark.Value, argNameForLogging string) *startosis_errors.InterpretationError {
+ starlarkList, ok := value.(*starlark.List)
+ if !ok {
+ return startosis_errors.NewInterpretationError("Value for '%s' was expected to be a starlark.List but was '%s'", argNameForLogging, reflect.TypeOf(value))
+ }
+ iterator := starlarkList.Iterate()
+ defer iterator.Done()
+ var itemValue starlark.Value
+ var index = 0
+ for iterator.Next(&itemValue) {
+ argumentDescription := fmt.Sprintf("element %d in argument '%s'", index, argNameForLogging)
+ if interpretationErr := NonEmptyString(itemValue, argumentDescription); interpretationErr != nil {
+ return interpretationErr
+ }
+ index++
+ }
+ return nil
+}
+
 func Uint64InRange(value starlark.Value, argNameForLogging string, min uint64, max uint64) *startosis_errors.InterpretationError {
  valueInt, ok := value.(starlark.Int)
  if !ok {

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_file_artifact_framework_test.go

@@ -20,17 +20,17 @@ func (suite *KurtosisTypeConstructorTestSuite) TestDirectoryFileArtifact() {
 }
 
 func (t *directoryFileArtifactTestCase) GetStarlarkCode() string {
- return fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.ArtifactNameAttr, testFilesArtifactName1)
+ return fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1)
 }
 
 func (t *directoryFileArtifactTestCase) Assert(typeValue builtin_argument.KurtosisValueType) {
  directoryStarlark, ok := typeValue.(*directory.Directory)
  require.True(t, ok)
 
- artifactName, found, err := directoryStarlark.GetArtifactNameIfSet()
+ artifactNames, found, err := directoryStarlark.GetArtifactNamesIfSet()
  require.Nil(t, err)
  require.True(t, found)
- require.Equal(t, testFilesArtifactName1, artifactName)
+ require.Equal(t, []string{testFilesArtifactName1}, artifactNames)
 
  persistentKey, found, err := directoryStarlark.GetPersistentKeyIfSet()
  require.Nil(t, err)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_multiple_file_artifacts_framework_test.go

@@ -0,0 +1,33 @@
+package test_engine
+
+import (
+ "fmt"
+ "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
+ "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/directory"
+ "github.com/stretchr/testify/require"
+ "testing"
+)
+
+type directoryMultipleFileArtifactsTestCase struct {
+ *testing.T
+}
+
+func (suite *KurtosisTypeConstructorTestSuite) TestDirectoryMultipleFileArtifacts() {
+ suite.run(&directoryMultipleFileArtifactsTestCase{
+ T: suite.T(),
+ })
+}
+
+func (t *directoryMultipleFileArtifactsTestCase) GetStarlarkCode() string {
+ return fmt.Sprintf("%s(%s=[%q, %q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1, testFilesArtifactName2)
+}
+
+func (t *directoryMultipleFileArtifactsTestCase) Assert(typeValue builtin_argument.KurtosisValueType) {
+ directoryStarlark, ok := typeValue.(*directory.Directory)
+ require.True(t, ok)
+
+ artifactNames, found, err := directoryStarlark.GetArtifactNamesIfSet()
+ require.Nil(t, err)
+ require.True(t, found)
+ require.Equal(t, []string{testFilesArtifactName1, testFilesArtifactName2}, artifactNames)
+}

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/directory_persistent_directory_framework_test.go

@@ -26,10 +26,10 @@ func (t *directoryPersistentDirectoryTestCase) Assert(typeValue builtin_argument
  directoryStarlark, ok := typeValue.(*directory.Directory)
  require.True(t, ok)
 
- artifactName, found, err := directoryStarlark.GetArtifactNameIfSet()
+ artifactNames, found, err := directoryStarlark.GetArtifactNamesIfSet()
  require.Nil(t, err)
  require.False(t, found)
- require.Empty(t, artifactName)
+ require.Empty(t, artifactNames)
 
  persistentKey, found, err := directoryStarlark.GetPersistentKeyIfSet()
  require.Nil(t, err)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_backward_compatible_test.go

@@ -82,9 +82,9 @@ func (t *serviceConfigFullTestCaseBackwardCompatible) Assert(typeValue builtin_a
  }
  require.Equal(t, expectedPublicPorts, serviceConfig.GetPublicPorts())
 
- expectedFilesArtifactMap := map[string]string{
- testFilesArtifactPath1: testFilesArtifactName1,
- testFilesArtifactPath2: testFilesArtifactName2,
+ expectedFilesArtifactMap := map[string][]string{
+ testFilesArtifactPath1: {testFilesArtifactName1},
+ testFilesArtifactPath2: {testFilesArtifactName2},
  }
  require.NotNil(t, serviceConfig.GetFilesArtifactsExpansion())
  require.Equal(t, expectedFilesArtifactMap, serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_full_framework_test.go

@@ -35,8 +35,8 @@ func (suite *KurtosisTypeConstructorTestSuite) TestServiceConfigFull() {
 }
 
 func (t *serviceConfigFullTestCase) GetStarlarkCode() string {
- fileArtifact1 := fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.ArtifactNameAttr, testFilesArtifactName1)
- fileArtifact2 := fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.ArtifactNameAttr, testFilesArtifactName2)
+ fileArtifact1 := fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1)
+ fileArtifact2 := fmt.Sprintf("%s(%s=[%q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName2)
  persistentDirectory := fmt.Sprintf("%s(%s=%q)", directory.DirectoryTypeName, directory.PersistentKeyAttr, testPersistentDirectoryKey)
  starlarkCode := fmt.Sprintf("%s(%s=%q, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%s, %s=%q, %s=%d, %s=%d, %s=%d, %s=%d, %s=%s, %s=%v)",
  service_config.ServiceConfigTypeName,
@@ -90,9 +90,9 @@ func (t *serviceConfigFullTestCase) Assert(typeValue builtin_argument.KurtosisVa
  }
  require.Equal(t, expectedPublicPorts, serviceConfig.GetPublicPorts())
 
- expectedFilesArtifactMap := map[string]string{
- testFilesArtifactPath1: testFilesArtifactName1,
- testFilesArtifactPath2: testFilesArtifactName2,
+ expectedFilesArtifactMap := map[string][]string{
+ testFilesArtifactPath1: {testFilesArtifactName1},
+ testFilesArtifactPath2: {testFilesArtifactName2},
  }
  require.NotNil(t, serviceConfig.GetFilesArtifactsExpansion())
  require.Equal(t, expectedFilesArtifactMap, serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers)

core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/test_engine/service_config_multiple_files_in_same_folder_test.go

@@ -0,0 +1,64 @@
+package test_engine
+
+import (
+ "fmt"
+ "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/service_network"
+ "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_starlark_framework/builtin_argument"
+ "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/directory"
+ "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/kurtosis_types/service_config"
+ "github.com/kurtosis-tech/kurtosis/core/server/api_container/server/startosis_engine/startosis_packages"
+ "github.com/stretchr/testify/require"
+ "net"
+ "testing"
+)
+
+type serviceConfigMultipleFilesInSameFolderTestCase struct {
+ *testing.T
+ serviceNetwork *service_network.MockServiceNetwork
+ packageContentProvider *startosis_packages.MockPackageContentProvider
+}
+
+func (suite *KurtosisTypeConstructorTestSuite) TestServiceConfigMultipleFilesInSameFolder() {
+
+ suite.serviceNetwork.EXPECT().GetApiContainerInfo().Times(1).Return(
+ service_network.NewApiContainerInfo(net.IPv4(0, 0, 0, 0), 0, "0.0.0"),
+ )
+
+ suite.run(&serviceConfigMultipleFilesInSameFolderTestCase{
+ T: suite.T(),
+ serviceNetwork: suite.serviceNetwork,
+ packageContentProvider: suite.packageContentProvider,
+ })
+}
+
+func (t *serviceConfigMultipleFilesInSameFolderTestCase) GetStarlarkCode() string {
+ filesArtifactsDirectory := fmt.Sprintf("%s(%s=[%q, %q])", directory.DirectoryTypeName, directory.ArtifactNamesAttr, testFilesArtifactName1, testFilesArtifactName2)
+ starlarkCode := fmt.Sprintf("%s(%s=%q, %s=%s)",
+ service_config.ServiceConfigTypeName,
+ service_config.ImageAttr, testContainerImageName,
+ service_config.FilesAttr, fmt.Sprintf("{%q: %s}", testFilesArtifactPath1, filesArtifactsDirectory),
+ )
+
+ return starlarkCode
+}
+
+func (t *serviceConfigMultipleFilesInSameFolderTestCase) Assert(typeValue builtin_argument.KurtosisValueType) {
+ serviceConfigStarlark, ok := typeValue.(*service_config.ServiceConfig)
+ require.True(t, ok)
+
+ serviceConfig, err := serviceConfigStarlark.ToKurtosisType(
+ t.serviceNetwork,
+ testModulePackageId,
+ testModuleMainFileLocator,
+ t.packageContentProvider,
+ testNoPackageReplaceOptions)
+ require.Nil(t, err)
+
+ require.Equal(t, testContainerImageName, serviceConfig.GetContainerImageName())
+
+ expectedFilesArtifactMap := map[string][]string{
+ testFilesArtifactPath1: {testFilesArtifactName1, testFilesArtifactName2},
+ }
+ require.NotNil(t, serviceConfig.GetFilesArtifactsExpansion())
+ require.Equal(t, expectedFilesArtifactMap, serviceConfig.GetFilesArtifactsExpansion().ServiceDirpathsToArtifactIdentifiers)
+}