From d29f417bbea24d13cb6569a36ad8a926172c1967 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Wed, 6 Nov 2024 09:17:14 +0100 Subject: [PATCH 01/11] feat: added flags to ignore --- internal/pkg/model/objectstate.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/pkg/model/objectstate.go b/internal/pkg/model/objectstate.go index bcb2ea0a98..6733ff545d 100644 --- a/internal/pkg/model/objectstate.go +++ b/internal/pkg/model/objectstate.go @@ -43,12 +43,14 @@ type ConfigState struct { *ConfigManifest Remote *Config Local *Config + Ignore bool } type ConfigRowState struct { *ConfigRowManifest Remote *ConfigRow Local *ConfigRow + Ignore bool } // ToAPIObjectKey ... From d4e2828f81343f7db09a8e7242225a59a47752a0 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Wed, 6 Nov 2024 09:18:35 +0100 Subject: [PATCH 02/11] feat: added necessary methods for config and config rows --- internal/pkg/state/registry/registry.go | 42 +++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/internal/pkg/state/registry/registry.go b/internal/pkg/state/registry/registry.go index 4f15869214..4b2a2a76f9 100644 --- a/internal/pkg/state/registry/registry.go +++ b/internal/pkg/state/registry/registry.go @@ -150,6 +150,27 @@ func (s *Registry) Configs() (configs []*ConfigState) { return configs } +func (s *Registry) IgnoreConfig(ignoreID string) { + for _, object := range s.All() { + if v, ok := object.(*ConfigState); ok { + if v.ID.String() == ignoreID { + v.Ignore = true + } + } + } +} + +func (s *Registry) IgnoredConfigs() (configs []*ConfigState) { + for _, object := range s.All() { + if v, ok := object.(*ConfigState); ok { + if v.Ignore { + configs = append(configs, v) + } + } + } + return configs +} + func (s *Registry) ConfigsFrom(branch BranchKey) (configs []*ConfigState) { for _, object := range s.All() { if v, ok := object.(*ConfigState); ok { @@ -171,6 +192,27 @@ func (s *Registry) ConfigRows() (rows []*ConfigRowState) { return rows } +func (s *Registry) IgnoreConfigRow(ignoreID string) { + for _, object := range s.All() { + if v, ok := object.(*ConfigRowState); ok { + if v.ID.String() == ignoreID { + v.Ignore = true + } + } + } +} + +func (s *Registry) IgnoredConfigRows() (rows []*ConfigRowState) { + for _, object := range s.All() { + if v, ok := object.(*ConfigRowState); ok { + if v.Ignore { + rows = append(rows, v) + } + } + } + return rows +} + func (s *Registry) ConfigRowsFrom(config ConfigKey) (rows []*ConfigRowState) { for _, object := range s.All() { if v, ok := object.(*ConfigRowState); ok { From 3b799f5d89ad6ef008ec9100c989ced55eadc501 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Fri, 8 Nov 2024 16:32:59 +0100 Subject: [PATCH 03/11] feat: implementation --- internal/pkg/state/registry/registry.go | 77 ++++++++++++++++++- .../operation/project/sync/pull/operation.go | 22 ++++++ 2 files changed, 95 insertions(+), 4 deletions(-) diff --git a/internal/pkg/state/registry/registry.go b/internal/pkg/state/registry/registry.go index 4b2a2a76f9..30030084b0 100644 --- a/internal/pkg/state/registry/registry.go +++ b/internal/pkg/state/registry/registry.go @@ -2,10 +2,12 @@ package registry import ( "context" + "strings" "sync" "github.com/keboola/go-utils/pkg/orderedmap" + "github.com/keboola/keboola-as-code/internal/pkg/filesystem" "github.com/keboola/keboola-as-code/internal/pkg/filesystem/knownpaths" . "github.com/keboola/keboola-as-code/internal/pkg/model" "github.com/keboola/keboola-as-code/internal/pkg/naming" @@ -14,6 +16,8 @@ import ( type pathsRO = knownpaths.PathsReadOnly +const KBCIgnoreFilePath = ".keboola/kbc_ignore" + type Registry struct { *pathsRO paths *knownpaths.Paths @@ -150,11 +154,19 @@ func (s *Registry) Configs() (configs []*ConfigState) { return configs } -func (s *Registry) IgnoreConfig(ignoreID string) { +func (s *Registry) IgnoreConfig(ignoreID string, componentID string) { for _, object := range s.All() { if v, ok := object.(*ConfigState); ok { - if v.ID.String() == ignoreID { + if v.ID.String() == ignoreID && v.ComponentID.String() == componentID { + // ignore configuration v.Ignore = true + + // ignore rows of the configuration + if len(s.ConfigRowsFrom(v.ConfigKey)) > 0 { + for _, configRowState := range s.ConfigRowsFrom(v.ConfigKey) { + configRowState.Ignore = true + } + } } } } @@ -192,10 +204,10 @@ func (s *Registry) ConfigRows() (rows []*ConfigRowState) { return rows } -func (s *Registry) IgnoreConfigRow(ignoreID string) { +func (s *Registry) IgnoreConfigRow(configID, rowID string) { for _, object := range s.All() { if v, ok := object.(*ConfigRowState); ok { - if v.ID.String() == ignoreID { + if v.ConfigID.String() == configID && v.ID.String() == rowID { v.Ignore = true } } @@ -225,6 +237,19 @@ func (s *Registry) ConfigRowsFrom(config ConfigKey) (rows []*ConfigRowState) { return rows } +func (s *Registry) SetIgnoredConfigsOrRows(ctx context.Context, fs filesystem.Fs, path string) error { + content, err := fs.ReadFile(ctx, filesystem.NewFileDef(path)) + if err != nil { + return err + } + + if content.Content == "" { + return nil + } + + return s.applyIgnoredPatterns(content.Content) +} + func (s *Registry) GetPath(key Key) (AbsPath, bool) { objectState, found := s.Get(key) if !found { @@ -303,3 +328,47 @@ func (s *Registry) GetOrCreateFrom(manifest ObjectManifest) (ObjectState, error) return s.CreateFrom(manifest) } + +// applyIgnorePattern applies a single ignore pattern, marking the appropriate config or row as ignored. +func (s *Registry) applyIgnorePattern(pattern string) error { + parts := strings.Split(pattern, "/") + + switch len(parts) { + case 2: + // Ignore config by ID and name. + configID, componentID := parts[1], parts[0] + s.IgnoreConfig(configID, componentID) + case 3: + // Ignore specific config row. + configID, rowID := parts[1], parts[2] + s.IgnoreConfigRow(configID, rowID) + default: + return errors.Errorf("invalid ignore pattern format: %s", pattern) + } + + return nil +} + +// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows. +func (s *Registry) applyIgnoredPatterns(content string) error { + for _, pattern := range parseIgnoredPatterns(content) { + if err := s.applyIgnorePattern(pattern); err != nil { + continue + } + } + return nil +} + +func parseIgnoredPatterns(content string) []string { + var ignorePatterns []string + lines := strings.Split(content, "\n") + for _, line := range lines { + trimmedLine := strings.TrimSpace(line) + // Skip empty lines and comments + if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + ignorePatterns = append(ignorePatterns, trimmedLine) + } + } + + return ignorePatterns +} diff --git a/pkg/lib/operation/project/sync/pull/operation.go b/pkg/lib/operation/project/sync/pull/operation.go index c828429e25..5e179f52e6 100644 --- a/pkg/lib/operation/project/sync/pull/operation.go +++ b/pkg/lib/operation/project/sync/pull/operation.go @@ -11,6 +11,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/plan/pull" "github.com/keboola/keboola-as-code/internal/pkg/project" "github.com/keboola/keboola-as-code/internal/pkg/project/cachefile" + "github.com/keboola/keboola-as-code/internal/pkg/state/registry" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" saveManifest "github.com/keboola/keboola-as-code/pkg/lib/operation/project/local/manifest/save" @@ -49,6 +50,13 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen logger := d.Logger() + if projectState.Fs().Exists(ctx, registry.KBCIgnoreFilePath) { + if err := projectState.SetIgnoredConfigsOrRows(ctx, projectState.Fs(), registry.KBCIgnoreFilePath); err != nil { + return err + } + ignoreConfigsAndRows(projectState) + } + // Diff results, err := createDiff.Run(ctx, createDiff.Options{Objects: projectState}, d, diff.WithIgnoreBranchName(projectState.ProjectManifest().AllowTargetENV())) if err != nil { @@ -118,3 +126,17 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen return nil } + +func ignoreConfigsAndRows(projectState *project.State) { + if len(projectState.IgnoredConfigRows()) > 0 { + for _, v := range projectState.IgnoredConfigRows() { + v.SetRemoteState(nil) + } + } + + if len(projectState.IgnoredConfigs()) > 0 { + for _, v := range projectState.IgnoredConfigs() { + v.SetRemoteState(nil) + } + } +} From 47417fbeba23bd30fa620e71d868f95342d8d037 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Fri, 8 Nov 2024 14:55:51 +0100 Subject: [PATCH 04/11] test: unit tests --- internal/pkg/state/registry/registry_test.go | 141 +++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/internal/pkg/state/registry/registry_test.go b/internal/pkg/state/registry/registry_test.go index 56b4039e16..55d542552d 100644 --- a/internal/pkg/state/registry/registry_test.go +++ b/internal/pkg/state/registry/registry_test.go @@ -2,6 +2,8 @@ package registry import ( "context" + "fmt" + "github.com/stretchr/testify/require" "testing" "github.com/stretchr/testify/assert" @@ -175,6 +177,22 @@ func TestStateTrackRecordInvalid(t *testing.T) { assert.Empty(t, s.UntrackedPaths()) } +func TestRegistry_SetIgnoredConfigsOrRows(t *testing.T) { + t.Parallel() + ctx := context.Background() + registry := newTestState(t, knownpaths.NewNop(ctx)) + fs := aferofs.NewMemoryFs() + require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile(`foo/bar1`, "keboola.bar/678/34\nkeboola.foo/345"))) + + require.NoError(t, registry.SetIgnoredConfigsOrRows(ctx, fs, "foo/bar1")) + + assert.Len(t, registry.IgnoredConfigRows(), 1) + assert.Len(t, registry.IgnoredConfigs(), 1) + assert.Equal(t, registry.IgnoredConfigRows()[0].ID.String(), "34") + assert.Equal(t, registry.IgnoredConfigs()[0].ID.String(), "345") + +} + func TestRegistry_GetPath(t *testing.T) { t.Parallel() ctx := context.Background() @@ -301,3 +319,126 @@ func newTestState(t *testing.T, paths *knownpaths.Paths) *Registry { return registry } + +func Test_parseIgnoredPatterns(t *testing.T) { + t.Parallel() + type args struct { + content string + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "with comments", + args: args{ + content: "##dataapps\nkeboola.data-apps/12345\n##dataapps2\nkeboola.data-apps/123/rowId123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123/rowId123", + "keboola.data-apps/12", + }, + }, + { + name: "with empty lines", + args: args{ + content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123", + "keboola.data-apps/12", + }, + }, + { + name: "empty file", + args: args{ + content: "\n", + }, + want: nil, + }, + { + name: "error", + args: args{ + content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123", + "keboola.data-apps/12", + }, + }, + { + name: "empty file", + args: args{ + content: "\n", + }, + want: nil, + }, + } + + for _, tt := range tests { + got := parseIgnoredPatterns(tt.args.content) + assert.Equal(t, tt.want, got) + } +} + +func TestRegistry_applyIgnoredPatterns(t *testing.T) { + t.Parallel() + ctx := context.Background() + registry := newTestState(t, knownpaths.NewNop(ctx)) + + type args struct { + pattern string + } + + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + }{ + { + name: "empty patterns", + args: args{ + pattern: "", + }, + wantErr: assert.Error, + }, + { + name: "wrong pattern", + args: args{ + pattern: "wrong pattern", + }, + wantErr: assert.Error, + }, + { + name: "too long pattern", + args: args{ + pattern: "keboola.bar/687/1234/1234", + }, + wantErr: assert.Error, + }, + { + name: "short pattern", + args: args{ + pattern: "keboola.bar", + }, + wantErr: assert.Error, + }, + { + name: "correct pattern", + args: args{ + pattern: "keboola.bar/687", + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tt.wantErr(t, registry.applyIgnorePattern(tt.args.pattern), fmt.Sprintf("applyIgnoredPatterns(%v)", tt.args.pattern)) + }) + } +} From 4a82ab69b9d5d9b2d6be9046b68246e563f35181 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Fri, 8 Nov 2024 15:24:00 +0100 Subject: [PATCH 05/11] test: e2e test --- internal/pkg/state/registry/registry_test.go | 3 +- .../allow-target-env/pull-ignore-configs/args | 1 + .../allow-target-env/pull-ignore-configs/env | 2 + .../pull-ignore-configs/expected-code | 1 + .../pull-ignore-configs/expected-stderr | 0 .../pull-ignore-configs/expected-stdout | 7 + .../pull-ignore-configs/in/.gitkeep | 0 .../in/.keboola/kbc_ignore | 2 + .../in/.keboola/manifest.json | 41 ++++ .../pull-ignore-configs/in/description.md | 0 .../in/main/description.md | 0 .../pull-ignore-configs/in/main/meta.json | 4 + .../pull-ignore-configs/initial-state.json | 18 ++ .../pull-ignore-configs/out/.gitkeep | 0 .../out/.keboola/kbc_ignore | 2 + .../out/.keboola/manifest.json | 61 ++++++ .../out/.keboola/project.json | 9 + .../pull-ignore-configs/out/description.md | 0 .../keboola.data-apps/data-app/config.json | 3 + .../keboola.data-apps/data-app/description.md | 1 + .../app/keboola.data-apps/data-app/meta.json | 4 + .../out/main/description.md | 0 .../keboola.ex-db-mysql/with-rows/config.json | 7 + .../with-rows/description.md | 1 + .../keboola.ex-db-mysql/with-rows/meta.json | 4 + .../with-rows/rows/test-view/config.json | 5 + .../with-rows/rows/test-view/description.md | 1 + .../with-rows/rows/test-view/meta.json | 4 + .../pull-ignore-configs/out/main/meta.json | 4 + test/cli/pull/ignore-configurations/args | 1 + .../pull/ignore-configurations/expected-code | 1 + .../ignore-configurations/expected-stderr | 0 .../ignore-configurations/expected-stdout | 31 +++ .../pull/ignore-configurations/in/.gitkeep | 0 .../in/.keboola/kbc_ignore | 3 + .../in/.keboola/manifest.json | 36 ++++ .../ignore-configurations/in/description.md | 0 .../ignore-configurations/initial-state.json | 52 +++++ .../pull/ignore-configurations/out/.gitkeep | 0 .../out/.keboola/kbc_ignore | 3 + .../out/.keboola/manifest.json | 180 ++++++++++++++++++ .../out/.keboola/project.json | 9 + .../codes/code-with-variables/code.py | 3 + .../codes/code-with-variables/config.json | 1 + .../codes/code-with-variables/description.md | 1 + .../codes/code-with-variables/meta.json | 3 + .../code-with-variables/variables/config.json | 12 ++ .../variables/description.md | 1 + .../code-with-variables/variables/meta.json | 4 + .../codes/my-code-1/code.py | 3 + .../codes/my-code-1/config.json | 1 + .../codes/my-code-1/description.md | 1 + .../codes/my-code-1/meta.json | 3 + .../codes/my-code-2/code.py | 3 + .../codes/my-code-2/config.json | 1 + .../codes/my-code-2/description.md | 1 + .../codes/my-code-2/meta.json | 3 + .../config.json | 3 + .../description.md | 1 + .../meta.json | 4 + .../out/bar/description.md | 0 .../ex-generic-v2/disabled/config.json | 7 + .../ex-generic-v2/disabled/description.md | 1 + .../ex-generic-v2/disabled/meta.json | 4 + .../extractor/ex-generic-v2/empty/config.json | 1 + .../ex-generic-v2/empty/description.md | 1 + .../extractor/ex-generic-v2/empty/meta.json | 4 + .../keboola.ex-db-mysql/with-rows/config.json | 7 + .../with-rows/description.md | 1 + .../keboola.ex-db-mysql/with-rows/meta.json | 4 + .../with-rows/rows/disabled/config.json | 5 + .../with-rows/rows/disabled/description.md | 1 + .../with-rows/rows/disabled/meta.json | 4 + .../with-rows/rows/test-view/config.json | 5 + .../with-rows/rows/test-view/description.md | 1 + .../with-rows/rows/test-view/meta.json | 4 + .../ignore-configurations/out/bar/meta.json | 4 + .../blocks/001-block-1/001-code-x/code.py | 1 + .../blocks/001-block-1/001-code-x/meta.json | 3 + .../blocks/001-block-1/002-code-y/code.py | 1 + .../blocks/001-block-1/002-code-y/meta.json | 3 + .../blocks/001-block-1/meta.json | 3 + .../blocks/002-block-2/001-code-z/code.py | 13 ++ .../blocks/002-block-2/001-code-z/meta.json | 3 + .../blocks/002-block-2/meta.json | 3 + .../python-transformation/config.json | 7 + .../python-transformation/description.md | 1 + .../python-transformation/meta.json | 4 + .../001-block-1/001-shared-code-used/code.py | 1 + .../001-shared-code-used/meta.json | 3 + .../blocks/001-block-1/meta.json | 3 + .../config.json | 4 + .../description.md | 1 + .../transformation-with-shared-code/meta.json | 4 + .../variables/config.json | 12 ++ .../variables/description.md | 1 + .../variables/meta.json | 4 + .../variables/values/default/config.json | 12 ++ .../variables/values/default/description.md | 1 + .../variables/values/default/meta.json | 4 + .../blocks/001-block-1/001-code-x/code.sql | 11 ++ .../blocks/001-block-1/001-code-x/meta.json | 3 + .../blocks/001-block-1/002-code-y/code.sql | 2 + .../blocks/001-block-1/002-code-y/meta.json | 3 + .../blocks/001-block-1/meta.json | 3 + .../002-block-2-my-name/001-code-z/code.sql | 2 + .../002-block-2-my-name/001-code-z/meta.json | 3 + .../blocks/002-block-2-my-name/meta.json | 3 + .../snowflake-transformation/config.json | 3 + .../snowflake-transformation/description.md | 1 + .../snowflake-transformation/meta.json | 4 + .../ignore-configurations/out/description.md | 0 .../out/foo/description.md | 0 .../extractor/ex-generic-v2/empty/config.json | 1 + .../ex-generic-v2/empty/description.md | 1 + .../extractor/ex-generic-v2/empty/meta.json | 4 + .../ex-generic-v2/without-rows/config.json | 7 + .../ex-generic-v2/without-rows/description.md | 1 + .../ex-generic-v2/without-rows/meta.json | 4 + .../ignore-configurations/out/foo/meta.json | 4 + .../out/main/description.md | 0 .../extractor/ex-generic-v2/empty/config.json | 1 + .../ex-generic-v2/empty/description.md | 1 + .../extractor/ex-generic-v2/empty/meta.json | 4 + .../ignore-configurations/out/main/meta.json | 4 + 125 files changed, 764 insertions(+), 2 deletions(-) create mode 100644 test/cli/allow-target-env/pull-ignore-configs/args create mode 100644 test/cli/allow-target-env/pull-ignore-configs/env create mode 100644 test/cli/allow-target-env/pull-ignore-configs/expected-code create mode 100644 test/cli/allow-target-env/pull-ignore-configs/expected-stderr create mode 100644 test/cli/allow-target-env/pull-ignore-configs/expected-stdout create mode 100644 test/cli/allow-target-env/pull-ignore-configs/in/.gitkeep create mode 100644 test/cli/allow-target-env/pull-ignore-configs/in/.keboola/kbc_ignore create mode 100644 test/cli/allow-target-env/pull-ignore-configs/in/.keboola/manifest.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/in/description.md create mode 100644 test/cli/allow-target-env/pull-ignore-configs/in/main/description.md create mode 100644 test/cli/allow-target-env/pull-ignore-configs/in/main/meta.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/initial-state.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/.gitkeep create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/.keboola/kbc_ignore create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/.keboola/manifest.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/.keboola/project.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/description.md create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/config.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/description.md create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/meta.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/description.md create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/config.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/description.md create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/meta.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json create mode 100644 test/cli/allow-target-env/pull-ignore-configs/out/main/meta.json create mode 100644 test/cli/pull/ignore-configurations/args create mode 100644 test/cli/pull/ignore-configurations/expected-code create mode 100644 test/cli/pull/ignore-configurations/expected-stderr create mode 100644 test/cli/pull/ignore-configurations/expected-stdout create mode 100644 test/cli/pull/ignore-configurations/in/.gitkeep create mode 100644 test/cli/pull/ignore-configurations/in/.keboola/kbc_ignore create mode 100644 test/cli/pull/ignore-configurations/in/.keboola/manifest.json create mode 100644 test/cli/pull/ignore-configurations/in/description.md create mode 100644 test/cli/pull/ignore-configurations/initial-state.json create mode 100644 test/cli/pull/ignore-configurations/out/.gitkeep create mode 100644 test/cli/pull/ignore-configurations/out/.keboola/kbc_ignore create mode 100644 test/cli/pull/ignore-configurations/out/.keboola/manifest.json create mode 100644 test/cli/pull/ignore-configurations/out/.keboola/project.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/code.py create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/code.py create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/code.py create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/code.py create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/code.py create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/code.py create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/code.py create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/code.sql create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/code.sql create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/code.sql create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/config.json create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/description.md create mode 100644 test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/description.md create mode 100644 test/cli/pull/ignore-configurations/out/foo/description.md create mode 100644 test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/config.json create mode 100644 test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/description.md create mode 100644 test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/config.json create mode 100644 test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/description.md create mode 100644 test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/foo/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/main/description.md create mode 100644 test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/config.json create mode 100644 test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/description.md create mode 100644 test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/meta.json create mode 100644 test/cli/pull/ignore-configurations/out/main/meta.json diff --git a/internal/pkg/state/registry/registry_test.go b/internal/pkg/state/registry/registry_test.go index 55d542552d..602331f9f3 100644 --- a/internal/pkg/state/registry/registry_test.go +++ b/internal/pkg/state/registry/registry_test.go @@ -3,10 +3,10 @@ package registry import ( "context" "fmt" - "github.com/stretchr/testify/require" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/keboola/keboola-as-code/internal/pkg/filesystem" "github.com/keboola/keboola-as-code/internal/pkg/filesystem/aferofs" @@ -190,7 +190,6 @@ func TestRegistry_SetIgnoredConfigsOrRows(t *testing.T) { assert.Len(t, registry.IgnoredConfigs(), 1) assert.Equal(t, registry.IgnoredConfigRows()[0].ID.String(), "34") assert.Equal(t, registry.IgnoredConfigs()[0].ID.String(), "345") - } func TestRegistry_GetPath(t *testing.T) { diff --git a/test/cli/allow-target-env/pull-ignore-configs/args b/test/cli/allow-target-env/pull-ignore-configs/args new file mode 100644 index 0000000000..1b886e0aed --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/args @@ -0,0 +1 @@ +pull --storage-api-token %%TEST_KBC_STORAGE_API_TOKEN%% diff --git a/test/cli/allow-target-env/pull-ignore-configs/env b/test/cli/allow-target-env/pull-ignore-configs/env new file mode 100644 index 0000000000..f92e12fb02 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/env @@ -0,0 +1,2 @@ +KBC_PROJECT_ID=%%TEST_KBC_PROJECT_ID%% +KBC_BRANCH_ID=%%TEST_BRANCH_MAIN_ID%% diff --git a/test/cli/allow-target-env/pull-ignore-configs/expected-code b/test/cli/allow-target-env/pull-ignore-configs/expected-code new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/expected-code @@ -0,0 +1 @@ +0 diff --git a/test/cli/allow-target-env/pull-ignore-configs/expected-stderr b/test/cli/allow-target-env/pull-ignore-configs/expected-stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/allow-target-env/pull-ignore-configs/expected-stdout b/test/cli/allow-target-env/pull-ignore-configs/expected-stdout new file mode 100644 index 0000000000..acc448dd5c --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/expected-stdout @@ -0,0 +1,7 @@ +Overriding the project ID by the environment variable KBC_PROJECT_ID=%s +Overriding the branch ID by the environment variable KBC_BRANCH_ID=%s +Plan for "pull" operation: + + C main/app/keboola.data-apps/data-app + + C main/extractor/keboola.ex-db-mysql/with-rows + + R main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view +Pull done. diff --git a/test/cli/allow-target-env/pull-ignore-configs/in/.gitkeep b/test/cli/allow-target-env/pull-ignore-configs/in/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/allow-target-env/pull-ignore-configs/in/.keboola/kbc_ignore b/test/cli/allow-target-env/pull-ignore-configs/in/.keboola/kbc_ignore new file mode 100644 index 0000000000..377d3c75b0 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/in/.keboola/kbc_ignore @@ -0,0 +1,2 @@ +keboola.ex-db-mysql/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ID%%/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ROW_USERS_ID%% +keboola.ex-db-mysql/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ID%%/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ROW_DISABLED_ID%% diff --git a/test/cli/allow-target-env/pull-ignore-configs/in/.keboola/manifest.json b/test/cli/allow-target-env/pull-ignore-configs/in/.keboola/manifest.json new file mode 100644 index 0000000000..36ad2bd908 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/in/.keboola/manifest.json @@ -0,0 +1,41 @@ +{ + "version": 2, + "project": { + "id": 1, + "apiHost": "%%TEST_KBC_STORAGE_API_HOST%%" + }, + "allowTargetEnv": true, + "sortBy": "path", + "naming": { + "branch": "{branch_name}", + "config": "{component_type}/{component_id}/{config_name}", + "configRow": "rows/{config_row_name}", + "schedulerConfig": "schedules/{config_name}", + "sharedCodeConfig": "_shared/{target_component_id}", + "sharedCodeConfigRow": "codes/{config_row_name}", + "variablesConfig": "variables", + "variablesValuesRow": "values/{config_row_name}", + "dataAppConfig": "app/{component_id}/{config_name}" + }, + "allowedBranches": [ + "2" + ], + "ignoredComponents": [], + "templates": { + "repositories": [ + { + "type": "git", + "name": "keboola", + "url": "https://github.com/keboola/keboola-as-code-templates.git", + "ref": "main" + } + ] + }, + "branches": [ + { + "id": 2, + "path": "main" + } + ], + "configurations": [] +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/in/description.md b/test/cli/allow-target-env/pull-ignore-configs/in/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/allow-target-env/pull-ignore-configs/in/main/description.md b/test/cli/allow-target-env/pull-ignore-configs/in/main/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/allow-target-env/pull-ignore-configs/in/main/meta.json b/test/cli/allow-target-env/pull-ignore-configs/in/main/meta.json new file mode 100644 index 0000000000..154a3ce050 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/in/main/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Main", + "isDefault": true +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/initial-state.json b/test/cli/allow-target-env/pull-ignore-configs/initial-state.json new file mode 100644 index 0000000000..58b091aa2f --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/initial-state.json @@ -0,0 +1,18 @@ +{ + "backend": { + "type": "snowflake" + }, + "allBranchesConfigs": [ ], + "branches": [ + { + "branch": { + "name": "Main", + "isDefault": true + }, + "configs": [ + "with-rows", + "data-app" + ] + } + ] +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/.gitkeep b/test/cli/allow-target-env/pull-ignore-configs/out/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/kbc_ignore b/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/kbc_ignore new file mode 100644 index 0000000000..377d3c75b0 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/kbc_ignore @@ -0,0 +1,2 @@ +keboola.ex-db-mysql/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ID%%/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ROW_USERS_ID%% +keboola.ex-db-mysql/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ID%%/%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ROW_DISABLED_ID%% diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/manifest.json b/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/manifest.json new file mode 100644 index 0000000000..0f95c06e6b --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/manifest.json @@ -0,0 +1,61 @@ +{ + "version": 2, + "project": { + "id": 1, + "apiHost": "%%TEST_KBC_STORAGE_API_HOST%%" + }, + "allowTargetEnv": true, + "sortBy": "path", + "naming": { + "branch": "{branch_name}", + "config": "{component_type}/{component_id}/{config_name}", + "configRow": "rows/{config_row_name}", + "schedulerConfig": "schedules/{config_name}", + "sharedCodeConfig": "_shared/{target_component_id}", + "sharedCodeConfigRow": "codes/{config_row_name}", + "variablesConfig": "variables", + "variablesValuesRow": "values/{config_row_name}", + "dataAppConfig": "app/{component_id}/{config_name}" + }, + "allowedBranches": [ + "2" + ], + "ignoredComponents": [], + "templates": { + "repositories": [ + { + "type": "git", + "name": "keboola", + "url": "https://github.com/keboola/keboola-as-code-templates.git", + "ref": "main" + } + ] + }, + "branches": [ + { + "id": 2, + "path": "main" + } + ], + "configurations": [ + { + "branchId": 2, + "componentId": "keboola.data-apps", + "id": "%%TEST_BRANCH_MAIN_CONFIG_DATA_APP_ID%%", + "path": "app/keboola.data-apps/data-app", + "rows": [] + }, + { + "branchId": 2, + "componentId": "keboola.ex-db-mysql", + "id": "%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ID%%", + "path": "extractor/keboola.ex-db-mysql/with-rows", + "rows": [ + { + "id": "%%TEST_BRANCH_MAIN_CONFIG_WITH_ROWS_ROW_TEST_VIEW_ID%%", + "path": "rows/test-view" + } + ] + } + ] +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/project.json b/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/project.json new file mode 100644 index 0000000000..2f8539cf19 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/.keboola/project.json @@ -0,0 +1,9 @@ +{ + "backends": [ + %A + ], + "features": [ + %A + ], + "defaultBranchId": %A +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/description.md b/test/cli/allow-target-env/pull-ignore-configs/out/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/config.json b/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/config.json new file mode 100644 index 0000000000..c8c4105eb5 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/config.json @@ -0,0 +1,3 @@ +{ + "foo": "bar" +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/description.md b/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/meta.json b/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/meta.json new file mode 100644 index 0000000000..92fac23796 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/app/keboola.data-apps/data-app/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Data App", + "isDisabled": false +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/description.md b/test/cli/allow-target-env/pull-ignore-configs/out/main/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/config.json b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/config.json new file mode 100644 index 0000000000..efebab79e7 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/config.json @@ -0,0 +1,7 @@ +{ + "parameters": { + "db": { + "host": "mysql.example.com" + } + } +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/description.md b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/meta.json b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/meta.json new file mode 100644 index 0000000000..b6b602f6ff --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/meta.json @@ -0,0 +1,4 @@ +{ + "name": "with-rows", + "isDisabled": false +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json new file mode 100644 index 0000000000..45fb08048a --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json @@ -0,0 +1,5 @@ +{ + "parameters": { + "incremental": false + } +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json new file mode 100644 index 0000000000..221e9ae8ed --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json @@ -0,0 +1,4 @@ +{ + "name": "test_view", + "isDisabled": false +} diff --git a/test/cli/allow-target-env/pull-ignore-configs/out/main/meta.json b/test/cli/allow-target-env/pull-ignore-configs/out/main/meta.json new file mode 100644 index 0000000000..154a3ce050 --- /dev/null +++ b/test/cli/allow-target-env/pull-ignore-configs/out/main/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Main", + "isDefault": true +} diff --git a/test/cli/pull/ignore-configurations/args b/test/cli/pull/ignore-configurations/args new file mode 100644 index 0000000000..9e65aa6fee --- /dev/null +++ b/test/cli/pull/ignore-configurations/args @@ -0,0 +1 @@ +pull --verbose --storage-api-token %%TEST_KBC_STORAGE_API_TOKEN%% diff --git a/test/cli/pull/ignore-configurations/expected-code b/test/cli/pull/ignore-configurations/expected-code new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/test/cli/pull/ignore-configurations/expected-code @@ -0,0 +1 @@ +0 diff --git a/test/cli/pull/ignore-configurations/expected-stderr b/test/cli/pull/ignore-configurations/expected-stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/expected-stdout b/test/cli/pull/ignore-configurations/expected-stdout new file mode 100644 index 0000000000..e6ad50adf3 --- /dev/null +++ b/test/cli/pull/ignore-configurations/expected-stdout @@ -0,0 +1,31 @@ +%A +DEBUG Project state has been successfully loaded. +%A +Plan for "pull" operation: + + B bar + + C bar/_shared/keboola.python-transformation-v2 + + R bar/_shared/keboola.python-transformation-v2/codes/code-with-variables + + C bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables + + R bar/_shared/keboola.python-transformation-v2/codes/my-code-1 + + R bar/_shared/keboola.python-transformation-v2/codes/my-code-2 + + C bar/extractor/ex-generic-v2/disabled + + C bar/extractor/ex-generic-v2/empty + + C bar/extractor/keboola.ex-db-mysql/with-rows + + R bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled + + R bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view + + C bar/transformation/keboola.python-transformation-v2/python-transformation + + C bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code + + C bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables + + R bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default + + C bar/transformation/keboola.snowflake-transformation/snowflake-transformation + + B foo + + C foo/extractor/ex-generic-v2/empty + + C foo/extractor/ex-generic-v2/without-rows + + B main + + C main/extractor/ex-generic-v2/empty +%A +DEBUG Saved ".keboola/manifest.json" +%A +INFO Pull done. +%A +DEBUG Sent "sync-pull" successful event id: "%s" diff --git a/test/cli/pull/ignore-configurations/in/.gitkeep b/test/cli/pull/ignore-configurations/in/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/in/.keboola/kbc_ignore b/test/cli/pull/ignore-configurations/in/.keboola/kbc_ignore new file mode 100644 index 0000000000..46de93ee5b --- /dev/null +++ b/test/cli/pull/ignore-configurations/in/.keboola/kbc_ignore @@ -0,0 +1,3 @@ +keboola.data-apps/%%TEST_BRANCH_MAIN_CONFIG_DATA_APP_ID%% +keboola.ex-db-mysql/%%TEST_BRANCH_BAR_CONFIG_WITH_ROWS_ID%%/%%TEST_BRANCH_BAR_CONFIG_WITH_ROWS_ROW_USERS_ID%% +keboola.ex-db-mysql/%%TEST_BRANCH_FOO_CONFIG_WITH_ROWS_ID%% diff --git a/test/cli/pull/ignore-configurations/in/.keboola/manifest.json b/test/cli/pull/ignore-configurations/in/.keboola/manifest.json new file mode 100644 index 0000000000..f6e46b5ff8 --- /dev/null +++ b/test/cli/pull/ignore-configurations/in/.keboola/manifest.json @@ -0,0 +1,36 @@ +{ + "version": 2, + "project": { + "id": %%TEST_KBC_PROJECT_ID%%, + "apiHost": "%%TEST_KBC_STORAGE_API_HOST%%" + }, + "allowTargetEnv": false, + "sortBy": "path", + "naming": { + "branch": "{branch_name}", + "config": "{component_type}/{component_id}/{config_name}", + "configRow": "rows/{config_row_name}", + "schedulerConfig": "schedules/{config_name}", + "sharedCodeConfig": "_shared/{target_component_id}", + "sharedCodeConfigRow": "codes/{config_row_name}", + "variablesConfig": "variables", + "variablesValuesRow": "values/{config_row_name}", + "dataAppConfig": "app/{component_id}/{config_name}" + }, + "allowedBranches": [ + "__all__" + ], + "ignoredComponents": [], + "templates": { + "repositories": [ + { + "type": "git", + "name": "keboola", + "url": "https://github.com/keboola/keboola-as-code-templates.git", + "ref": "main" + } + ] + }, + "branches": [], + "configurations": [] +} diff --git a/test/cli/pull/ignore-configurations/in/description.md b/test/cli/pull/ignore-configurations/in/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/initial-state.json b/test/cli/pull/ignore-configurations/initial-state.json new file mode 100644 index 0000000000..198ae4390d --- /dev/null +++ b/test/cli/pull/ignore-configurations/initial-state.json @@ -0,0 +1,52 @@ +{ + "backend": { + "type": "snowflake" + }, + "allBranchesConfigs": [ + "empty" + ], + "branches": [ + { + "branch": { + "name": "Main", + "isDefault": true + }, + "configs": [ + "data-app" + ] + }, + { + "branch": { + "name": "foo", + "isDefault": false + }, + "configs": [ + "with-rows", + "without-rows" + ] + }, + { + "branch": { + "name": "bar", + "isDefault": false + }, + "configs": [ + "with-rows", + "python-transformation", + "shared-codes", + "shared-codes-variables", + "transformation-with-shared-code", + "transformation-with-shared-code-variables", + "snowflake-transformation", + "disabled" + ] + } + ], + "envs": { + "TEST_SHARED_CODE_ID": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ID%%", + "TEST_SHARED_CODE_ROW_ID": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ROW_CODE_WITH_VARIABLES_ID%%", + "TEST_SHARED_CODE_VARIABLES_ID": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODE_VARIABLES_ID%%", + "TEST_TRANSFORMATION_WITH_SHARED_CODE_VARIABLES_ID": "%%TEST_BRANCH_BAR_CONFIG_TRANSFORMATION_WITH_SHARED_CODE_VARIABLES_ID%%", + "TEST_TRANSFORMATION_WITH_SHARED_CODE_VARIABLES_VALUES_ID": "%%TEST_BRANCH_BAR_CONFIG_TRANSFORMATION_WITH_SHARED_CODE_VARIABLES_ROW_DEFAULT_VALUES_ID%%" + } +} diff --git a/test/cli/pull/ignore-configurations/out/.gitkeep b/test/cli/pull/ignore-configurations/out/.gitkeep new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/out/.keboola/kbc_ignore b/test/cli/pull/ignore-configurations/out/.keboola/kbc_ignore new file mode 100644 index 0000000000..46de93ee5b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/.keboola/kbc_ignore @@ -0,0 +1,3 @@ +keboola.data-apps/%%TEST_BRANCH_MAIN_CONFIG_DATA_APP_ID%% +keboola.ex-db-mysql/%%TEST_BRANCH_BAR_CONFIG_WITH_ROWS_ID%%/%%TEST_BRANCH_BAR_CONFIG_WITH_ROWS_ROW_USERS_ID%% +keboola.ex-db-mysql/%%TEST_BRANCH_FOO_CONFIG_WITH_ROWS_ID%% diff --git a/test/cli/pull/ignore-configurations/out/.keboola/manifest.json b/test/cli/pull/ignore-configurations/out/.keboola/manifest.json new file mode 100644 index 0000000000..de403f091b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/.keboola/manifest.json @@ -0,0 +1,180 @@ +{ + "version": 2, + "project": { + "id": %%TEST_KBC_PROJECT_ID%%, + "apiHost": "%%TEST_KBC_STORAGE_API_HOST%%" + }, + "allowTargetEnv": false, + "sortBy": "path", + "naming": { + "branch": "{branch_name}", + "config": "{component_type}/{component_id}/{config_name}", + "configRow": "rows/{config_row_name}", + "schedulerConfig": "schedules/{config_name}", + "sharedCodeConfig": "_shared/{target_component_id}", + "sharedCodeConfigRow": "codes/{config_row_name}", + "variablesConfig": "variables", + "variablesValuesRow": "values/{config_row_name}", + "dataAppConfig": "app/{component_id}/{config_name}" + }, + "allowedBranches": [ + "__all__" + ], + "ignoredComponents": [], + "templates": { + "repositories": [ + { + "type": "git", + "name": "keboola", + "url": "https://github.com/keboola/keboola-as-code-templates.git", + "ref": "main" + } + ] + }, + "branches": [ + { + "id": %%TEST_BRANCH_BAR_ID%%, + "path": "bar" + }, + { + "id": %%TEST_BRANCH_FOO_ID%%, + "path": "foo" + }, + { + "id": %%TEST_BRANCH_MAIN_ID%%, + "path": "main" + } + ], + "configurations": [ + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "keboola.shared-code", + "id": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ID%%", + "path": "_shared/keboola.python-transformation-v2", + "rows": [ + { + "id": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ROW_CODE_WITH_VARIABLES_ID%%", + "path": "codes/code-with-variables" + }, + { + "id": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ROW_MY_CODE_1_ID%%", + "path": "codes/my-code-1" + }, + { + "id": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ROW_MY_CODE_2_ID%%", + "path": "codes/my-code-2" + } + ] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "keboola.variables", + "id": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODE_VARIABLES_ID%%", + "path": "variables", + "relations": [ + { + "configId": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ID%%", + "rowId": "%%TEST_BRANCH_BAR_CONFIG_SHARED_CODES_ROW_CODE_WITH_VARIABLES_ID%%", + "type": "sharedCodeVariablesFor" + } + ], + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "ex-generic-v2", + "id": "%%TEST_BRANCH_BAR_CONFIG_DISABLED_ID%%", + "path": "extractor/ex-generic-v2/disabled", + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "ex-generic-v2", + "id": "%%TEST_BRANCH_ALL_CONFIG_EMPTY_ID%%", + "path": "extractor/ex-generic-v2/empty", + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "keboola.ex-db-mysql", + "id": "%%TEST_BRANCH_BAR_CONFIG_WITH_ROWS_ID%%", + "path": "extractor/keboola.ex-db-mysql/with-rows", + "rows": [ + { + "id": "%%TEST_BRANCH_BAR_CONFIG_WITH_ROWS_ROW_DISABLED_ID%%", + "path": "rows/disabled" + }, + { + "id": "%%TEST_BRANCH_BAR_CONFIG_WITH_ROWS_ROW_TEST_VIEW_ID%%", + "path": "rows/test-view" + } + ] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "keboola.python-transformation-v2", + "id": "%%TEST_BRANCH_BAR_CONFIG_PYTHON_TRANSFORMATION_ID%%", + "path": "transformation/keboola.python-transformation-v2/python-transformation", + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "keboola.python-transformation-v2", + "id": "%%TEST_BRANCH_BAR_CONFIG_TRANSFORMATION_WITH_SHARED_CODE_ID%%", + "path": "transformation/keboola.python-transformation-v2/transformation-with-shared-code", + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "keboola.variables", + "id": "%%TEST_BRANCH_BAR_CONFIG_TRANSFORMATION_WITH_SHARED_CODE_VARIABLES_ID%%", + "path": "variables", + "relations": [ + { + "componentId": "keboola.python-transformation-v2", + "configId": "%%TEST_BRANCH_BAR_CONFIG_TRANSFORMATION_WITH_SHARED_CODE_ID%%", + "type": "variablesFor" + } + ], + "rows": [ + { + "id": "%%TEST_BRANCH_BAR_CONFIG_TRANSFORMATION_WITH_SHARED_CODE_VARIABLES_ROW_DEFAULT_VALUES_ID%%", + "path": "values/default", + "relations": [ + { + "type": "variablesValuesFor" + } + ] + } + ] + }, + { + "branchId": %%TEST_BRANCH_BAR_ID%%, + "componentId": "keboola.snowflake-transformation", + "id": "%%TEST_BRANCH_BAR_CONFIG_SNOWFLAKE_TRANSFORMATION_ID%%", + "path": "transformation/keboola.snowflake-transformation/snowflake-transformation", + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_FOO_ID%%, + "componentId": "ex-generic-v2", + "id": "%%TEST_BRANCH_ALL_CONFIG_EMPTY_ID%%", + "path": "extractor/ex-generic-v2/empty", + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_FOO_ID%%, + "componentId": "ex-generic-v2", + "id": "%%TEST_BRANCH_FOO_CONFIG_WITHOUT_ROWS_ID%%", + "path": "extractor/ex-generic-v2/without-rows", + "rows": [] + }, + { + "branchId": %%TEST_BRANCH_MAIN_ID%%, + "componentId": "ex-generic-v2", + "id": "%%TEST_BRANCH_ALL_CONFIG_EMPTY_ID%%", + "path": "extractor/ex-generic-v2/empty", + "rows": [] + } + ] +} diff --git a/test/cli/pull/ignore-configurations/out/.keboola/project.json b/test/cli/pull/ignore-configurations/out/.keboola/project.json new file mode 100644 index 0000000000..2f8539cf19 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/.keboola/project.json @@ -0,0 +1,9 @@ +{ + "backends": [ + %A + ], + "features": [ + %A + ], + "defaultBranchId": %A +} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/code.py b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/code.py new file mode 100644 index 0000000000..5b04076290 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/code.py @@ -0,0 +1,3 @@ +num1 = {{num1}} +num2 = {{num2}} +sum = num1 + num2 diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/config.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/config.json @@ -0,0 +1 @@ +{} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/description.md b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/meta.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/meta.json new file mode 100644 index 0000000000..81d037f4ec --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Code with variables" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/config.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/config.json new file mode 100644 index 0000000000..fd78a74174 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/config.json @@ -0,0 +1,12 @@ +{ + "variables": [ + { + "name": "num1", + "type": "string" + }, + { + "name": "num2", + "type": "string" + } + ] +} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/description.md b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/meta.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/meta.json new file mode 100644 index 0000000000..f5a9bbd3d6 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/code-with-variables/variables/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Shared Code Variables", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/code.py b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/code.py new file mode 100644 index 0000000000..0e3295e312 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/code.py @@ -0,0 +1,3 @@ +# This program prints Hello, world! + +print('Hello, world!') diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/config.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/config.json @@ -0,0 +1 @@ +{} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/description.md b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/meta.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/meta.json new file mode 100644 index 0000000000..e817e6ae22 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-1/meta.json @@ -0,0 +1,3 @@ +{ + "name": "My code 1" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/code.py b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/code.py new file mode 100644 index 0000000000..9e6b7cca58 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/code.py @@ -0,0 +1,3 @@ +num1 = 1.5 +num2 = 6.3 +sum = num1 + num2 diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/config.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/config.json @@ -0,0 +1 @@ +{} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/description.md b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/meta.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/meta.json new file mode 100644 index 0000000000..810c16091f --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/codes/my-code-2/meta.json @@ -0,0 +1,3 @@ +{ + "name": "My code 2" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/config.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/config.json new file mode 100644 index 0000000000..3a90b5a0b0 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/config.json @@ -0,0 +1,3 @@ +{ + "componentId": "keboola.python-transformation-v2" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/description.md b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/meta.json b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/meta.json new file mode 100644 index 0000000000..1e46968328 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/_shared/keboola.python-transformation-v2/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Shared Codes", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/description.md b/test/cli/pull/ignore-configurations/out/bar/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/config.json b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/config.json new file mode 100644 index 0000000000..e7bf7fb680 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/config.json @@ -0,0 +1,7 @@ +{ + "parameters": { + "api": { + "baseUrl": "https://jsonplaceholder.typicode.com" + } + } +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/description.md b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/meta.json b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/meta.json new file mode 100644 index 0000000000..f17f3f6867 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/disabled/meta.json @@ -0,0 +1,4 @@ +{ + "name": "disabled", + "isDisabled": true +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/config.json b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/config.json @@ -0,0 +1 @@ +{} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/description.md b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/meta.json b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/meta.json new file mode 100644 index 0000000000..eb2c70d865 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/ex-generic-v2/empty/meta.json @@ -0,0 +1,4 @@ +{ + "name": "empty", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/config.json b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/config.json new file mode 100644 index 0000000000..efebab79e7 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/config.json @@ -0,0 +1,7 @@ +{ + "parameters": { + "db": { + "host": "mysql.example.com" + } + } +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/description.md b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/meta.json b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/meta.json new file mode 100644 index 0000000000..b6b602f6ff --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/meta.json @@ -0,0 +1,4 @@ +{ + "name": "with-rows", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/config.json b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/config.json new file mode 100644 index 0000000000..45fb08048a --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/config.json @@ -0,0 +1,5 @@ +{ + "parameters": { + "incremental": false + } +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/description.md b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/meta.json b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/meta.json new file mode 100644 index 0000000000..f17f3f6867 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/disabled/meta.json @@ -0,0 +1,4 @@ +{ + "name": "disabled", + "isDisabled": true +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json new file mode 100644 index 0000000000..45fb08048a --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/config.json @@ -0,0 +1,5 @@ +{ + "parameters": { + "incremental": false + } +} diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json new file mode 100644 index 0000000000..221e9ae8ed --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/extractor/keboola.ex-db-mysql/with-rows/rows/test-view/meta.json @@ -0,0 +1,4 @@ +{ + "name": "test_view", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/meta.json b/test/cli/pull/ignore-configurations/out/bar/meta.json new file mode 100644 index 0000000000..430f14d1c3 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/meta.json @@ -0,0 +1,4 @@ +{ + "name": "bar", + "isDefault": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/code.py b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/code.py new file mode 100644 index 0000000000..ae2310732e --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/code.py @@ -0,0 +1 @@ +print(100) diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/meta.json new file mode 100644 index 0000000000..ae0c1fa955 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/001-code-x/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Code X" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/code.py b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/code.py new file mode 100644 index 0000000000..b025ada513 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/code.py @@ -0,0 +1 @@ +print(200) diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/meta.json new file mode 100644 index 0000000000..1ea592c76f --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/002-code-y/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Code Y" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/meta.json new file mode 100644 index 0000000000..94df8933d4 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/001-block-1/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Block 1" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/code.py b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/code.py new file mode 100644 index 0000000000..51ad5cc936 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/code.py @@ -0,0 +1,13 @@ +# Sum of natural numbers up to num + +num = 16 + +if num < 0: + print("Enter a positive number") +else: + sum = 0 + # use while loop to iterate until zero + while (num > 0): + sum += num + num -= 1 + print("The sum is", sum) diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/meta.json new file mode 100644 index 0000000000..6fa8542de9 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/001-code-z/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Code Z" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/meta.json new file mode 100644 index 0000000000..6be6b83ab4 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/blocks/002-block-2/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Block 2" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/config.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/config.json new file mode 100644 index 0000000000..a6a383d965 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/config.json @@ -0,0 +1,7 @@ +{ + "parameters": { + "packages": [ + "xgboost" + ] + } +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/description.md b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/meta.json new file mode 100644 index 0000000000..1cf5adb04f --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/python-transformation/meta.json @@ -0,0 +1,4 @@ +{ + "name": "python-transformation", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/code.py b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/code.py new file mode 100644 index 0000000000..1c121c83cb --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/code.py @@ -0,0 +1 @@ +# {{:codes/code-with-variables}} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/meta.json new file mode 100644 index 0000000000..035182eda2 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/001-shared-code-used/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Shared Code Used" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/meta.json new file mode 100644 index 0000000000..94df8933d4 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/blocks/001-block-1/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Block 1" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/config.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/config.json new file mode 100644 index 0000000000..d7e610265b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/config.json @@ -0,0 +1,4 @@ +{ + "parameters": {}, + "shared_code_path": "_shared/keboola.python-transformation-v2" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/description.md b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/meta.json new file mode 100644 index 0000000000..fd06d64c88 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Transformation With Shared Code", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/config.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/config.json new file mode 100644 index 0000000000..fd78a74174 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/config.json @@ -0,0 +1,12 @@ +{ + "variables": [ + { + "name": "num1", + "type": "string" + }, + { + "name": "num2", + "type": "string" + } + ] +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/description.md b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/meta.json new file mode 100644 index 0000000000..a52fedda9a --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Transformation With Shared Code Variables", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/config.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/config.json new file mode 100644 index 0000000000..49208262d4 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/config.json @@ -0,0 +1,12 @@ +{ + "values": [ + { + "name": "num1", + "value": "10" + }, + { + "name": "num2", + "value": "20" + } + ] +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/description.md b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/description.md new file mode 100644 index 0000000000..28f6091c29 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/description.md @@ -0,0 +1 @@ +test fixture diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/meta.json new file mode 100644 index 0000000000..236c929375 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.python-transformation-v2/transformation-with-shared-code/variables/values/default/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Default values", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/code.sql b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/code.sql new file mode 100644 index 0000000000..452269a7f2 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/code.sql @@ -0,0 +1,11 @@ +CREATE OR REPLACE TABLE "test" ( + "id" number default null, + "orderstatus" varchar(100) default null, + "price" float +); + +INSERT INTO "test" ("id", "orderstatus", "price") VALUES(123, 'ok', 12.34); + +INSERT INTO "test" ("id", "orderstatus", "price") VALUES(456, 'ko', 45.67); + +DROP TABLE IF EXISTS "test"; diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/meta.json new file mode 100644 index 0000000000..ae0c1fa955 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/001-code-x/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Code X" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/code.sql b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/code.sql new file mode 100644 index 0000000000..c919a44cf5 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/code.sql @@ -0,0 +1,2 @@ +CREATE TABLE "test2" AS SELECT $1, $2 +FROM VALUES( 1, 'two' ), ( 3, 'four' ), ( 5, 'six' ); diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/meta.json new file mode 100644 index 0000000000..1ea592c76f --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/002-code-y/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Code Y" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/meta.json new file mode 100644 index 0000000000..94df8933d4 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/001-block-1/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Block 1" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/code.sql b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/code.sql new file mode 100644 index 0000000000..107056d764 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/code.sql @@ -0,0 +1,2 @@ +CREATE TABLE "test3" AS SELECT $1, $2 +FROM VALUES( 123, 'x' ), ( 345, 'y' ); diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/meta.json new file mode 100644 index 0000000000..6fa8542de9 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/001-code-z/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Code Z" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/meta.json new file mode 100644 index 0000000000..77267ea32d --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/blocks/002-block-2-my-name/meta.json @@ -0,0 +1,3 @@ +{ + "name": "Block 2 - my name" +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/config.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/config.json new file mode 100644 index 0000000000..4b448a37f2 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/config.json @@ -0,0 +1,3 @@ +{ + "parameters": {} +} diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/description.md b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/meta.json b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/meta.json new file mode 100644 index 0000000000..a92d9f2235 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/bar/transformation/keboola.snowflake-transformation/snowflake-transformation/meta.json @@ -0,0 +1,4 @@ +{ + "name": "snowflake-transformation", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/description.md b/test/cli/pull/ignore-configurations/out/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/out/foo/description.md b/test/cli/pull/ignore-configurations/out/foo/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/config.json b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/config.json @@ -0,0 +1 @@ +{} diff --git a/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/description.md b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/meta.json b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/meta.json new file mode 100644 index 0000000000..eb2c70d865 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/empty/meta.json @@ -0,0 +1,4 @@ +{ + "name": "empty", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/config.json b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/config.json new file mode 100644 index 0000000000..e7bf7fb680 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/config.json @@ -0,0 +1,7 @@ +{ + "parameters": { + "api": { + "baseUrl": "https://jsonplaceholder.typicode.com" + } + } +} diff --git a/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/description.md b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/meta.json b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/meta.json new file mode 100644 index 0000000000..fb770b9775 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/foo/extractor/ex-generic-v2/without-rows/meta.json @@ -0,0 +1,4 @@ +{ + "name": "without-rows", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/foo/meta.json b/test/cli/pull/ignore-configurations/out/foo/meta.json new file mode 100644 index 0000000000..76aaab5c4a --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/foo/meta.json @@ -0,0 +1,4 @@ +{ + "name": "foo", + "isDefault": false +} diff --git a/test/cli/pull/ignore-configurations/out/main/description.md b/test/cli/pull/ignore-configurations/out/main/description.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/config.json b/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/config.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/config.json @@ -0,0 +1 @@ +{} diff --git a/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/description.md b/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/description.md new file mode 100644 index 0000000000..e8edb6fc85 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/description.md @@ -0,0 +1 @@ +test fixture \ No newline at end of file diff --git a/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/meta.json b/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/meta.json new file mode 100644 index 0000000000..eb2c70d865 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/main/extractor/ex-generic-v2/empty/meta.json @@ -0,0 +1,4 @@ +{ + "name": "empty", + "isDisabled": false +} diff --git a/test/cli/pull/ignore-configurations/out/main/meta.json b/test/cli/pull/ignore-configurations/out/main/meta.json new file mode 100644 index 0000000000..154a3ce050 --- /dev/null +++ b/test/cli/pull/ignore-configurations/out/main/meta.json @@ -0,0 +1,4 @@ +{ + "name": "Main", + "isDefault": true +} From 9ab9c9c3502afe193f0df1ecd9f9ae0587671690 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Fri, 8 Nov 2024 16:39:23 +0100 Subject: [PATCH 06/11] refactor: deleted unnecessary param --- internal/pkg/plan/push/push.go | 2 +- pkg/lib/operation/project/sync/push/operation.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/plan/push/push.go b/internal/pkg/plan/push/push.go index fd49883556..086a798d22 100644 --- a/internal/pkg/plan/push/push.go +++ b/internal/pkg/plan/push/push.go @@ -7,7 +7,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" ) -func NewPlan(diffResults *diff.Results, allowTargetEnv bool) (*diffop.Plan, error) { +func NewPlan(diffResults *diff.Results) (*diffop.Plan, error) { plan := diffop.NewPlan(`push`) for _, result := range diffResults.Results { switch result.State { diff --git a/pkg/lib/operation/project/sync/push/operation.go b/pkg/lib/operation/project/sync/push/operation.go index 3bd5a66a98..427d13a79b 100644 --- a/pkg/lib/operation/project/sync/push/operation.go +++ b/pkg/lib/operation/project/sync/push/operation.go @@ -72,7 +72,7 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen } // Get plan - plan, err := push.NewPlan(results, projectState.ProjectManifest().AllowTargetENV()) + plan, err := push.NewPlan(results) if err != nil { return err } From 920869a08feb7ac58e340cef155fe893781a2562 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Mon, 11 Nov 2024 16:50:27 +0100 Subject: [PATCH 07/11] refactor: moved to separate module --- internal/pkg/project/ignore/file.go | 90 +++++++ internal/pkg/project/ignore/file_test.go | 242 +++++++++++++++++++ internal/pkg/state/registry/registry.go | 71 +----- internal/pkg/state/registry/registry_test.go | 153 ++---------- 4 files changed, 359 insertions(+), 197 deletions(-) create mode 100644 internal/pkg/project/ignore/file.go create mode 100644 internal/pkg/project/ignore/file_test.go diff --git a/internal/pkg/project/ignore/file.go b/internal/pkg/project/ignore/file.go new file mode 100644 index 0000000000..2176437ee5 --- /dev/null +++ b/internal/pkg/project/ignore/file.go @@ -0,0 +1,90 @@ +package ignore + +import ( + "context" + "strings" + + "github.com/keboola/keboola-as-code/internal/pkg/filesystem" + "github.com/keboola/keboola-as-code/internal/pkg/state" + "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" +) + +const KBCIgnoreFilePath = ".keboola/kbc_ignore" + +type File struct { + rawStringPattern string + state *state.Registry +} + +func newFile(pattern string, state *state.Registry) *File { + return &File{ + rawStringPattern: pattern, + state: state, + } +} + +func LoadFile(ctx context.Context, fs filesystem.Fs, state *state.Registry, path string) (*File, error) { + if !fs.Exists(ctx, path) { + return nil, errors.Errorf("ignore file \"%s\" not found", path) + } + + content, err := fs.ReadFile(ctx, filesystem.NewFileDef(path)) + if err != nil { + return nil, err + } + + f := &File{ + state: state, + rawStringPattern: content.Content, + } + + return f, nil +} + +func (f *File) IgnoreConfigsOrRows() error { + return f.applyIgnoredPatterns() +} + +func (f *File) parseIgnoredPatterns() []string { + var ignorePatterns []string + lines := strings.Split(f.rawStringPattern, "\n") + for _, line := range lines { + trimmedLine := strings.TrimSpace(line) + // Skip empty lines and comments + if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + ignorePatterns = append(ignorePatterns, trimmedLine) + } + } + + return ignorePatterns +} + +// applyIgnorePattern applies a single ignore pattern, marking the appropriate config or row as ignored. +func (f *File) applyIgnorePattern(ignoreConfig string) error { + parts := strings.Split(ignoreConfig, "/") + + switch len(parts) { + case 2: + // Ignore config by ID and name. + configID, componentID := parts[1], parts[0] + f.state.IgnoreConfig(configID, componentID) + case 3: + // Ignore specific config row. + configID, rowID := parts[1], parts[2] + f.state.IgnoreConfigRow(configID, rowID) + default: + return errors.Errorf("invalid ignore ignoreConfig format: %s", ignoreConfig) + } + + return nil +} + +// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows. +func (f *File) applyIgnoredPatterns() error { + for _, pattern := range f.parseIgnoredPatterns() { + if err := f.applyIgnorePattern(pattern); err != nil { + continue + } + } + return nil +} diff --git a/internal/pkg/project/ignore/file_test.go b/internal/pkg/project/ignore/file_test.go new file mode 100644 index 0000000000..8f95187e58 --- /dev/null +++ b/internal/pkg/project/ignore/file_test.go @@ -0,0 +1,242 @@ +package ignore + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/keboola/keboola-as-code/internal/pkg/filesystem" + "github.com/keboola/keboola-as-code/internal/pkg/filesystem/aferofs" + "github.com/keboola/keboola-as-code/internal/pkg/filesystem/knownpaths" + "github.com/keboola/keboola-as-code/internal/pkg/model" + "github.com/keboola/keboola-as-code/internal/pkg/naming" + "github.com/keboola/keboola-as-code/internal/pkg/state/registry" +) + +func Test_loadFile(t *testing.T) { + t.Parallel() + ctx := context.Background() + + projectState := newTestRegistry(t) + + fs := aferofs.NewMemoryFs() + require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile(`foo/bar1`, "keboola.bar/678/34\nkeboola.foo/345"))) + + file, err := LoadFile(ctx, fs, projectState, "foo/bar1") + require.NoError(t, err) + + assert.Equal(t, "keboola.bar/678/34\nkeboola.foo/345", file.rawStringPattern) + + assert.NoError(t, file.IgnoreConfigsOrRows()) + + assert.Len(t, projectState.IgnoredConfigRows(), 1) + assert.Len(t, projectState.IgnoredConfigs(), 1) + assert.Equal(t, projectState.IgnoredConfigRows()[0].ID.String(), "34") + assert.Equal(t, projectState.IgnoredConfigs()[0].ID.String(), "345") +} + +func Test_applyIgnoredPatterns(t *testing.T) { + t.Parallel() + projectState := newTestRegistry(t) + + type args struct { + pattern string + } + + file := &File{ + state: projectState, + } + + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + }{ + { + name: "empty patterns", + args: args{ + pattern: "", + }, + wantErr: assert.Error, + }, + { + name: "wrong pattern", + args: args{ + pattern: "wrong pattern", + }, + wantErr: assert.Error, + }, + { + name: "too long pattern", + args: args{ + pattern: "keboola.bar/687/1234/1234", + }, + wantErr: assert.Error, + }, + { + name: "short pattern", + args: args{ + pattern: "keboola.bar", + }, + wantErr: assert.Error, + }, + { + name: "correct pattern", + args: args{ + pattern: "keboola.bar/687", + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tt.wantErr(t, file.applyIgnorePattern(tt.args.pattern), fmt.Sprintf("applyIgnoredPatterns(%v)", tt.args.pattern)) + }) + } +} + +func Test_parseIgnoredPatterns(t *testing.T) { + t.Parallel() + + projectState := newTestRegistry(t) + + type args struct { + content string + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "with comments", + args: args{ + content: "##dataapps\nkeboola.data-apps/12345\n##dataapps2\nkeboola.data-apps/123/rowId123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123/rowId123", + "keboola.data-apps/12", + }, + }, + { + name: "with empty lines", + args: args{ + content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123", + "keboola.data-apps/12", + }, + }, + { + name: "empty file", + args: args{ + content: "\n", + }, + want: nil, + }, + { + name: "error", + args: args{ + content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123", + "keboola.data-apps/12", + }, + }, + { + name: "empty file", + args: args{ + content: "\n", + }, + want: nil, + }, + } + + for _, tt := range tests { + file := newFile(tt.args.content, projectState) + got := file.parseIgnoredPatterns() + assert.Equal(t, tt.want, got) + } +} + +func newTestRegistry(t *testing.T) *registry.Registry { + t.Helper() + + r := registry.New(knownpaths.NewNop(context.Background()), naming.NewRegistry(), model.NewComponentsMap(nil), model.SortByPath) + // Branch 1 + branch1Key := model.BranchKey{ID: 123} + branch1 := &model.BranchState{ + BranchManifest: &model.BranchManifest{ + BranchKey: branch1Key, + }, + Local: &model.Branch{ + Name: "Main", + IsDefault: true, + }, + } + assert.NoError(t, r.Set(branch1)) + + // Branch 2 + branch2Key := model.BranchKey{ID: 567} + branch2 := &model.BranchState{ + BranchManifest: &model.BranchManifest{ + BranchKey: branch2Key, + }, + Local: &model.Branch{ + Name: "Foo Bar Branch", + IsDefault: false, + }, + } + assert.NoError(t, r.Set(branch2)) + + // Config 1 + config1Key := model.ConfigKey{BranchID: 123, ComponentID: "keboola.foo", ID: `345`} + config1 := &model.ConfigState{ + ConfigManifest: &model.ConfigManifest{ConfigKey: config1Key}, + Local: &model.Config{ + Name: "Config 1", + }, + } + assert.NoError(t, r.Set(config1)) + + // Config 2 + config2Key := model.ConfigKey{BranchID: 123, ComponentID: "keboola.bar", ID: `678`} + config2 := &model.ConfigState{ + ConfigManifest: &model.ConfigManifest{ConfigKey: config2Key}, + Local: &model.Config{ + Name: "Config 2", + }, + } + assert.NoError(t, r.Set(config2)) + + // Config Row 1 + row1Key := model.ConfigRowKey{BranchID: 123, ComponentID: "keboola.bar", ConfigID: `678`, ID: `12`} + row1 := &model.ConfigRowState{ + ConfigRowManifest: &model.ConfigRowManifest{ConfigRowKey: row1Key}, + Local: &model.ConfigRow{ + Name: "Config Row 1", + }, + } + assert.NoError(t, r.Set(row1)) + + // Config Row 2 + row2Key := model.ConfigRowKey{BranchID: 123, ComponentID: "keboola.bar", ConfigID: `678`, ID: `34`} + row2 := &model.ConfigRowState{ + ConfigRowManifest: &model.ConfigRowManifest{ConfigRowKey: row2Key}, + Local: &model.ConfigRow{ + Name: "Config Row 2", + }, + } + assert.NoError(t, r.Set(row2)) + + return r +} diff --git a/internal/pkg/state/registry/registry.go b/internal/pkg/state/registry/registry.go index 30030084b0..95eb9c289d 100644 --- a/internal/pkg/state/registry/registry.go +++ b/internal/pkg/state/registry/registry.go @@ -2,12 +2,10 @@ package registry import ( "context" - "strings" "sync" "github.com/keboola/go-utils/pkg/orderedmap" - "github.com/keboola/keboola-as-code/internal/pkg/filesystem" "github.com/keboola/keboola-as-code/internal/pkg/filesystem/knownpaths" . "github.com/keboola/keboola-as-code/internal/pkg/model" "github.com/keboola/keboola-as-code/internal/pkg/naming" @@ -237,18 +235,19 @@ func (s *Registry) ConfigRowsFrom(config ConfigKey) (rows []*ConfigRowState) { return rows } -func (s *Registry) SetIgnoredConfigsOrRows(ctx context.Context, fs filesystem.Fs, path string) error { - content, err := fs.ReadFile(ctx, filesystem.NewFileDef(path)) - if err != nil { - return err - } - - if content.Content == "" { - return nil - } - - return s.applyIgnoredPatterns(content.Content) -} +// func (s *Registry) SetIgnoredConfigsOrRows(ctx context.Context, fs filesystem.Fs, path string) error { +// //strukturu pattre.field +// content, err := fs.ReadFile(ctx, filesystem.NewFileDef(path)) +// if err != nil { +// return err +// } +// +// if content.Content == "" { +// return nil +// } +// +// return s.applyIgnoredPatterns(content.Content) +//} func (s *Registry) GetPath(key Key) (AbsPath, bool) { objectState, found := s.Get(key) @@ -328,47 +327,3 @@ func (s *Registry) GetOrCreateFrom(manifest ObjectManifest) (ObjectState, error) return s.CreateFrom(manifest) } - -// applyIgnorePattern applies a single ignore pattern, marking the appropriate config or row as ignored. -func (s *Registry) applyIgnorePattern(pattern string) error { - parts := strings.Split(pattern, "/") - - switch len(parts) { - case 2: - // Ignore config by ID and name. - configID, componentID := parts[1], parts[0] - s.IgnoreConfig(configID, componentID) - case 3: - // Ignore specific config row. - configID, rowID := parts[1], parts[2] - s.IgnoreConfigRow(configID, rowID) - default: - return errors.Errorf("invalid ignore pattern format: %s", pattern) - } - - return nil -} - -// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows. -func (s *Registry) applyIgnoredPatterns(content string) error { - for _, pattern := range parseIgnoredPatterns(content) { - if err := s.applyIgnorePattern(pattern); err != nil { - continue - } - } - return nil -} - -func parseIgnoredPatterns(content string) []string { - var ignorePatterns []string - lines := strings.Split(content, "\n") - for _, line := range lines { - trimmedLine := strings.TrimSpace(line) - // Skip empty lines and comments - if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - ignorePatterns = append(ignorePatterns, trimmedLine) - } - } - - return ignorePatterns -} diff --git a/internal/pkg/state/registry/registry_test.go b/internal/pkg/state/registry/registry_test.go index 602331f9f3..0ad0bdb965 100644 --- a/internal/pkg/state/registry/registry_test.go +++ b/internal/pkg/state/registry/registry_test.go @@ -2,11 +2,9 @@ package registry import ( "context" - "fmt" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/keboola/keboola-as-code/internal/pkg/filesystem" "github.com/keboola/keboola-as-code/internal/pkg/filesystem/aferofs" @@ -177,20 +175,20 @@ func TestStateTrackRecordInvalid(t *testing.T) { assert.Empty(t, s.UntrackedPaths()) } -func TestRegistry_SetIgnoredConfigsOrRows(t *testing.T) { - t.Parallel() - ctx := context.Background() - registry := newTestState(t, knownpaths.NewNop(ctx)) - fs := aferofs.NewMemoryFs() - require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile(`foo/bar1`, "keboola.bar/678/34\nkeboola.foo/345"))) - - require.NoError(t, registry.SetIgnoredConfigsOrRows(ctx, fs, "foo/bar1")) - - assert.Len(t, registry.IgnoredConfigRows(), 1) - assert.Len(t, registry.IgnoredConfigs(), 1) - assert.Equal(t, registry.IgnoredConfigRows()[0].ID.String(), "34") - assert.Equal(t, registry.IgnoredConfigs()[0].ID.String(), "345") -} +// func TestRegistry_SetIgnoredConfigsOrRows(t *testing.T) { +// t.Parallel() +// ctx := context.Background() +// registry := newTestState(t, knownpaths.NewNop(ctx)) +// fs := aferofs.NewMemoryFs() +// require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile(`foo/bar1`, "keboola.bar/678/34\nkeboola.foo/345"))) +// +// require.NoError(t, registry.SetIgnoredConfigsOrRows(ctx, fs, "foo/bar1")) +// +// assert.Len(t, registry.IgnoredConfigRows(), 1) +// assert.Len(t, registry.IgnoredConfigs(), 1) +// assert.Equal(t, registry.IgnoredConfigRows()[0].ID.String(), "34") +// assert.Equal(t, registry.IgnoredConfigs()[0].ID.String(), "345") +//} func TestRegistry_GetPath(t *testing.T) { t.Parallel() @@ -318,126 +316,3 @@ func newTestState(t *testing.T, paths *knownpaths.Paths) *Registry { return registry } - -func Test_parseIgnoredPatterns(t *testing.T) { - t.Parallel() - type args struct { - content string - } - tests := []struct { - name string - args args - want []string - }{ - { - name: "with comments", - args: args{ - content: "##dataapps\nkeboola.data-apps/12345\n##dataapps2\nkeboola.data-apps/123/rowId123\n##dataaps3\nkeboola.data-apps/12", - }, - want: []string{ - "keboola.data-apps/12345", - "keboola.data-apps/123/rowId123", - "keboola.data-apps/12", - }, - }, - { - name: "with empty lines", - args: args{ - content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", - }, - want: []string{ - "keboola.data-apps/12345", - "keboola.data-apps/123", - "keboola.data-apps/12", - }, - }, - { - name: "empty file", - args: args{ - content: "\n", - }, - want: nil, - }, - { - name: "error", - args: args{ - content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", - }, - want: []string{ - "keboola.data-apps/12345", - "keboola.data-apps/123", - "keboola.data-apps/12", - }, - }, - { - name: "empty file", - args: args{ - content: "\n", - }, - want: nil, - }, - } - - for _, tt := range tests { - got := parseIgnoredPatterns(tt.args.content) - assert.Equal(t, tt.want, got) - } -} - -func TestRegistry_applyIgnoredPatterns(t *testing.T) { - t.Parallel() - ctx := context.Background() - registry := newTestState(t, knownpaths.NewNop(ctx)) - - type args struct { - pattern string - } - - tests := []struct { - name string - args args - wantErr assert.ErrorAssertionFunc - }{ - { - name: "empty patterns", - args: args{ - pattern: "", - }, - wantErr: assert.Error, - }, - { - name: "wrong pattern", - args: args{ - pattern: "wrong pattern", - }, - wantErr: assert.Error, - }, - { - name: "too long pattern", - args: args{ - pattern: "keboola.bar/687/1234/1234", - }, - wantErr: assert.Error, - }, - { - name: "short pattern", - args: args{ - pattern: "keboola.bar", - }, - wantErr: assert.Error, - }, - { - name: "correct pattern", - args: args{ - pattern: "keboola.bar/687", - }, - wantErr: assert.NoError, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - tt.wantErr(t, registry.applyIgnorePattern(tt.args.pattern), fmt.Sprintf("applyIgnoredPatterns(%v)", tt.args.pattern)) - }) - } -} From 688c50d892f9b7ef54910f0a839056b5541caa90 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Mon, 11 Nov 2024 16:52:12 +0100 Subject: [PATCH 08/11] refactor: used a new ignore module --- pkg/lib/operation/project/sync/pull/operation.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/lib/operation/project/sync/pull/operation.go b/pkg/lib/operation/project/sync/pull/operation.go index 5e179f52e6..e200611b65 100644 --- a/pkg/lib/operation/project/sync/pull/operation.go +++ b/pkg/lib/operation/project/sync/pull/operation.go @@ -11,7 +11,7 @@ import ( "github.com/keboola/keboola-as-code/internal/pkg/plan/pull" "github.com/keboola/keboola-as-code/internal/pkg/project" "github.com/keboola/keboola-as-code/internal/pkg/project/cachefile" - "github.com/keboola/keboola-as-code/internal/pkg/state/registry" + "github.com/keboola/keboola-as-code/internal/pkg/project/ignore" "github.com/keboola/keboola-as-code/internal/pkg/telemetry" "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" saveManifest "github.com/keboola/keboola-as-code/pkg/lib/operation/project/local/manifest/save" @@ -50,10 +50,17 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen logger := d.Logger() - if projectState.Fs().Exists(ctx, registry.KBCIgnoreFilePath) { - if err := projectState.SetIgnoredConfigsOrRows(ctx, projectState.Fs(), registry.KBCIgnoreFilePath); err != nil { + if projectState.Fs().Exists(ctx, ignore.KBCIgnoreFilePath) { + // Load ignore file + file, err := ignore.LoadFile(ctx, projectState.Fs(), projectState.Registry, ignore.KBCIgnoreFilePath) + if err != nil { return err } + + if err = file.IgnoreConfigsOrRows(); err != nil { + return err + } + ignoreConfigsAndRows(projectState) } From b90c5f36929739aa874a3eabf179934f293beebc Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Nov 2024 09:13:04 +0100 Subject: [PATCH 09/11] refactor: moved to separate file --- internal/pkg/project/ignore/file.go | 49 ------- internal/pkg/project/ignore/file_test.go | 138 ----------------- internal/pkg/project/ignore/ignore.go | 55 +++++++ internal/pkg/project/ignore/ignore_test.go | 163 +++++++++++++++++++++ 4 files changed, 218 insertions(+), 187 deletions(-) create mode 100644 internal/pkg/project/ignore/ignore.go create mode 100644 internal/pkg/project/ignore/ignore_test.go diff --git a/internal/pkg/project/ignore/file.go b/internal/pkg/project/ignore/file.go index 2176437ee5..3cbeb7de74 100644 --- a/internal/pkg/project/ignore/file.go +++ b/internal/pkg/project/ignore/file.go @@ -2,7 +2,6 @@ package ignore import ( "context" - "strings" "github.com/keboola/keboola-as-code/internal/pkg/filesystem" "github.com/keboola/keboola-as-code/internal/pkg/state" @@ -40,51 +39,3 @@ func LoadFile(ctx context.Context, fs filesystem.Fs, state *state.Registry, path return f, nil } - -func (f *File) IgnoreConfigsOrRows() error { - return f.applyIgnoredPatterns() -} - -func (f *File) parseIgnoredPatterns() []string { - var ignorePatterns []string - lines := strings.Split(f.rawStringPattern, "\n") - for _, line := range lines { - trimmedLine := strings.TrimSpace(line) - // Skip empty lines and comments - if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { - ignorePatterns = append(ignorePatterns, trimmedLine) - } - } - - return ignorePatterns -} - -// applyIgnorePattern applies a single ignore pattern, marking the appropriate config or row as ignored. -func (f *File) applyIgnorePattern(ignoreConfig string) error { - parts := strings.Split(ignoreConfig, "/") - - switch len(parts) { - case 2: - // Ignore config by ID and name. - configID, componentID := parts[1], parts[0] - f.state.IgnoreConfig(configID, componentID) - case 3: - // Ignore specific config row. - configID, rowID := parts[1], parts[2] - f.state.IgnoreConfigRow(configID, rowID) - default: - return errors.Errorf("invalid ignore ignoreConfig format: %s", ignoreConfig) - } - - return nil -} - -// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows. -func (f *File) applyIgnoredPatterns() error { - for _, pattern := range f.parseIgnoredPatterns() { - if err := f.applyIgnorePattern(pattern); err != nil { - continue - } - } - return nil -} diff --git a/internal/pkg/project/ignore/file_test.go b/internal/pkg/project/ignore/file_test.go index 8f95187e58..ddd38ac80d 100644 --- a/internal/pkg/project/ignore/file_test.go +++ b/internal/pkg/project/ignore/file_test.go @@ -2,7 +2,6 @@ package ignore import ( "context" - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -29,143 +28,6 @@ func Test_loadFile(t *testing.T) { require.NoError(t, err) assert.Equal(t, "keboola.bar/678/34\nkeboola.foo/345", file.rawStringPattern) - - assert.NoError(t, file.IgnoreConfigsOrRows()) - - assert.Len(t, projectState.IgnoredConfigRows(), 1) - assert.Len(t, projectState.IgnoredConfigs(), 1) - assert.Equal(t, projectState.IgnoredConfigRows()[0].ID.String(), "34") - assert.Equal(t, projectState.IgnoredConfigs()[0].ID.String(), "345") -} - -func Test_applyIgnoredPatterns(t *testing.T) { - t.Parallel() - projectState := newTestRegistry(t) - - type args struct { - pattern string - } - - file := &File{ - state: projectState, - } - - tests := []struct { - name string - args args - wantErr assert.ErrorAssertionFunc - }{ - { - name: "empty patterns", - args: args{ - pattern: "", - }, - wantErr: assert.Error, - }, - { - name: "wrong pattern", - args: args{ - pattern: "wrong pattern", - }, - wantErr: assert.Error, - }, - { - name: "too long pattern", - args: args{ - pattern: "keboola.bar/687/1234/1234", - }, - wantErr: assert.Error, - }, - { - name: "short pattern", - args: args{ - pattern: "keboola.bar", - }, - wantErr: assert.Error, - }, - { - name: "correct pattern", - args: args{ - pattern: "keboola.bar/687", - }, - wantErr: assert.NoError, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - tt.wantErr(t, file.applyIgnorePattern(tt.args.pattern), fmt.Sprintf("applyIgnoredPatterns(%v)", tt.args.pattern)) - }) - } -} - -func Test_parseIgnoredPatterns(t *testing.T) { - t.Parallel() - - projectState := newTestRegistry(t) - - type args struct { - content string - } - tests := []struct { - name string - args args - want []string - }{ - { - name: "with comments", - args: args{ - content: "##dataapps\nkeboola.data-apps/12345\n##dataapps2\nkeboola.data-apps/123/rowId123\n##dataaps3\nkeboola.data-apps/12", - }, - want: []string{ - "keboola.data-apps/12345", - "keboola.data-apps/123/rowId123", - "keboola.data-apps/12", - }, - }, - { - name: "with empty lines", - args: args{ - content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", - }, - want: []string{ - "keboola.data-apps/12345", - "keboola.data-apps/123", - "keboola.data-apps/12", - }, - }, - { - name: "empty file", - args: args{ - content: "\n", - }, - want: nil, - }, - { - name: "error", - args: args{ - content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", - }, - want: []string{ - "keboola.data-apps/12345", - "keboola.data-apps/123", - "keboola.data-apps/12", - }, - }, - { - name: "empty file", - args: args{ - content: "\n", - }, - want: nil, - }, - } - - for _, tt := range tests { - file := newFile(tt.args.content, projectState) - got := file.parseIgnoredPatterns() - assert.Equal(t, tt.want, got) - } } func newTestRegistry(t *testing.T) *registry.Registry { diff --git a/internal/pkg/project/ignore/ignore.go b/internal/pkg/project/ignore/ignore.go new file mode 100644 index 0000000000..98bb148030 --- /dev/null +++ b/internal/pkg/project/ignore/ignore.go @@ -0,0 +1,55 @@ +package ignore + +import ( + "strings" + + "github.com/keboola/keboola-as-code/internal/pkg/utils/errors" +) + +func (f *File) IgnoreConfigsOrRows() error { + return f.applyIgnoredPatterns() +} + +func (f *File) parseIgnoredPatterns() []string { + var ignorePatterns []string + lines := strings.Split(f.rawStringPattern, "\n") + for _, line := range lines { + trimmedLine := strings.TrimSpace(line) + // Skip empty lines and comments + if trimmedLine != "" && !strings.HasPrefix(trimmedLine, "#") { + ignorePatterns = append(ignorePatterns, trimmedLine) + } + } + + return ignorePatterns +} + +// applyIgnorePattern applies a single ignore pattern, marking the appropriate config or row as ignored. +func (f *File) applyIgnorePattern(ignoreConfig string) error { + parts := strings.Split(ignoreConfig, "/") + + switch len(parts) { + case 2: + // Ignore config by ID and name. + configID, componentID := parts[1], parts[0] + f.state.IgnoreConfig(configID, componentID) + case 3: + // Ignore specific config row. + configID, rowID := parts[1], parts[2] + f.state.IgnoreConfigRow(configID, rowID) + default: + return errors.Errorf("invalid ignore ignoreConfig format: %s", ignoreConfig) + } + + return nil +} + +// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows. +func (f *File) applyIgnoredPatterns() error { + for _, pattern := range f.parseIgnoredPatterns() { + if err := f.applyIgnorePattern(pattern); err != nil { + continue + } + } + return nil +} diff --git a/internal/pkg/project/ignore/ignore_test.go b/internal/pkg/project/ignore/ignore_test.go new file mode 100644 index 0000000000..933505eb77 --- /dev/null +++ b/internal/pkg/project/ignore/ignore_test.go @@ -0,0 +1,163 @@ +package ignore + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/keboola/keboola-as-code/internal/pkg/filesystem" + "github.com/keboola/keboola-as-code/internal/pkg/filesystem/aferofs" +) + +func TestFile_IgnoreConfigsOrRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + registry := newTestRegistry(t) + fs := aferofs.NewMemoryFs() + + require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile(`foo/bar1`, "keboola.bar/678/34\nkeboola.foo/345"))) + + file, err := LoadFile(ctx, fs, registry, "foo/bar1") + require.NoError(t, err) + + assert.NoError(t, file.IgnoreConfigsOrRows()) + + assert.Len(t, registry.IgnoredConfigRows(), 1) + assert.Len(t, registry.IgnoredConfigs(), 1) + assert.Equal(t, registry.IgnoredConfigRows()[0].ID.String(), "34") + assert.Equal(t, registry.IgnoredConfigs()[0].ID.String(), "345") +} + +func Test_applyIgnoredPatterns(t *testing.T) { + t.Parallel() + projectState := newTestRegistry(t) + + type args struct { + pattern string + } + + file := &File{ + state: projectState, + } + + tests := []struct { + name string + args args + wantErr assert.ErrorAssertionFunc + }{ + { + name: "empty patterns", + args: args{ + pattern: "", + }, + wantErr: assert.Error, + }, + { + name: "wrong pattern", + args: args{ + pattern: "wrong pattern", + }, + wantErr: assert.Error, + }, + { + name: "too long pattern", + args: args{ + pattern: "keboola.bar/687/1234/1234", + }, + wantErr: assert.Error, + }, + { + name: "short pattern", + args: args{ + pattern: "keboola.bar", + }, + wantErr: assert.Error, + }, + { + name: "correct pattern", + args: args{ + pattern: "keboola.bar/687", + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tt.wantErr(t, file.applyIgnorePattern(tt.args.pattern), fmt.Sprintf("applyIgnoredPatterns(%v)", tt.args.pattern)) + }) + } +} + +func Test_parseIgnoredPatterns(t *testing.T) { + t.Parallel() + + projectState := newTestRegistry(t) + + type args struct { + content string + } + tests := []struct { + name string + args args + want []string + }{ + { + name: "with comments", + args: args{ + content: "##dataapps\nkeboola.data-apps/12345\n##dataapps2\nkeboola.data-apps/123/rowId123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123/rowId123", + "keboola.data-apps/12", + }, + }, + { + name: "with empty lines", + args: args{ + content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123", + "keboola.data-apps/12", + }, + }, + { + name: "empty file", + args: args{ + content: "\n", + }, + want: nil, + }, + { + name: "error", + args: args{ + content: "##dataapps\n\n\n\n\n\nkeboola.data-apps/12345\n\t##dataapps2\nkeboola.data-apps/123\n##dataaps3\nkeboola.data-apps/12", + }, + want: []string{ + "keboola.data-apps/12345", + "keboola.data-apps/123", + "keboola.data-apps/12", + }, + }, + { + name: "empty file", + args: args{ + content: "\n", + }, + want: nil, + }, + } + + for _, tt := range tests { + file := newFile(tt.args.content, projectState) + got := file.parseIgnoredPatterns() + assert.Equal(t, tt.want, got) + } +} From 2c14c7783abac54d60b6f12e32664d6b2704cd74 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Nov 2024 10:34:49 +0100 Subject: [PATCH 10/11] refactor: deleted unnecessary code,recommended changes --- internal/pkg/project/ignore/file.go | 7 +------ internal/pkg/project/ignore/ignore.go | 20 ++++++++++---------- internal/pkg/state/registry/registry.go | 16 ---------------- internal/pkg/state/registry/registry_test.go | 15 --------------- 4 files changed, 11 insertions(+), 47 deletions(-) diff --git a/internal/pkg/project/ignore/file.go b/internal/pkg/project/ignore/file.go index 3cbeb7de74..48bc2b8716 100644 --- a/internal/pkg/project/ignore/file.go +++ b/internal/pkg/project/ignore/file.go @@ -32,10 +32,5 @@ func LoadFile(ctx context.Context, fs filesystem.Fs, state *state.Registry, path return nil, err } - f := &File{ - state: state, - rawStringPattern: content.Content, - } - - return f, nil + return newFile(content.Content, state), nil } diff --git a/internal/pkg/project/ignore/ignore.go b/internal/pkg/project/ignore/ignore.go index 98bb148030..897711445a 100644 --- a/internal/pkg/project/ignore/ignore.go +++ b/internal/pkg/project/ignore/ignore.go @@ -10,6 +10,16 @@ func (f *File) IgnoreConfigsOrRows() error { return f.applyIgnoredPatterns() } +// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows. +func (f *File) applyIgnoredPatterns() error { + for _, pattern := range f.parseIgnoredPatterns() { + if err := f.applyIgnorePattern(pattern); err != nil { + continue + } + } + return nil +} + func (f *File) parseIgnoredPatterns() []string { var ignorePatterns []string lines := strings.Split(f.rawStringPattern, "\n") @@ -43,13 +53,3 @@ func (f *File) applyIgnorePattern(ignoreConfig string) error { return nil } - -// applyIgnoredPatterns parses the content for ignore patterns and applies them to configurations or rows. -func (f *File) applyIgnoredPatterns() error { - for _, pattern := range f.parseIgnoredPatterns() { - if err := f.applyIgnorePattern(pattern); err != nil { - continue - } - } - return nil -} diff --git a/internal/pkg/state/registry/registry.go b/internal/pkg/state/registry/registry.go index 95eb9c289d..e250fba148 100644 --- a/internal/pkg/state/registry/registry.go +++ b/internal/pkg/state/registry/registry.go @@ -14,8 +14,6 @@ import ( type pathsRO = knownpaths.PathsReadOnly -const KBCIgnoreFilePath = ".keboola/kbc_ignore" - type Registry struct { *pathsRO paths *knownpaths.Paths @@ -235,20 +233,6 @@ func (s *Registry) ConfigRowsFrom(config ConfigKey) (rows []*ConfigRowState) { return rows } -// func (s *Registry) SetIgnoredConfigsOrRows(ctx context.Context, fs filesystem.Fs, path string) error { -// //strukturu pattre.field -// content, err := fs.ReadFile(ctx, filesystem.NewFileDef(path)) -// if err != nil { -// return err -// } -// -// if content.Content == "" { -// return nil -// } -// -// return s.applyIgnoredPatterns(content.Content) -//} - func (s *Registry) GetPath(key Key) (AbsPath, bool) { objectState, found := s.Get(key) if !found { diff --git a/internal/pkg/state/registry/registry_test.go b/internal/pkg/state/registry/registry_test.go index 0ad0bdb965..56b4039e16 100644 --- a/internal/pkg/state/registry/registry_test.go +++ b/internal/pkg/state/registry/registry_test.go @@ -175,21 +175,6 @@ func TestStateTrackRecordInvalid(t *testing.T) { assert.Empty(t, s.UntrackedPaths()) } -// func TestRegistry_SetIgnoredConfigsOrRows(t *testing.T) { -// t.Parallel() -// ctx := context.Background() -// registry := newTestState(t, knownpaths.NewNop(ctx)) -// fs := aferofs.NewMemoryFs() -// require.NoError(t, fs.WriteFile(ctx, filesystem.NewRawFile(`foo/bar1`, "keboola.bar/678/34\nkeboola.foo/345"))) -// -// require.NoError(t, registry.SetIgnoredConfigsOrRows(ctx, fs, "foo/bar1")) -// -// assert.Len(t, registry.IgnoredConfigRows(), 1) -// assert.Len(t, registry.IgnoredConfigs(), 1) -// assert.Equal(t, registry.IgnoredConfigRows()[0].ID.String(), "34") -// assert.Equal(t, registry.IgnoredConfigs()[0].ID.String(), "345") -//} - func TestRegistry_GetPath(t *testing.T) { t.Parallel() ctx := context.Background() From 2d060240bdde34cb6fbaae823087c9c44056e653 Mon Sep 17 00:00:00 2001 From: Petr Hosek Date: Tue, 12 Nov 2024 12:02:01 +0100 Subject: [PATCH 11/11] fix: removed useless conditions --- internal/pkg/state/registry/registry.go | 6 ++---- pkg/lib/operation/project/sync/pull/operation.go | 12 ++++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/internal/pkg/state/registry/registry.go b/internal/pkg/state/registry/registry.go index e250fba148..b666892a22 100644 --- a/internal/pkg/state/registry/registry.go +++ b/internal/pkg/state/registry/registry.go @@ -158,10 +158,8 @@ func (s *Registry) IgnoreConfig(ignoreID string, componentID string) { v.Ignore = true // ignore rows of the configuration - if len(s.ConfigRowsFrom(v.ConfigKey)) > 0 { - for _, configRowState := range s.ConfigRowsFrom(v.ConfigKey) { - configRowState.Ignore = true - } + for _, configRowState := range s.ConfigRowsFrom(v.ConfigKey) { + configRowState.Ignore = true } } } diff --git a/pkg/lib/operation/project/sync/pull/operation.go b/pkg/lib/operation/project/sync/pull/operation.go index e200611b65..4d15140a1f 100644 --- a/pkg/lib/operation/project/sync/pull/operation.go +++ b/pkg/lib/operation/project/sync/pull/operation.go @@ -135,15 +135,11 @@ func Run(ctx context.Context, projectState *project.State, o Options, d dependen } func ignoreConfigsAndRows(projectState *project.State) { - if len(projectState.IgnoredConfigRows()) > 0 { - for _, v := range projectState.IgnoredConfigRows() { - v.SetRemoteState(nil) - } + for _, v := range projectState.IgnoredConfigRows() { + v.SetRemoteState(nil) } - if len(projectState.IgnoredConfigs()) > 0 { - for _, v := range projectState.IgnoredConfigs() { - v.SetRemoteState(nil) - } + for _, v := range projectState.IgnoredConfigs() { + v.SetRemoteState(nil) } }