Merge pull request #2131 from keboola/jt-psgo-918-rollback

fix: Preserve state on rollback
keboola · Nov 4, 2024 · d25244c · d25244c
2 parents c89867f + 4e26407
commit d25244c
Show file tree

Hide file tree

Showing 239 changed files with 796 additions and 180 deletions.
diff --git a/internal/pkg/service/stream/definition/repository/sink/sink_enable_test.go b/internal/pkg/service/stream/definition/repository/sink/sink_enable_test.go
@@ -106,7 +106,7 @@ func TestSinkRepository_Enable(t *testing.T) {
 		now = now.Add(time.Hour)
 		sink, err := repo.RollbackVersion(sinkKey, now, by, 2).Do(ctx).ResultOrErr()
 		require.NoError(t, err)
-		assert.True(t, sink.IsDisabled())
+		assert.False(t, sink.IsDisabled()) // Rollback intentionally doesn't change disabled/enabled state
 		assert.Equal(t, definition.VersionNumber(4), sink.VersionNumber())
 	}
 
@@ -293,6 +293,13 @@ func TestSinkRepository_EnableSinksOnSourceEnable(t *testing.T) {
 		source, err := d.DefinitionRepository().Source().RollbackVersion(sourceKey, now, by, 2).Do(ctx).ResultOrErr()
 		require.NoError(t, err)
 		assert.Equal(t, definition.VersionNumber(4), source.Version.Number)
+		assert.False(t, source.IsDisabled()) // Rollback intentionally doesn't change disabled/enabled state
+	}
+	{
+		// Disable the Source
+		now = now.Add(time.Hour)
+		source, err := d.DefinitionRepository().Source().Disable(sourceKey, now, by, "test").Do(ctx).ResultOrErr()
+		require.NoError(t, err)
 		assert.True(t, source.IsDisabled())
 	}
 	{

diff --git a/internal/pkg/service/stream/definition/repository/sink/sink_version.go b/internal/pkg/service/stream/definition/repository/sink/sink_version.go
@@ -54,6 +54,15 @@ func (r *Repository) RollbackVersion(k key.SinkKey, now time.Time, by definition
 			// Prepare the new value
 			versionDescription := fmt.Sprintf(`Rollback to version "%d".`, targetVersion.Version.Number)
 			updated = deepcopy.Copy(*targetVersion).(definition.Sink)
+
+			// Preserve Disabled and Deleted states from the latest version.
+			// Without this it would be possible to bypass the Enable/Undelete event and avoid billing.
+			updated.Disabled = deepcopy.Copy(latestVersion.Disabled).(*definition.Disabled)
+			updated.Enabled = deepcopy.Copy(latestVersion.Enabled).(*definition.Enabled)
+			updated.Deleted = deepcopy.Copy(latestVersion.Deleted).(*definition.Deleted)
+			updated.Undeleted = deepcopy.Copy(latestVersion.Undeleted).(*definition.Undeleted)
+
+			// Increase version
 			updated.Version = latestVersion.Version
 			updated.IncrementVersion(updated, now, by, versionDescription)
 			return r.save(ctx, now, by, latestVersion, &updated)

diff --git a/internal/pkg/service/stream/definition/repository/source/source_version.go b/internal/pkg/service/stream/definition/repository/source/source_version.go
@@ -54,6 +54,15 @@ func (r *Repository) RollbackVersion(k key.SourceKey, now time.Time, by definiti
 			// Prepare the new value
 			versionDescription := fmt.Sprintf(`Rollback to version "%d".`, targetVersion.Version.Number)
 			updated = deepcopy.Copy(*targetVersion).(definition.Source)
+
+			// Preserve Disabled and Deleted states from the latest version.
+			// Without this it would be possible to bypass the Enable/Undelete event and avoid billing.
+			updated.Disabled = deepcopy.Copy(latestVersion.Disabled).(*definition.Disabled)
+			updated.Enabled = deepcopy.Copy(latestVersion.Enabled).(*definition.Enabled)
+			updated.Deleted = deepcopy.Copy(latestVersion.Deleted).(*definition.Deleted)
+			updated.Undeleted = deepcopy.Copy(latestVersion.Undeleted).(*definition.Undeleted)
+
+			// Increase version
 			updated.Version = latestVersion.Version
 			updated.IncrementVersion(updated, now, by, versionDescription)
 			return r.save(ctx, now, by, latestVersion, &updated)

diff --git a/test/stream/api/sink/create-001-minimal-and-get/001-create-source/request.json b/test/stream/api/sink/create-001-minimal-and-get/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-001-minimal-and-get/003-create-sink/request.json b/test/stream/api/sink/create-001-minimal-and-get/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-002-with-id-and-get/001-create-source/request.json b/test/stream/api/sink/create-002-with-id-and-get/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-002-with-id-and-get/003-create-sink/request.json b/test/stream/api/sink/create-002-with-id-and-get/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-003-weird-id-and-get/001-create-source/request.json b/test/stream/api/sink/create-003-weird-id-and-get/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-003-weird-id-and-get/003-create-sink/request.json b/test/stream/api/sink/create-003-weird-id-and-get/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-004-with-description-and-get/001-create-source/request.json b/test/stream/api/sink/create-004-with-description-and-get/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-004-with-description-and-get/003-create-sink/request.json b/test/stream/api/sink/create-004-with-description-and-get/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-005-invalid-type-400/001-create-source/request.json b/test/stream/api/sink/create-005-invalid-type-400/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-005-invalid-type-400/003-create-sink/request.json b/test/stream/api/sink/create-005-invalid-type-400/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-006-id-empty-400/001-create-source/request.json b/test/stream/api/sink/create-006-id-empty-400/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-006-id-empty-400/003-create-sink/request.json b/test/stream/api/sink/create-006-id-empty-400/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-006-id-max-length-400/001-create-source/request.json b/test/stream/api/sink/create-006-id-max-length-400/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-006-id-max-length-400/003-create-sink/request.json b/test/stream/api/sink/create-006-id-max-length-400/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-008-name-empty-400/001-create-source/request.json b/test/stream/api/sink/create-008-name-empty-400/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-008-name-empty-400/003-create-sink/request.json b/test/stream/api/sink/create-008-name-empty-400/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-009-name-max-length-400/001-create-source/request.json b/test/stream/api/sink/create-009-name-max-length-400/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-009-name-max-length-400/003-create-sink/request.json b/test/stream/api/sink/create-009-name-max-length-400/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-010-missing-storage-token-400/001-create-sink/request.json b/test/stream/api/sink/create-010-missing-storage-token-400/001-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json"

diff --git a/test/stream/api/sink/create-011-exists-409/001-create-source/request.json b/test/stream/api/sink/create-011-exists-409/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-011-exists-409/003-create-sink/request.json b/test/stream/api/sink/create-011-exists-409/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-011-exists-409/005-create-sink-exists/request.json b/test/stream/api/sink/create-011-exists-409/005-create-sink-exists/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-012-invalid-jsonnet-template/001-create-source/request.json b/test/stream/api/sink/create-012-invalid-jsonnet-template/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-012-invalid-jsonnet-template/003-create-sink/request.json b/test/stream/api/sink/create-012-invalid-jsonnet-template/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-013-duplicite-creation/001-create-source/request.json b/test/stream/api/sink/create-013-duplicite-creation/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-013-duplicite-creation/003-create-sink/request.json b/test/stream/api/sink/create-013-duplicite-creation/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-013-duplicite-creation/004-create-duplicate-sink/request.json b/test/stream/api/sink/create-013-duplicite-creation/004-create-duplicate-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-013-in-parallel/001-create-source/request.json b/test/stream/api/sink/create-013-in-parallel/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-013-in-parallel/003-create-sink-1/request.json b/test/stream/api/sink/create-013-in-parallel/003-create-sink-1/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-013-in-parallel/003-create-sink-2/request.json b/test/stream/api/sink/create-013-in-parallel/003-create-sink-2/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-013-in-parallel/003-create-sink-3/request.json b/test/stream/api/sink/create-013-in-parallel/003-create-sink-3/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-014-disable-enable/001-create-source/request.json b/test/stream/api/sink/create-014-disable-enable/001-create-source/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-014-disable-enable/003-create-sink/request.json b/test/stream/api/sink/create-014-disable-enable/003-create-sink/request.json
@@ -1,5 +1,5 @@
 {
-  "path": "/v1/branches/default/sources/my-source/sinks",
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
   "method": "POST",
   "headers": {
     "Content-Type": "application/json",

diff --git a/test/stream/api/sink/create-015-disable-rollback/001-create-source/expected-http-code b/test/stream/api/sink/create-015-disable-rollback/001-create-source/expected-http-code
@@ -0,0 +1 @@
+202
diff --git a/test/stream/api/sink/create-015-disable-rollback/001-create-source/expected-response.json b/test/stream/api/sink/create-015-disable-rollback/001-create-source/expected-response.json
@@ -0,0 +1,8 @@
+{
+  "taskId": "api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-source/%s",
+  "type": "api.create.source",
+  "url": "https://stream.keboola.local/v1/tasks/api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-source/%s",
+  "status": "processing",
+  "isFinished": false,
+  "createdAt": "%s"
+}
diff --git a/test/stream/api/sink/create-015-disable-rollback/001-create-source/request.json b/test/stream/api/sink/create-015-disable-rollback/001-create-source/request.json
@@ -0,0 +1,12 @@
+{
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources",
+  "method": "POST",
+  "headers": {
+    "Content-Type": "application/json",
+    "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%"
+  },
+  "body": {
+    "name": "My Source",
+    "type": "http"
+  }
+}
diff --git a/...ete/005-list-instances/expected-http-code → ...llback/002-poll-source/expected-http-code b/...ete/005-list-instances/expected-http-code → ...llback/002-poll-source/expected-http-code
diff --git a/test/stream/api/sink/create-015-disable-rollback/002-poll-source/expected-response.json b/test/stream/api/sink/create-015-disable-rollback/002-poll-source/expected-response.json
@@ -0,0 +1,17 @@
+{
+  "taskId": "api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-source/%s",
+  "type": "api.create.source",
+  "url": "https://stream.keboola.local/v1/tasks/api.create.source/%%TEST_DEFAULT_BRANCH_ID%%/my-source/%s",
+  "status": "success",
+  "isFinished": true,
+  "createdAt": "%s",
+  "finishedAt": "%s",
+  "duration": %d,
+  "result": "Source has been created successfully.",
+  "outputs": {
+    "url": "https://stream.keboola.local/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source",
+    "projectId": %%TEST_KBC_PROJECT_ID%%,
+    "branchId": %%TEST_DEFAULT_BRANCH_ID%%,
+    "sourceId": "my-source"
+  }
+}
diff --git a/test/stream/api/sink/create-015-disable-rollback/002-poll-source/request.json b/test/stream/api/sink/create-015-disable-rollback/002-poll-source/request.json
@@ -0,0 +1,11 @@
+{
+  "path": "<<001-create-source:response.url>>",
+  "method": "GET",
+  "headers": {
+    "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%"
+  },
+  "repeat": {
+    "until": "status != 'processing'",
+    "timeout": 60
+  }
+}
diff --git a/test/stream/api/sink/create-015-disable-rollback/003-create-sink/expected-http-code b/test/stream/api/sink/create-015-disable-rollback/003-create-sink/expected-http-code
@@ -0,0 +1 @@
+202
diff --git a/test/stream/api/sink/create-015-disable-rollback/003-create-sink/expected-response.json b/test/stream/api/sink/create-015-disable-rollback/003-create-sink/expected-response.json
@@ -0,0 +1,8 @@
+{
+  "taskId": "api.create.sink/%%TEST_DEFAULT_BRANCH_ID%%/my-source/my-sink/%s",
+  "type": "api.create.sink",
+  "url": "https://stream.keboola.local/v1/tasks/api.create.sink/%%TEST_DEFAULT_BRANCH_ID%%/my-source/my-sink/%s",
+  "status": "processing",
+  "isFinished": false,
+  "createdAt": "%s"
+}
diff --git a/test/stream/api/sink/create-015-disable-rollback/003-create-sink/request.json b/test/stream/api/sink/create-015-disable-rollback/003-create-sink/request.json
@@ -0,0 +1,28 @@
+{
+  "path": "/v1/branches/%%TEST_DEFAULT_BRANCH_ID%%/sources/my-source/sinks",
+  "method": "POST",
+  "headers": {
+    "Content-Type": "application/json",
+    "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%"
+  },
+  "body": {
+    "name": "My Sink",
+    "type": "table",
+    "table": {
+      "type": "keboola",
+      "tableId": "in.c-my-bucket.my-table",
+      "mapping": {
+        "columns": [
+          {
+            "type": "uuid",
+            "name": "id"
+          },
+          {
+            "type": "body",
+            "name": "body"
+          }
+        ]
+      }
+    }
+  }
+}
diff --git a/test/stream/api/sink/create-015-disable-rollback/004-poll-sink/expected-http-code b/test/stream/api/sink/create-015-disable-rollback/004-poll-sink/expected-http-code
@@ -0,0 +1 @@
+200