WIP

internal/pkg/service/templates/api/service/service.go

@@ -3,10 +3,6 @@ package service
 import (
 	"context"
 	"fmt"
-	"github.com/keboola/keboola-as-code/internal/pkg/encoding/jsonnet"
-	"github.com/keboola/keboola-as-code/internal/pkg/encoding/jsonnet/fsimporter"
-	"github.com/keboola/keboola-as-code/internal/pkg/filesystem/aferofs/mountfs"
-	"github.com/keboola/keboola-as-code/internal/pkg/template/jsonnet/function"
 	"net/http"
 	"path"
 	"strconv"
@@ -261,7 +257,7 @@ func (s *service) UseTemplateVersion(ctx context.Context, d dependencies.Project
 	}
 	defer unlockFn(ctx)
 
-	s.mapper
+	/*s.mapper
 	mountPoint := mountfs.NewMountPoint(repository.CommonDirectoryMountPoint, commonDir)
 	templateDir, err := aferofs.NewMountFs(templateDir, []mountfs.MountPoint{mountPoint})
 	if err != nil {
@@ -277,7 +273,8 @@ func (s *service) UseTemplateVersion(ctx context.Context, d dependencies.Project
 	}
 	jsonnetCtx := jsonnet.NewContext().WithCtx(ctx).WithImporter(fsimporter.New(objectsRoot))
 
-	jsonnetCtx.NativeFunctionWithAlias(function.SnowflakeWriterComponentID(d.Components()))
+	jsonnetCtx.NativeFunctionWithAlias(function.SnowflakeWriterComponentID(d.Components()))*/
+
 	// Note:
 	//   A very strange code follows.
 	//   Since I did not manage to complete the refactoring - separation of remote and local state.

internal/pkg/service/templates/dependencies/api.go

@@ -27,7 +27,6 @@ const (
 // apiScope implements APIScope interface.
 type apiScope struct {
 	dependencies.BaseScope
-	dependencies.ProjectScope
 	dependencies.PublicScope
 	dependencies.EtcdClientScope
 	dependencies.DistributionScope
@@ -42,7 +41,6 @@ type apiScope struct {
 
 type parentScopes struct {
 	dependencies.BaseScope
-	dependencies.ProjectScope
 	dependencies.PublicScope
 	dependencies.EtcdClientScope
 	dependencies.DistributionScope
@@ -132,8 +130,6 @@ func newAPIScope(ctx context.Context, p *parentScopes, cfg config.Config) (v *ap
 
 	d.BaseScope = p.BaseScope
 
-	d.ProjectScope = p.ProjectScope
-
 	d.PublicScope = p.PublicScope
 
 	d.EtcdClientScope = p.EtcdClientScope

internal/pkg/template/context/load/context.go

@@ -43,6 +43,6 @@ func (c *Context) JsonnetContext() *jsonnet.Context {
 
 func (c *Context) registerJsonnetFunctions() {
 	c.jsonnetCtx.NativeFunctionWithAlias(function.ComponentIsAvailable(c.components))
-	//c.jsonnetCtx.NativeFunctionWithAlias(function.SnowflakeWriterComponentID(c.components))
+	c.jsonnetCtx.NativeFunctionWithAlias(function.SnowflakeWriterComponentID(c.components))
 	c.jsonnetCtx.NativeFunctionWithAlias(function.HasProjectBackend(c.projectBackends))
 }

internal/pkg/template/repository/manager/manager.go

@@ -17,7 +17,6 @@ package manager
 
 import (
 	"context"
-	"fmt"
 	"sort"
 	"sync"
 	"time"
@@ -60,7 +59,6 @@ type dependencies interface {
 	Telemetry() telemetry.Telemetry
 	Process() *servicectx.Process
 	Components() *model.ComponentsMap
-	ProjectBackends() []string
 }
 
 func New(ctx context.Context, d dependencies, defaultRepositories []model.TemplateRepository) (*Manager, error) {
@@ -74,7 +72,7 @@ func New(ctx context.Context, d dependencies, defaultRepositories []model.Templa
 		repositoriesInit:    &singleflight.Group{},
 		repositoriesLock:    &sync.RWMutex{},
 	}
-	fmt.Println("BB: ", d.ProjectBackends())
+
 	// Free all repositories on server shutdown
 	d.Process().OnShutdown(func(ctx context.Context) {
 		m.Free(ctx)

internal/pkg/template/repository/manager/repository.go

@@ -108,7 +108,7 @@ func (r *CachedRepository) Template(ctx context.Context, reference model.Templat
 
 		// Load template
 		r.d.Logger().Infof(ctx, r.repo.Fs().BasePath())
-		tmpl, err := loadTemplateOp.Run(ctx, r.d, r.repo, reference, template.Option{ProjectBackend: r.d.ProjectBackends()})
+		tmpl, err := loadTemplateOp.Run(ctx, r.d, r.repo, reference, template.Option{})
 		if err != nil {
 			return nil, errors.Errorf(`cannot load template "%s": %w`, reference.FullName(), err)
 		}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/001-ok/expected-http-code

@@ -0,0 +1 @@
+202

test/templates/api/repositories/template-use-wr-bigquery-snowflake/001-ok/expected-response.json

@@ -0,0 +1,8 @@
+{
+  "id": "template.use/%s",
+  "type": "template.use",
+  "url": "https://templates.keboola.local/v1/tasks/template.use/%s",
+  "status": "processing",
+  "isFinished": false,
+  "createdAt": "%s"
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/001-ok/request.json

@@ -0,0 +1,23 @@
+{
+  "path": "/v1/repositories/keboola/templates/my-template-id/1.2.3/use",
+  "method": "POST",
+  "headers": {
+    "Content-Type": "application/json",
+    "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%"
+  },
+  "body": {
+    "branch": "default",
+    "name": "inst-001",
+    "steps": [
+      {
+        "id": "g01-s01",
+        "inputs": [
+          {
+            "id": "wr-snowflake-host",
+            "value": "my-host"
+          }
+        ]
+      }
+    ]
+  }
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/002-poll/expected-http-code

@@ -0,0 +1 @@
+200

test/templates/api/repositories/template-use-wr-bigquery-snowflake/002-poll/expected-response.json

@@ -0,0 +1,14 @@
+{
+  "id": "template.use/%s",
+  "type": "template.use",
+  "url": "https://templates.keboola.local/v1/tasks/template.use/%s",
+  "status": "success",
+  "isFinished": true,
+  "createdAt": "%s",
+  "finishedAt": "%s",
+  "duration": %d,
+  "result": "template instance with id \"%s\" created",
+  "outputs": {
+    "instanceId": "%s"
+  }
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/002-poll/request.json

@@ -0,0 +1,11 @@
+{
+  "path": "<<001-ok:response.url>>",
+  "method": "GET",
+  "headers": {
+    "X-StorageApi-Token": "%%TEST_KBC_STORAGE_API_TOKEN%%"
+  },
+  "repeat": {
+    "until": "status != 'processing'",
+    "timeout": 60
+  }
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/expected-state.json

@@ -0,0 +1,51 @@
+{
+  "branches": [
+    {
+      "branch": {
+        "name": "Main",
+        "description": "",
+        "isDefault": true,
+        "metadata": {
+          "KBC.KAC.templates.instances": "[{\"instanceId\":\"%s\",\"instanceName\":\"inst-001\",\"templateId\":\"my-template-id\",\"repositoryName\":\"keboola\",\"version\":\"1.2.3\",\"created\":{\"date\":\"%s\",\"tokenId\":\"%s\"},\"updated\":{\"date\":\"%s\",\"tokenId\":\"%s\"}}]"
+        }
+      },
+      "configs": [
+        {
+          "componentId": "keboola.%s",
+          "name": "Data Destination",
+          "description": "## Description:\nData load to Snowflake DB.",
+          "changeDescription": "From template keboola/my-template-id/1.2.3",
+          "configuration": {
+            "parameters": {
+              "db": {
+                "host": "my-host"
+              }
+            }
+          },
+          "rows": [
+            {
+              "name": "activity",
+              "description": "",
+              "changeDescription": "From template keboola/my-template-id/1.2.3",
+              "isDisabled": false,
+              "configuration": {
+                "parameters": {
+                  "foo": "bar"
+                }
+              }
+            }
+          ],
+          "metadata": {
+            "KBC.KAC.templates.configId": "{\"idInTemplate\":\"destination\"}",
+            "KBC.KAC.templates.configInputs": "[{\"input\":\"wr-snowflake-host\",\"key\":\"parameters.db.host\"}]",
+            "KBC.KAC.templates.instanceId": "%s",
+            "KBC.KAC.templates.repository": "keboola",
+            "KBC.KAC.templates.rowsIds": "[{\"idInProject\":\"%s\",\"idInTemplate\":\"activity\"}]",
+            "KBC.KAC.templates.templateId": "my-template-id"
+          },
+          "isDisabled": false
+        }
+      ]
+    }
+  ]
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/initial-state.json

@@ -0,0 +1,5 @@
+{
+  "backend": {
+    "type": "bigquery"
+  }
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/.keboola/repository.json

@@ -0,0 +1,23 @@
+{
+  "version": 2,
+  "author": {
+    "name": "Example Author",
+    "url": "https://example.com"
+  },
+  "templates": [
+    {
+      "id": "my-template-id",
+      "name": "My Template",
+      "description": "Full workflow to ...",
+      "path": "my-template",
+      "versions": [
+        {
+          "version": "1.2.3",
+          "description": "",
+          "stable": false,
+          "path": "v1"
+        }
+      ]
+    }
+  ]
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/README.md

@@ -0,0 +1,4 @@
+### My Template
+
+Full workflow to ...
+

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/inputs.jsonnet

@@ -0,0 +1,24 @@
+{
+  stepsGroups: [
+    {
+      description: "Configure the Snowflake",
+      required: "all",
+      steps: [
+        {
+          icon: "component:"+SnowflakeWriterComponentId(),
+          name: "Snowflake",
+          description: "Destination",
+          inputs: [
+             {
+                id: "wr-snowflake-host",
+                name: "Hostname",
+                description: "Insert database hostname",
+                type: "string",
+                kind: "input",
+              },
+          ]
+        },
+      ],
+    },
+  ],
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/manifest.jsonnet

@@ -0,0 +1,11 @@
+{
+  configurations: [
+    if HasProjectBackend('snowflake') == true then
+      {
+        componentId: SnowflakeWriterComponentId(),
+        id: ConfigId('destination'),
+        path: 'writer/keboola.wr-snowflake/destination',
+        rows: [],
+      },
+  ],
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/transformation/keboola.bigquery/destination2/config.jsonnet

@@ -0,0 +1,7 @@
+{
+  parameters: {
+    db: {
+      host: 'abc',
+    },
+  },
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/transformation/keboola.bigquery/destination2/description.md

@@ -0,0 +1,2 @@
+## Description:
+Data load to Bigquery DB.

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/transformation/keboola.bigquery/destination2/meta.jsonnet

@@ -0,0 +1,3 @@
+{
+  name: 'Data Destination2',
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/writer/keboola.wr-snowflake/destination/config.jsonnet

@@ -0,0 +1,7 @@
+{
+  parameters: {
+    db: {
+      host: Input("wr-snowflake-host"),
+    },
+  },
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/writer/keboola.wr-snowflake/destination/description.md

@@ -0,0 +1,2 @@
+## Description:
+Data load to Snowflake DB.

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/writer/keboola.wr-snowflake/destination/meta.jsonnet

@@ -0,0 +1,3 @@
+{
+  name: "Data Destination",
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/writer/keboola.wr-snowflake/destination/rows/activity/config.jsonnet

@@ -0,0 +1,5 @@
+{
+  parameters: {
+    "foo": "bar",
+  },
+}

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/writer/keboola.wr-snowflake/destination/rows/activity/description.md

@@ -0,0 +1 @@
+

test/templates/api/repositories/template-use-wr-bigquery-snowflake/repository/my-template/v1/src/writer/keboola.wr-snowflake/destination/rows/activity/meta.jsonnet

@@ -0,0 +1,4 @@
+{
+  name: "activity",
+  isDisabled: false,
+}